1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/Attributor.h"
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/DenseMapInfo.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/SCCIterator.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SetOperations.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallPtrSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/Statistic.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/Analysis/AssumeBundleQueries.h"
31#include "llvm/Analysis/AssumptionCache.h"
32#include "llvm/Analysis/CaptureTracking.h"
33#include "llvm/Analysis/CycleAnalysis.h"
34#include "llvm/Analysis/InstructionSimplify.h"
35#include "llvm/Analysis/LazyValueInfo.h"
36#include "llvm/Analysis/MemoryBuiltins.h"
37#include "llvm/Analysis/ScalarEvolution.h"
38#include "llvm/Analysis/TargetTransformInfo.h"
39#include "llvm/Analysis/ValueTracking.h"
40#include "llvm/IR/Argument.h"
41#include "llvm/IR/Assumptions.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/BasicBlock.h"
44#include "llvm/IR/Constant.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DataLayout.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
50#include "llvm/IR/InlineAsm.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicInst.h"
55#include "llvm/IR/IntrinsicsAMDGPU.h"
56#include "llvm/IR/IntrinsicsNVPTX.h"
57#include "llvm/IR/LLVMContext.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/NoFolder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueHandle.h"
62#include "llvm/Support/Alignment.h"
63#include "llvm/Support/Casting.h"
64#include "llvm/Support/CommandLine.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/GraphWriter.h"
67#include "llvm/Support/InterleavedRange.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/TypeSize.h"
71#include "llvm/Support/raw_ostream.h"
72#include "llvm/Transforms/Utils/BasicBlockUtils.h"
73#include "llvm/Transforms/Utils/CallPromotionUtils.h"
74#include "llvm/Transforms/Utils/Local.h"
75#include "llvm/Transforms/Utils/ValueMapper.h"
76#include <cassert>
77#include <numeric>
78#include <optional>
79#include <string>
80
81using namespace llvm;
82
83#define DEBUG_TYPE "attributor"
84
85static cl::opt<bool> ManifestInternal(
86 "attributor-manifest-internal", cl::Hidden,
87 cl::desc("Manifest Attributor internal string attributes."),
88 cl::init(Val: false));
89
90static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(Val: 128),
91 cl::Hidden);
92
93template <>
94unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
95
96template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
97
98static cl::opt<unsigned, true> MaxPotentialValues(
99 "attributor-max-potential-values", cl::Hidden,
100 cl::desc("Maximum number of potential values to be "
101 "tracked for each position."),
102 cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues),
103 cl::init(Val: 7));
104
105static cl::opt<int> MaxPotentialValuesIterations(
106 "attributor-max-potential-values-iterations", cl::Hidden,
107 cl::desc(
108 "Maximum number of iterations we keep dismantling potential values."),
109 cl::init(Val: 64));
110
111STATISTIC(NumAAs, "Number of abstract attributes created");
112STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
113
114// Some helper macros to deal with statistics tracking.
115//
116// Usage:
117// For simple IR attribute tracking overload trackStatistics in the abstract
118// attribute and choose the right STATS_DECLTRACK_********* macro,
119// e.g.,:
120// void trackStatistics() const override {
121// STATS_DECLTRACK_ARG_ATTR(returned)
122// }
123// If there is a single "increment" side one can use the macro
124// STATS_DECLTRACK with a custom message. If there are multiple increment
125// sides, STATS_DECL and STATS_TRACK can also be used separately.
126//
127#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
128 ("Number of " #TYPE " marked '" #NAME "'")
129#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
130#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
131#define STATS_DECL(NAME, TYPE, MSG) \
132 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
133#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
134#define STATS_DECLTRACK(NAME, TYPE, MSG) \
135 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
136#define STATS_DECLTRACK_ARG_ATTR(NAME) \
137 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
138#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
139 STATS_DECLTRACK(NAME, CSArguments, \
140 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
141#define STATS_DECLTRACK_FN_ATTR(NAME) \
142 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
143#define STATS_DECLTRACK_CS_ATTR(NAME) \
144 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
145#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
146 STATS_DECLTRACK(NAME, FunctionReturn, \
147 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
148#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
149 STATS_DECLTRACK(NAME, CSReturn, \
150 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
151#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
152 STATS_DECLTRACK(NAME, Floating, \
153 ("Number of floating values known to be '" #NAME "'"))
154
155// Specialization of the operator<< for abstract attributes subclasses. This
156// disambiguates situations where multiple operators are applicable.
157namespace llvm {
158#define PIPE_OPERATOR(CLASS) \
159 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
160 return OS << static_cast<const AbstractAttribute &>(AA); \
161 }
162
163PIPE_OPERATOR(AAIsDead)
164PIPE_OPERATOR(AANoUnwind)
165PIPE_OPERATOR(AANoSync)
166PIPE_OPERATOR(AANoRecurse)
167PIPE_OPERATOR(AANonConvergent)
168PIPE_OPERATOR(AAWillReturn)
169PIPE_OPERATOR(AANoReturn)
170PIPE_OPERATOR(AANonNull)
171PIPE_OPERATOR(AAMustProgress)
172PIPE_OPERATOR(AANoAlias)
173PIPE_OPERATOR(AADereferenceable)
174PIPE_OPERATOR(AAAlign)
175PIPE_OPERATOR(AAInstanceInfo)
176PIPE_OPERATOR(AANoCapture)
177PIPE_OPERATOR(AAValueSimplify)
178PIPE_OPERATOR(AANoFree)
179PIPE_OPERATOR(AAHeapToStack)
180PIPE_OPERATOR(AAIntraFnReachability)
181PIPE_OPERATOR(AAMemoryBehavior)
182PIPE_OPERATOR(AAMemoryLocation)
183PIPE_OPERATOR(AAValueConstantRange)
184PIPE_OPERATOR(AAPrivatizablePtr)
185PIPE_OPERATOR(AAUndefinedBehavior)
186PIPE_OPERATOR(AAPotentialConstantValues)
187PIPE_OPERATOR(AAPotentialValues)
188PIPE_OPERATOR(AANoUndef)
189PIPE_OPERATOR(AANoFPClass)
190PIPE_OPERATOR(AACallEdges)
191PIPE_OPERATOR(AAInterFnReachability)
192PIPE_OPERATOR(AAPointerInfo)
193PIPE_OPERATOR(AAAssumptionInfo)
194PIPE_OPERATOR(AAUnderlyingObjects)
195PIPE_OPERATOR(AAInvariantLoadPointer)
196PIPE_OPERATOR(AAAddressSpace)
197PIPE_OPERATOR(AANoAliasAddrSpace)
198PIPE_OPERATOR(AAAllocationInfo)
199PIPE_OPERATOR(AAIndirectCallInfo)
200PIPE_OPERATOR(AAGlobalValueInfo)
201PIPE_OPERATOR(AADenormalFPMath)
202
203#undef PIPE_OPERATOR
204
205template <>
206ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
207 const DerefState &R) {
208 ChangeStatus CS0 =
209 clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
210 ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
211 return CS0 | CS1;
212}
213
214} // namespace llvm
215
216static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
217 bool HeaderOnly, CycleRef *CPtr = nullptr) {
218 if (!CI)
219 return true;
220 auto *BB = I->getParent();
221 CycleRef C = CI->getCycle(Block: BB);
222 if (!C)
223 return false;
224 if (CPtr)
225 *CPtr = C;
226 return !HeaderOnly || BB == CI->getHeader(C);
227}
228
229/// Checks if a type could have padding bytes.
230static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
231 // There is no size information, so be conservative.
232 if (!Ty->isSized())
233 return false;
234
235 // If the alloc size is not equal to the storage size, then there are padding
236 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
237 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
238 return false;
239
240 // FIXME: This isn't the right way to check for padding in vectors with
241 // non-byte-size elements.
242 if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
243 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
244
245 // For array types, check for padding within members.
246 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
247 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
248
249 if (!isa<StructType>(Val: Ty))
250 return true;
251
252 // Check for padding within and between elements of a struct.
253 StructType *StructTy = cast<StructType>(Val: Ty);
254 const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
255 uint64_t StartPos = 0;
256 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
257 Type *ElTy = StructTy->getElementType(N: I);
258 if (!isDenselyPacked(Ty: ElTy, DL))
259 return false;
260 if (StartPos != Layout->getElementOffsetInBits(Idx: I))
261 return false;
262 StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
263 }
264
265 return true;
266}
267
268/// Get pointer operand of memory accessing instruction. If \p I is
269/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
270/// is set to false and the instruction is volatile, return nullptr.
271static const Value *getPointerOperand(const Instruction *I,
272 bool AllowVolatile) {
273 if (!AllowVolatile && I->isVolatile())
274 return nullptr;
275
276 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
277 return LI->getPointerOperand();
278 }
279
280 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
281 return SI->getPointerOperand();
282 }
283
284 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
285 return CXI->getPointerOperand();
286 }
287
288 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
289 return RMWI->getPointerOperand();
290 }
291
292 return nullptr;
293}
294
295/// Helper function to create a pointer based on \p Ptr, and advanced by \p
296/// Offset bytes.
297static Value *constructPointer(Value *Ptr, int64_t Offset,
298 IRBuilder<NoFolder> &IRB) {
299 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
300 << "-bytes\n");
301
302 if (Offset)
303 Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset),
304 Name: Ptr->getName() + ".b" + Twine(Offset));
305 return Ptr;
306}
307
308static const Value *
309stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
310 const Value *Val, const DataLayout &DL, APInt &Offset,
311 bool GetMinOffset, bool AllowNonInbounds,
312 bool UseAssumed = false) {
313
314 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
315 const IRPosition &Pos = IRPosition::value(V);
316 // Only track dependence if we are going to use the assumed info.
317 const AAValueConstantRange *ValueConstantRangeAA =
318 A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
319 DepClass: UseAssumed ? DepClassTy::OPTIONAL
320 : DepClassTy::NONE);
321 if (!ValueConstantRangeAA)
322 return false;
323 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
324 : ValueConstantRangeAA->getKnown();
325 if (Range.isFullSet())
326 return false;
327
328 // We can only use the lower part of the range because the upper part can
329 // be higher than what the value can really be.
330 if (GetMinOffset)
331 ROffset = Range.getSignedMin();
332 else
333 ROffset = Range.getSignedMax();
334 return true;
335 };
336
337 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
338 /* AllowInvariant */ AllowInvariantGroup: true,
339 ExternalAnalysis: AttributorAnalysis);
340}
341
342static const Value *
343getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
344 const Value *Ptr, int64_t &BytesOffset,
345 const DataLayout &DL, bool AllowNonInbounds = false) {
346 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
347 const Value *Base =
348 stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
349 /* GetMinOffset */ true, AllowNonInbounds);
350
351 BytesOffset = OffsetAPInt.getSExtValue();
352 return Base;
353}
354
355/// Clamp the information known for all returned values of a function
356/// (identified by \p QueryingAA) into \p S.
357template <typename AAType, typename StateType = typename AAType::StateType,
358 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
359 bool RecurseForSelectAndPHI = true>
360static void clampReturnedValueStates(
361 Attributor &A, const AAType &QueryingAA, StateType &S,
362 const IRPosition::CallBaseContext *CBContext = nullptr) {
363 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
364 << QueryingAA << " into " << S << "\n");
365
366 assert((QueryingAA.getIRPosition().getPositionKind() ==
367 IRPosition::IRP_RETURNED ||
368 QueryingAA.getIRPosition().getPositionKind() ==
369 IRPosition::IRP_CALL_SITE_RETURNED) &&
370 "Can only clamp returned value states for a function returned or call "
371 "site returned position!");
372
373 // Use an optional state as there might not be any return values and we want
374 // to join (IntegerState::operator&) the state of all there are.
375 std::optional<StateType> T;
376
377 // Callback for each possibly returned value.
378 auto CheckReturnValue = [&](Value &RV) -> bool {
379 const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
380 // If possible, use the hasAssumedIRAttr interface.
381 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
382 bool IsKnown;
383 return AA::hasAssumedIRAttr<IRAttributeKind>(
384 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
385 }
386
387 const AAType *AA =
388 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
389 if (!AA)
390 return false;
391 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
392 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
393 const StateType &AAS = AA->getState();
394 if (!T)
395 T = StateType::getBestState(AAS);
396 *T &= AAS;
397 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
398 << "\n");
399 return T->isValidState();
400 };
401
402 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
403 S: AA::ValueScope::Intraprocedural,
404 RecurseForSelectAndPHI))
405 S.indicatePessimisticFixpoint();
406 else if (T)
407 S ^= *T;
408}
409
410namespace {
411/// Helper class for generic deduction: return value -> returned position.
412template <typename AAType, typename BaseType,
413 typename StateType = typename BaseType::StateType,
414 bool PropagateCallBaseContext = false,
415 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
416 bool RecurseForSelectAndPHI = true>
417struct AAReturnedFromReturnedValues : public BaseType {
418 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
419 : BaseType(IRP, A) {}
420
421 /// See AbstractAttribute::updateImpl(...).
422 ChangeStatus updateImpl(Attributor &A) override {
423 StateType S(StateType::getBestState(this->getState()));
424 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
425 RecurseForSelectAndPHI>(
426 A, *this, S,
427 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
428 // TODO: If we know we visited all returned values, thus no are assumed
429 // dead, we can take the known information from the state T.
430 return clampStateAndIndicateChange<StateType>(this->getState(), S);
431 }
432};
433
434/// Clamp the information known at all call sites for a given argument
435/// (identified by \p QueryingAA) into \p S.
436template <typename AAType, typename StateType = typename AAType::StateType,
437 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
438static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
439 StateType &S) {
440 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
441 << QueryingAA << " into " << S << "\n");
442
443 assert(QueryingAA.getIRPosition().getPositionKind() ==
444 IRPosition::IRP_ARGUMENT &&
445 "Can only clamp call site argument states for an argument position!");
446
447 // Use an optional state as there might not be any return values and we want
448 // to join (IntegerState::operator&) the state of all there are.
449 std::optional<StateType> T;
450
451 // The argument number which is also the call site argument number.
452 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
453
454 auto CallSiteCheck = [&](AbstractCallSite ACS) {
455 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
456 // Check if a coresponding argument was found or if it is on not associated
457 // (which can happen for callback calls).
458 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
459 return false;
460
461 // If possible, use the hasAssumedIRAttr interface.
462 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
463 bool IsKnown;
464 return AA::hasAssumedIRAttr<IRAttributeKind>(
465 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
466 }
467
468 const AAType *AA =
469 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
470 if (!AA)
471 return false;
472 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
473 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
474 << "\n");
475 const StateType &AAS = AA->getState();
476 if (!T)
477 T = StateType::getBestState(AAS);
478 *T &= AAS;
479 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
480 << "\n");
481 return T->isValidState();
482 };
483
484 bool UsedAssumedInformation = false;
485 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
486 UsedAssumedInformation))
487 S.indicatePessimisticFixpoint();
488 else if (T)
489 S ^= *T;
490}
491
492/// This function is the bridge between argument position and the call base
493/// context.
494template <typename AAType, typename BaseType,
495 typename StateType = typename AAType::StateType,
496 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
497bool getArgumentStateFromCallBaseContext(Attributor &A,
498 BaseType &QueryingAttribute,
499 IRPosition &Pos, StateType &State) {
500 assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
501 "Expected an 'argument' position !");
502 const CallBase *CBContext = Pos.getCallBaseContext();
503 if (!CBContext)
504 return false;
505
506 int ArgNo = Pos.getCallSiteArgNo();
507 assert(ArgNo >= 0 && "Invalid Arg No!");
508 const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
509
510 // If possible, use the hasAssumedIRAttr interface.
511 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
512 bool IsKnown;
513 return AA::hasAssumedIRAttr<IRAttributeKind>(
514 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
515 }
516
517 const auto *AA =
518 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
519 if (!AA)
520 return false;
521 const StateType &CBArgumentState =
522 static_cast<const StateType &>(AA->getState());
523
524 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
525 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
526 << "\n");
527
528 // NOTE: If we want to do call site grouping it should happen here.
529 State ^= CBArgumentState;
530 return true;
531}
532
533/// Helper class for generic deduction: call site argument -> argument position.
534template <typename AAType, typename BaseType,
535 typename StateType = typename AAType::StateType,
536 bool BridgeCallBaseContext = false,
537 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
538struct AAArgumentFromCallSiteArguments : public BaseType {
539 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
540 : BaseType(IRP, A) {}
541
542 /// See AbstractAttribute::updateImpl(...).
543 ChangeStatus updateImpl(Attributor &A) override {
544 StateType S = StateType::getBestState(this->getState());
545
546 if (BridgeCallBaseContext) {
547 bool Success =
548 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
549 IRAttributeKind>(
550 A, *this, this->getIRPosition(), S);
551 if (Success)
552 return clampStateAndIndicateChange<StateType>(this->getState(), S);
553 }
554 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
555 S);
556
557 // TODO: If we know we visited all incoming values, thus no are assumed
558 // dead, we can take the known information from the state T.
559 return clampStateAndIndicateChange<StateType>(this->getState(), S);
560 }
561};
562
563/// Helper class for generic replication: function returned -> cs returned.
564template <typename AAType, typename BaseType,
565 typename StateType = typename BaseType::StateType,
566 bool IntroduceCallBaseContext = false,
567 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
568struct AACalleeToCallSite : public BaseType {
569 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
570
571 /// See AbstractAttribute::updateImpl(...).
572 ChangeStatus updateImpl(Attributor &A) override {
573 auto IRPKind = this->getIRPosition().getPositionKind();
574 assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
575 IRPKind == IRPosition::IRP_CALL_SITE) &&
576 "Can only wrap function returned positions for call site "
577 "returned positions!");
578 auto &S = this->getState();
579
580 CallBase &CB = cast<CallBase>(this->getAnchorValue());
581 if (IntroduceCallBaseContext)
582 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
583 << "\n");
584
585 ChangeStatus Changed = ChangeStatus::UNCHANGED;
586 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
587 for (const Function *Callee : Callees) {
588 IRPosition FnPos =
589 IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
590 ? IRPosition::returned(F: *Callee,
591 CBContext: IntroduceCallBaseContext ? &CB : nullptr)
592 : IRPosition::function(
593 F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
594 // If possible, use the hasAssumedIRAttr interface.
595 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
596 bool IsKnown;
597 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
598 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
599 return false;
600 continue;
601 }
602
603 const AAType *AA =
604 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
605 if (!AA)
606 return false;
607 Changed |= clampStateAndIndicateChange(S, AA->getState());
608 if (S.isAtFixpoint())
609 return S.isValidState();
610 }
611 return true;
612 };
613 if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
614 return S.indicatePessimisticFixpoint();
615 return Changed;
616 }
617};
618
619/// Helper function to accumulate uses.
620template <class AAType, typename StateType = typename AAType::StateType>
621static void followUsesInContext(AAType &AA, Attributor &A,
622 MustBeExecutedContextExplorer &Explorer,
623 const Instruction *CtxI,
624 SetVector<const Use *> &Uses,
625 StateType &State) {
626 auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI);
627 for (unsigned u = 0; u < Uses.size(); ++u) {
628 const Use *U = Uses[u];
629 if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
630 bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd);
631 if (Found && AA.followUseInMBEC(A, U, UserI, State))
632 Uses.insert_range(R: llvm::make_pointer_range(Range: UserI->uses()));
633 }
634 }
635}
636
637/// Use the must-be-executed-context around \p I to add information into \p S.
638/// The AAType class is required to have `followUseInMBEC` method with the
639/// following signature and behaviour:
640///
641/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
642/// U - Underlying use.
643/// I - The user of the \p U.
644/// Returns true if the value should be tracked transitively.
645///
646template <class AAType, typename StateType = typename AAType::StateType>
647static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
648 Instruction &CtxI) {
649 const Value &Val = AA.getIRPosition().getAssociatedValue();
650 if (isa<ConstantData>(Val))
651 return;
652
653 MustBeExecutedContextExplorer *Explorer =
654 A.getInfoCache().getMustBeExecutedContextExplorer();
655 if (!Explorer)
656 return;
657
658 // Container for (transitive) uses of the associated value.
659 SetVector<const Use *> Uses;
660 for (const Use &U : Val.uses())
661 Uses.insert(X: &U);
662
663 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
664
665 if (S.isAtFixpoint())
666 return;
667
668 SmallVector<const CondBrInst *, 4> BrInsts;
669 auto Pred = [&](const Instruction *I) {
670 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: I))
671 BrInsts.push_back(Elt: Br);
672 return true;
673 };
674
675 // Here, accumulate conditional branch instructions in the context. We
676 // explore the child paths and collect the known states. The disjunction of
677 // those states can be merged to its own state. Let ParentState_i be a state
678 // to indicate the known information for an i-th branch instruction in the
679 // context. ChildStates are created for its successors respectively.
680 //
681 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
682 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
683 // ...
684 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
685 //
686 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
687 //
688 // FIXME: Currently, recursive branches are not handled. For example, we
689 // can't deduce that ptr must be dereferenced in below function.
690 //
691 // void f(int a, int c, int *ptr) {
692 // if(a)
693 // if (b) {
694 // *ptr = 0;
695 // } else {
696 // *ptr = 1;
697 // }
698 // else {
699 // if (b) {
700 // *ptr = 0;
701 // } else {
702 // *ptr = 1;
703 // }
704 // }
705 // }
706
707 Explorer->checkForAllContext(PP: &CtxI, Pred);
708 for (const CondBrInst *Br : BrInsts) {
709 StateType ParentState;
710
711 // The known state of the parent state is a conjunction of children's
712 // known states so it is initialized with a best state.
713 ParentState.indicateOptimisticFixpoint();
714
715 for (const BasicBlock *BB : Br->successors()) {
716 StateType ChildState;
717
718 size_t BeforeSize = Uses.size();
719 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
720
721 // Erase uses which only appear in the child.
722 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
723 It = Uses.erase(I: It);
724
725 ParentState &= ChildState;
726 }
727
728 // Use only known state.
729 S += ParentState;
730 }
731}
732} // namespace
733
734/// ------------------------ PointerInfo ---------------------------------------
735
736namespace llvm {
737namespace AA {
738namespace PointerInfo {
739
740struct State;
741
742} // namespace PointerInfo
743} // namespace AA
744
745/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
746template <>
747struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
748 using Access = AAPointerInfo::Access;
749 static unsigned getHashValue(const Access &A);
750 static bool isEqual(const Access &LHS, const Access &RHS);
751};
752
753/// Helper that allows RangeTy as a key in a DenseMap.
754template <> struct DenseMapInfo<AA::RangeTy> {
755 static unsigned getHashValue(const AA::RangeTy &Range) {
756 return detail::combineHashValue(
757 a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
758 b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
759 }
760
761 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
762 return A == B;
763 }
764};
765
766} // namespace llvm
767
768/// A type to track pointer/struct usage and accesses for AAPointerInfo.
769struct AA::PointerInfo::State : public AbstractState {
770 /// Return the best possible representable state.
771 static State getBestState(const State &SIS) { return State(); }
772
773 /// Return the worst possible representable state.
774 static State getWorstState(const State &SIS) {
775 State R;
776 R.indicatePessimisticFixpoint();
777 return R;
778 }
779
780 State() = default;
781 State(State &&SIS) = default;
782
783 const State &getAssumed() const { return *this; }
784
785 /// See AbstractState::isValidState().
786 bool isValidState() const override { return BS.isValidState(); }
787
788 /// See AbstractState::isAtFixpoint().
789 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
790
791 /// See AbstractState::indicateOptimisticFixpoint().
792 ChangeStatus indicateOptimisticFixpoint() override {
793 BS.indicateOptimisticFixpoint();
794 return ChangeStatus::UNCHANGED;
795 }
796
797 /// See AbstractState::indicatePessimisticFixpoint().
798 ChangeStatus indicatePessimisticFixpoint() override {
799 BS.indicatePessimisticFixpoint();
800 return ChangeStatus::CHANGED;
801 }
802
803 State &operator=(const State &R) {
804 if (this == &R)
805 return *this;
806 BS = R.BS;
807 AccessList = R.AccessList;
808 OffsetBins = R.OffsetBins;
809 RemoteIMap = R.RemoteIMap;
810 ReturnedOffsets = R.ReturnedOffsets;
811 return *this;
812 }
813
814 State &operator=(State &&R) {
815 if (this == &R)
816 return *this;
817 std::swap(a&: BS, b&: R.BS);
818 std::swap(LHS&: AccessList, RHS&: R.AccessList);
819 std::swap(a&: OffsetBins, b&: R.OffsetBins);
820 std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
821 std::swap(a&: ReturnedOffsets, b&: R.ReturnedOffsets);
822 return *this;
823 }
824
825 /// Add a new Access to the state at offset \p Offset and with size \p Size.
826 /// The access is associated with \p I, writes \p Content (if anything), and
827 /// is of kind \p Kind. If an Access already exists for the same \p I and same
828 /// \p RemoteI, the two are combined, potentially losing information about
829 /// offset and size. The resulting access must now be moved from its original
830 /// OffsetBin to the bin for its new offset.
831 ///
832 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
833 ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
834 Instruction &I, std::optional<Value *> Content,
835 AAPointerInfo::AccessKind Kind, Type *Ty,
836 Instruction *RemoteI = nullptr);
837
838 AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
839 AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
840 int64_t numOffsetBins() const { return OffsetBins.size(); }
841
842 const AAPointerInfo::Access &getAccess(unsigned Index) const {
843 return AccessList[Index];
844 }
845
846protected:
847 // Every memory instruction results in an Access object. We maintain a list of
848 // all Access objects that we own, along with the following maps:
849 //
850 // - OffsetBins: RangeTy -> { Access }
851 // - RemoteIMap: RemoteI x LocalI -> Access
852 //
853 // A RemoteI is any instruction that accesses memory. RemoteI is different
854 // from LocalI if and only if LocalI is a call; then RemoteI is some
855 // instruction in the callgraph starting from LocalI. Multiple paths in the
856 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
857 // are all combined into a single Access object. This may result in loss of
858 // information in RangeTy in the Access object.
859 SmallVector<AAPointerInfo::Access> AccessList;
860 AAPointerInfo::OffsetBinsTy OffsetBins;
861 DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
862
863 /// Flag to determine if the underlying pointer is reaching a return statement
864 /// in the associated function or not. Returns in other functions cause
865 /// invalidation.
866 AAPointerInfo::OffsetInfo ReturnedOffsets;
867
868 /// See AAPointerInfo::forallInterferingAccesses.
869 template <typename F>
870 bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {
871 if (!isValidState() || !ReturnedOffsets.isUnassigned())
872 return false;
873
874 for (const auto &It : OffsetBins) {
875 AA::RangeTy ItRange = It.getFirst();
876 if (!Range.mayOverlap(Range: ItRange))
877 continue;
878 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
879 for (auto Index : It.getSecond()) {
880 auto &Access = AccessList[Index];
881 if (!CB(Access, IsExact))
882 return false;
883 }
884 }
885 return true;
886 }
887
888 /// See AAPointerInfo::forallInterferingAccesses.
889 template <typename F>
890 bool forallInterferingAccesses(Instruction &I, F CB,
891 AA::RangeTy &Range) const {
892 if (!isValidState() || !ReturnedOffsets.isUnassigned())
893 return false;
894
895 auto LocalList = RemoteIMap.find(Val: &I);
896 if (LocalList == RemoteIMap.end()) {
897 return true;
898 }
899
900 for (unsigned Index : LocalList->getSecond()) {
901 for (auto &R : AccessList[Index]) {
902 Range &= R;
903 if (Range.offsetAndSizeAreUnknown())
904 break;
905 }
906 }
907 return forallInterferingAccesses(Range, CB);
908 }
909
910private:
911 /// State to track fixpoint and validity.
912 BooleanState BS;
913};
914
915ChangeStatus AA::PointerInfo::State::addAccess(
916 Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
917 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
918 Instruction *RemoteI) {
919 RemoteI = RemoteI ? RemoteI : &I;
920
921 // Check if we have an access for this instruction, if not, simply add it.
922 auto &LocalList = RemoteIMap[RemoteI];
923 bool AccExists = false;
924 unsigned AccIndex = AccessList.size();
925 for (auto Index : LocalList) {
926 auto &A = AccessList[Index];
927 if (A.getLocalInst() == &I) {
928 AccExists = true;
929 AccIndex = Index;
930 break;
931 }
932 }
933
934 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
935 LLVM_DEBUG(if (ToAdd.size()) dbgs()
936 << "[AAPointerInfo] Inserting access in new offset bins\n";);
937
938 for (auto Key : ToAdd) {
939 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
940 OffsetBins[Key].insert(V: AccIndex);
941 }
942 };
943
944 if (!AccExists) {
945 AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
946 assert((AccessList.size() == AccIndex + 1) &&
947 "New Access should have been at AccIndex");
948 LocalList.push_back(Elt: AccIndex);
949 AddToBins(AccessList[AccIndex].getRanges());
950 return ChangeStatus::CHANGED;
951 }
952
953 // Combine the new Access with the existing Access, and then update the
954 // mapping in the offset bins.
955 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
956 auto &Current = AccessList[AccIndex];
957 auto Before = Current;
958 Current &= Acc;
959 if (Current == Before)
960 return ChangeStatus::UNCHANGED;
961
962 auto &ExistingRanges = Before.getRanges();
963 auto &NewRanges = Current.getRanges();
964
965 // Ranges that are in the old access but not the new access need to be removed
966 // from the offset bins.
967 AAPointerInfo::RangeList ToRemove;
968 AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
969 LLVM_DEBUG(if (ToRemove.size()) dbgs()
970 << "[AAPointerInfo] Removing access from old offset bins\n";);
971
972 for (auto Key : ToRemove) {
973 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
974 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
975 auto &Bin = OffsetBins[Key];
976 assert(Bin.count(AccIndex) &&
977 "Expected bin to actually contain the Access.");
978 Bin.erase(V: AccIndex);
979 }
980
981 // Ranges that are in the new access but not the old access need to be added
982 // to the offset bins.
983 AAPointerInfo::RangeList ToAdd;
984 AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
985 AddToBins(ToAdd);
986 return ChangeStatus::CHANGED;
987}
988
989namespace {
990
991#ifndef NDEBUG
992static raw_ostream &operator<<(raw_ostream &OS,
993 const AAPointerInfo::OffsetInfo &OI) {
994 OS << llvm::interleaved_array(OI);
995 return OS;
996}
997#endif // NDEBUG
998
999struct AAPointerInfoImpl
1000 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1001 using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
1002 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1003
1004 /// See AbstractAttribute::getAsStr().
1005 const std::string getAsStr(Attributor *A) const override {
1006 return std::string("PointerInfo ") +
1007 (isValidState() ? (std::string("#") +
1008 std::to_string(val: OffsetBins.size()) + " bins")
1009 : "<invalid>") +
1010 (reachesReturn()
1011 ? (" (returned:" +
1012 join(R: map_range(C: ReturnedOffsets,
1013 F: [](int64_t O) { return std::to_string(val: O); }),
1014 Separator: ", ") +
1015 ")")
1016 : "");
1017 }
1018
1019 /// See AbstractAttribute::manifest(...).
1020 ChangeStatus manifest(Attributor &A) override {
1021 return AAPointerInfo::manifest(A);
1022 }
1023
1024 const_bin_iterator begin() const override { return State::begin(); }
1025 const_bin_iterator end() const override { return State::end(); }
1026 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
1027 bool reachesReturn() const override {
1028 return !ReturnedOffsets.isUnassigned();
1029 }
1030 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1031 if (ReturnedOffsets.isUnknown()) {
1032 OI.setUnknown();
1033 return;
1034 }
1035
1036 OffsetInfo MergedOI;
1037 for (auto Offset : ReturnedOffsets) {
1038 OffsetInfo TmpOI = OI;
1039 TmpOI.addToAll(Inc: Offset);
1040 MergedOI.merge(R: TmpOI);
1041 }
1042 OI = std::move(MergedOI);
1043 }
1044
1045 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1046 if (ReturnedOffsets.isUnknown())
1047 return ChangeStatus::UNCHANGED;
1048 if (ReachedReturnedOffsets.isUnknown()) {
1049 ReturnedOffsets.setUnknown();
1050 return ChangeStatus::CHANGED;
1051 }
1052 if (ReturnedOffsets.merge(R: ReachedReturnedOffsets))
1053 return ChangeStatus::CHANGED;
1054 return ChangeStatus::UNCHANGED;
1055 }
1056
1057 bool forallInterferingAccesses(
1058 AA::RangeTy Range,
1059 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1060 const override {
1061 return State::forallInterferingAccesses(Range, CB);
1062 }
1063
1064 bool forallInterferingAccesses(
1065 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1066 bool FindInterferingWrites, bool FindInterferingReads,
1067 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1068 AA::RangeTy &Range,
1069 function_ref<bool(const Access &)> SkipCB) const override {
1070 HasBeenWrittenTo = false;
1071
1072 SmallPtrSet<const Access *, 8> DominatingWrites;
1073 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1074
1075 Function &Scope = *I.getFunction();
1076 bool IsKnownNoSync;
1077 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1078 A, QueryingAA: &QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1079 IsKnown&: IsKnownNoSync);
1080 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1081 IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1082 bool AllInSameNoSyncFn = IsAssumedNoSync;
1083 bool InstIsExecutedByInitialThreadOnly =
1084 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1085
1086 // If the function is not ending in aligned barriers, we need the stores to
1087 // be in aligned barriers. The load being in one is not sufficient since the
1088 // store might be executed by a thread that disappears after, causing the
1089 // aligned barrier guarding the load to unblock and the load to read a value
1090 // that has no CFG path to the load.
1091 bool InstIsExecutedInAlignedRegion =
1092 FindInterferingReads && ExecDomainAA &&
1093 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1094
1095 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1096 A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1097
1098 InformationCache &InfoCache = A.getInfoCache();
1099 bool IsThreadLocalObj =
1100 AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1101
1102 // Helper to determine if we need to consider threading, which we cannot
1103 // right now. However, if the function is (assumed) nosync or the thread
1104 // executing all instructions is the main thread only we can ignore
1105 // threading. Also, thread-local objects do not require threading reasoning.
1106 // Finally, we can ignore threading if either access is executed in an
1107 // aligned region.
1108 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1109 if (IsThreadLocalObj || AllInSameNoSyncFn)
1110 return true;
1111 const auto *FnExecDomainAA =
1112 I.getFunction() == &Scope
1113 ? ExecDomainAA
1114 : A.lookupAAFor<AAExecutionDomain>(
1115 IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1116 DepClass: DepClassTy::NONE);
1117 if (!FnExecDomainAA)
1118 return false;
1119 if (InstIsExecutedInAlignedRegion ||
1120 (FindInterferingWrites &&
1121 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1122 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1123 return true;
1124 }
1125 if (InstIsExecutedByInitialThreadOnly &&
1126 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1127 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1128 return true;
1129 }
1130 return false;
1131 };
1132
1133 // Helper to determine if the access is executed by the same thread as the
1134 // given instruction, for now it is sufficient to avoid any potential
1135 // threading effects as we cannot deal with them anyway.
1136 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1137 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1138 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1139 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1140 };
1141
1142 // TODO: Use inter-procedural reachability and dominance.
1143 bool IsKnownNoRecurse;
1144 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1145 A, QueryingAA: this, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1146 IsKnown&: IsKnownNoRecurse);
1147
1148 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1149 // AAExecutionDomain) such that we allow scopes other than kernels as long
1150 // as the reaching kernels are disjoint.
1151 bool InstInKernel = A.getInfoCache().isKernel(F: Scope);
1152 bool ObjHasKernelLifetime = false;
1153 const bool UseDominanceReasoning =
1154 FindInterferingWrites && IsKnownNoRecurse;
1155 const DominatorTree *DT =
1156 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1157
1158 // Helper to check if a value has "kernel lifetime", that is it will not
1159 // outlive a GPU kernel. This is true for shared, constant, and local
1160 // globals on AMD and NVIDIA GPUs.
1161 auto HasKernelLifetime = [&](Value *V, Module &M) {
1162 if (!AA::isGPU(M))
1163 return false;
1164 unsigned VAS = V->getType()->getPointerAddressSpace();
1165 return AA::isGPUSharedAddressSpace(M, AS: VAS) ||
1166 AA::isGPUConstantAddressSpace(M, AS: VAS) ||
1167 AA::isGPULocalAddressSpace(M, AS: VAS);
1168 };
1169
1170 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1171 // to determine if we should look at reachability from the callee. For
1172 // certain pointers we know the lifetime and we do not have to step into the
1173 // callee to determine reachability as the pointer would be dead in the
1174 // callee. See the conditional initialization below.
1175 std::function<bool(const Function &)> IsLiveInCalleeCB;
1176
1177 if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1178 // If the alloca containing function is not recursive the alloca
1179 // must be dead in the callee.
1180 const Function *AIFn = AI->getFunction();
1181 ObjHasKernelLifetime = A.getInfoCache().isKernel(F: *AIFn);
1182 bool IsKnownNoRecurse;
1183 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1184 A, QueryingAA: this, IRP: IRPosition::function(F: *AIFn), DepClass: DepClassTy::OPTIONAL,
1185 IsKnown&: IsKnownNoRecurse)) {
1186 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1187 }
1188 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1189 // If the global has kernel lifetime we can stop if we reach a kernel
1190 // as it is "dead" in the (unknown) callees.
1191 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1192 if (ObjHasKernelLifetime)
1193 IsLiveInCalleeCB = [&A](const Function &Fn) {
1194 return !A.getInfoCache().isKernel(F: Fn);
1195 };
1196 }
1197
1198 // Set of accesses/instructions that will overwrite the result and are
1199 // therefore blockers in the reachability traversal.
1200 AA::InstExclusionSetTy ExclusionSet;
1201
1202 auto AccessCB = [&](const Access &Acc, bool Exact) {
1203 Function *AccScope = Acc.getRemoteInst()->getFunction();
1204 bool AccInSameScope = AccScope == &Scope;
1205
1206 // If the object has kernel lifetime we can ignore accesses only reachable
1207 // by other kernels. For now we only skip accesses *in* other kernels.
1208 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1209 A.getInfoCache().isKernel(F: *AccScope))
1210 return true;
1211
1212 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1213 if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1214 ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1215 }
1216
1217 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1218 (!FindInterferingReads || !Acc.isRead()))
1219 return true;
1220
1221 bool Dominates = FindInterferingWrites && DT && Exact &&
1222 Acc.isMustAccess() && AccInSameScope &&
1223 DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1224 if (Dominates)
1225 DominatingWrites.insert(Ptr: &Acc);
1226
1227 // Track if all interesting accesses are in the same `nosync` function as
1228 // the given instruction.
1229 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1230
1231 InterferingAccesses.push_back(Elt: {&Acc, Exact});
1232 return true;
1233 };
1234 if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1235 return false;
1236
1237 HasBeenWrittenTo = !DominatingWrites.empty();
1238
1239 // Dominating writes form a chain, find the least/lowest member.
1240 Instruction *LeastDominatingWriteInst = nullptr;
1241 for (const Access *Acc : DominatingWrites) {
1242 if (!LeastDominatingWriteInst) {
1243 LeastDominatingWriteInst = Acc->getRemoteInst();
1244 } else if (DT->dominates(Def: LeastDominatingWriteInst,
1245 User: Acc->getRemoteInst())) {
1246 LeastDominatingWriteInst = Acc->getRemoteInst();
1247 }
1248 }
1249
1250 // Helper to determine if we can skip a specific write access.
1251 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1252 if (SkipCB && SkipCB(Acc))
1253 return true;
1254 if (!CanIgnoreThreading(Acc))
1255 return false;
1256
1257 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1258 // If we successfully excluded all effects we are interested in, the
1259 // access can be skipped.
1260 bool ReadChecked = !FindInterferingReads;
1261 bool WriteChecked = !FindInterferingWrites;
1262
1263 // If the instruction cannot reach the access, the former does not
1264 // interfere with what the access reads.
1265 if (!ReadChecked) {
1266 if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1267 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1268 ReadChecked = true;
1269 }
1270 // If the instruction cannot be reach from the access, the latter does not
1271 // interfere with what the instruction reads.
1272 if (!WriteChecked) {
1273 if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1274 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1275 WriteChecked = true;
1276 }
1277
1278 // If we still might be affected by the write of the access but there are
1279 // dominating writes in the function of the instruction
1280 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1281 // by them. This would have happend above if they are all in the same
1282 // function, so we only check the inter-procedural case. Effectively, we
1283 // want to show that there is no call after the dominting write that might
1284 // reach the access, and when it returns reach the instruction with the
1285 // updated value. To this end, we iterate all call sites, check if they
1286 // might reach the instruction without going through another access
1287 // (ExclusionSet) and at the same time might reach the access. However,
1288 // that is all part of AAInterFnReachability.
1289 if (!WriteChecked && HasBeenWrittenTo &&
1290 Acc.getRemoteInst()->getFunction() != &Scope) {
1291
1292 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1293 QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1294 if (FnReachabilityAA) {
1295 // Without going backwards in the call tree, can we reach the access
1296 // from the least dominating write. Do not allow to pass the
1297 // instruction itself either.
1298 bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1299
1300 if (!FnReachabilityAA->instructionCanReach(
1301 A, Inst: *LeastDominatingWriteInst,
1302 Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1303 WriteChecked = true;
1304
1305 if (Inserted)
1306 ExclusionSet.erase(Ptr: &I);
1307 }
1308 }
1309
1310 if (ReadChecked && WriteChecked)
1311 return true;
1312
1313 if (!DT || !UseDominanceReasoning)
1314 return false;
1315 if (!DominatingWrites.count(Ptr: &Acc))
1316 return false;
1317 return LeastDominatingWriteInst != Acc.getRemoteInst();
1318 };
1319
1320 // Run the user callback on all accesses we cannot skip and return if
1321 // that succeeded for all or not.
1322 for (auto &It : InterferingAccesses) {
1323 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1324 !CanSkipAccess(*It.first, It.second)) {
1325 if (!UserCB(*It.first, It.second))
1326 return false;
1327 }
1328 }
1329 return true;
1330 }
1331
1332 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1333 const AAPointerInfo &OtherAA,
1334 CallBase &CB) {
1335 using namespace AA::PointerInfo;
1336 if (!OtherAA.getState().isValidState() || !isValidState())
1337 return indicatePessimisticFixpoint();
1338
1339 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1340 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1341 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1342 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1343
1344 // Combine the accesses bin by bin.
1345 const auto &State = OtherAAImpl.getState();
1346 for (const auto &It : State) {
1347 for (auto Index : It.getSecond()) {
1348 const auto &RAcc = State.getAccess(Index);
1349 if (IsByval && !RAcc.isRead())
1350 continue;
1351 bool UsedAssumedInformation = false;
1352 AccessKind AK = RAcc.getKind();
1353 auto Content = A.translateArgumentToCallSiteContent(
1354 V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1355 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1356 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1357
1358 Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1359 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1360 }
1361 }
1362 return Changed;
1363 }
1364
1365 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1366 const OffsetInfo &Offsets, CallBase &CB,
1367 bool IsMustAcc) {
1368 using namespace AA::PointerInfo;
1369 if (!OtherAA.getState().isValidState() || !isValidState())
1370 return indicatePessimisticFixpoint();
1371
1372 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1373
1374 // Combine the accesses bin by bin.
1375 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1376 const auto &State = OtherAAImpl.getState();
1377 for (const auto &It : State) {
1378 for (auto Index : It.getSecond()) {
1379 const auto &RAcc = State.getAccess(Index);
1380 if (!IsMustAcc && RAcc.isAssumption())
1381 continue;
1382 for (auto Offset : Offsets) {
1383 auto NewRanges = Offset == AA::RangeTy::Unknown
1384 ? AA::RangeTy::getUnknown()
1385 : RAcc.getRanges();
1386 if (!NewRanges.isUnknown()) {
1387 NewRanges.addToAllOffsets(Inc: Offset);
1388 }
1389 AccessKind AK = RAcc.getKind();
1390 if (!IsMustAcc)
1391 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1392 Changed |= addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: AK,
1393 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1394 }
1395 }
1396 }
1397 return Changed;
1398 }
1399
1400 /// Statistic tracking for all AAPointerInfo implementations.
1401 /// See AbstractAttribute::trackStatistics().
1402 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1403
1404 /// Dump the state into \p O.
1405 void dumpState(raw_ostream &O) {
1406 for (auto &It : OffsetBins) {
1407 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1408 << "] : " << It.getSecond().size() << "\n";
1409 for (auto AccIndex : It.getSecond()) {
1410 auto &Acc = AccessList[AccIndex];
1411 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1412 if (Acc.getLocalInst() != Acc.getRemoteInst())
1413 O << " --> " << *Acc.getRemoteInst()
1414 << "\n";
1415 if (!Acc.isWrittenValueYetUndetermined()) {
1416 if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1417 O << " - c: func " << Acc.getWrittenValue()->getName()
1418 << "\n";
1419 else if (Acc.getWrittenValue())
1420 O << " - c: " << *Acc.getWrittenValue() << "\n";
1421 else
1422 O << " - c: <unknown>\n";
1423 }
1424 }
1425 }
1426 }
1427};
1428
1429struct AAPointerInfoFloating : public AAPointerInfoImpl {
1430 using AccessKind = AAPointerInfo::AccessKind;
1431 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1432 : AAPointerInfoImpl(IRP, A) {}
1433
1434 /// Deal with an access and signal if it was handled successfully.
1435 bool handleAccess(Attributor &A, Instruction &I,
1436 std::optional<Value *> Content, AccessKind Kind,
1437 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1438 Type &Ty) {
1439 using namespace AA::PointerInfo;
1440 auto Size = AA::RangeTy::Unknown;
1441 const DataLayout &DL = A.getDataLayout();
1442 TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1443 if (!AccessSize.isScalable())
1444 Size = AccessSize.getFixedValue();
1445
1446 // Make a strictly ascending list of offsets as required by addAccess()
1447 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1448 llvm::sort(C&: OffsetsSorted);
1449
1450 VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1451 if (!VT || VT->getElementCount().isScalable() ||
1452 !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1453 (*Content)->getType() != VT ||
1454 DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1455 Changed =
1456 Changed | addAccess(A, Ranges: {OffsetsSorted, Size}, I, Content, Kind, Ty: &Ty);
1457 } else {
1458 // Handle vector stores with constant content element-wise.
1459 // TODO: We could look for the elements or create instructions
1460 // representing them.
1461 // TODO: We need to push the Content into the range abstraction
1462 // (AA::RangeTy) to allow different content values for different
1463 // ranges. ranges. Hence, support vectors storing different values.
1464 Type *ElementType = VT->getElementType();
1465 int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1466 auto *ConstContent = cast<Constant>(Val: *Content);
1467 Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1468 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1469
1470 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1471 Value *ElementContent = ConstantExpr::getExtractElement(
1472 Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1473
1474 // Add the element access.
1475 Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1476 Content: ElementContent, Kind, Ty: ElementType);
1477
1478 // Advance the offsets for the next element.
1479 for (auto &ElementOffset : ElementOffsets)
1480 ElementOffset += ElementSize;
1481 }
1482 }
1483 return true;
1484 };
1485
1486 /// See AbstractAttribute::updateImpl(...).
1487 ChangeStatus updateImpl(Attributor &A) override;
1488
1489 /// If the indices to \p GEP can be traced to constants, incorporate all
1490 /// of these into \p UsrOI.
1491 ///
1492 /// \return true iff \p UsrOI is updated.
1493 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1494 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1495 const GEPOperator *GEP);
1496
1497 /// See AbstractAttribute::trackStatistics()
1498 void trackStatistics() const override {
1499 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1500 }
1501};
1502
1503bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1504 const DataLayout &DL,
1505 OffsetInfo &UsrOI,
1506 const OffsetInfo &PtrOI,
1507 const GEPOperator *GEP) {
1508 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1509 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1510 APInt ConstantOffset(BitWidth, 0);
1511
1512 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1513 "Don't look for constant values if the offset has already been "
1514 "determined to be unknown.");
1515
1516 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1517 UsrOI.setUnknown();
1518 return true;
1519 }
1520
1521 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1522 << (VariableOffsets.empty() ? "" : "not") << " constant "
1523 << *GEP << "\n");
1524
1525 auto Union = PtrOI;
1526 Union.addToAll(Inc: ConstantOffset.getSExtValue());
1527
1528 // Each VI in VariableOffsets has a set of potential constant values. Every
1529 // combination of elements, picked one each from these sets, is separately
1530 // added to the original set of offsets, thus resulting in more offsets.
1531 for (const auto &VI : VariableOffsets) {
1532 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1533 QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1534 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1535 UsrOI.setUnknown();
1536 return true;
1537 }
1538
1539 // UndefValue is treated as a zero, which leaves Union as is.
1540 if (PotentialConstantsAA->undefIsContained())
1541 continue;
1542
1543 // We need at least one constant in every set to compute an actual offset.
1544 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1545 // don't actually exist. In other words, the absence of constant values
1546 // implies that the operation can be assumed dead for now.
1547 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1548 if (AssumedSet.empty())
1549 return false;
1550
1551 OffsetInfo Product;
1552 for (const auto &ConstOffset : AssumedSet) {
1553 auto CopyPerOffset = Union;
1554 CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1555 VI.second.getZExtValue());
1556 Product.merge(R: CopyPerOffset);
1557 }
1558 Union = Product;
1559 }
1560
1561 UsrOI = std::move(Union);
1562 return true;
1563}
1564
1565ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1566 using namespace AA::PointerInfo;
1567 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1568 const DataLayout &DL = A.getDataLayout();
1569 Value &AssociatedValue = getAssociatedValue();
1570
1571 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1572 OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1573
1574 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1575 // One does not simply walk into a map and assign a reference to a possibly
1576 // new location. That can cause an invalidation before the assignment
1577 // happens, like so:
1578 //
1579 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1580 //
1581 // The RHS is a reference that may be invalidated by an insertion caused by
1582 // the LHS. So we ensure that the side-effect of the LHS happens first.
1583
1584 assert(OffsetInfoMap.contains(CurPtr) &&
1585 "CurPtr does not exist in the map!");
1586
1587 auto &UsrOI = OffsetInfoMap[Usr];
1588 auto &PtrOI = OffsetInfoMap[CurPtr];
1589 assert(!PtrOI.isUnassigned() &&
1590 "Cannot pass through if the input Ptr was not visited!");
1591 UsrOI.merge(R: PtrOI);
1592 Follow = true;
1593 return true;
1594 };
1595
1596 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1597 Value *CurPtr = U.get();
1598 User *Usr = U.getUser();
1599 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1600 << "\n");
1601 assert(OffsetInfoMap.count(CurPtr) &&
1602 "The current pointer offset should have been seeded!");
1603 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1604 "Current pointer should be assigned");
1605
1606 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: Usr)) {
1607 if (CE->isCast())
1608 return HandlePassthroughUser(Usr, CurPtr, Follow);
1609 if (!isa<GEPOperator>(Val: CE)) {
1610 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1611 << "\n");
1612 return false;
1613 }
1614 }
1615 if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1616 // Note the order here, the Usr access might change the map, CurPtr is
1617 // already in it though.
1618 auto &UsrOI = OffsetInfoMap[Usr];
1619 auto &PtrOI = OffsetInfoMap[CurPtr];
1620
1621 if (UsrOI.isUnknown())
1622 return true;
1623
1624 if (PtrOI.isUnknown()) {
1625 Follow = true;
1626 UsrOI.setUnknown();
1627 return true;
1628 }
1629
1630 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1631 return true;
1632 }
1633 if (isa<PtrToIntInst>(Val: Usr))
1634 return false;
1635 if (isa<CastInst>(Val: Usr) || isa<SelectInst>(Val: Usr))
1636 return HandlePassthroughUser(Usr, CurPtr, Follow);
1637 // Returns are allowed if they are in the associated functions. Users can
1638 // then check the call site return. Returns from other functions can't be
1639 // tracked and are cause for invalidation.
1640 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
1641 if (RI->getFunction() == getAssociatedFunction()) {
1642 auto &PtrOI = OffsetInfoMap[CurPtr];
1643 Changed |= setReachesReturn(PtrOI);
1644 return true;
1645 }
1646 return false;
1647 }
1648
1649 // For PHIs we need to take care of the recurrence explicitly as the value
1650 // might change while we iterate through a loop. For now, we give up if
1651 // the PHI is not invariant.
1652 if (auto *PHI = dyn_cast<PHINode>(Val: Usr)) {
1653 // Note the order here, the Usr access might change the map, CurPtr is
1654 // already in it though.
1655 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(Key: PHI);
1656 auto &UsrOI = PhiIt->second;
1657 auto &PtrOI = OffsetInfoMap[CurPtr];
1658
1659 // Check if the PHI operand has already an unknown offset as we can't
1660 // improve on that anymore.
1661 if (PtrOI.isUnknown()) {
1662 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1663 << *CurPtr << " in " << *PHI << "\n");
1664 Follow = !UsrOI.isUnknown();
1665 UsrOI.setUnknown();
1666 return true;
1667 }
1668
1669 // Check if the PHI is invariant (so far).
1670 if (UsrOI == PtrOI) {
1671 assert(!PtrOI.isUnassigned() &&
1672 "Cannot assign if the current Ptr was not visited!");
1673 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1674 return true;
1675 }
1676
1677 // Check if the PHI operand can be traced back to AssociatedValue.
1678 APInt Offset(
1679 DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1680 0);
1681 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1682 DL, Offset, /* AllowNonInbounds */ true);
1683 auto It = OffsetInfoMap.find(Val: CurPtrBase);
1684 if (It == OffsetInfoMap.end()) {
1685 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1686 << *CurPtr << " in " << *PHI
1687 << " (base: " << *CurPtrBase << ")\n");
1688 UsrOI.setUnknown();
1689 Follow = true;
1690 return true;
1691 }
1692
1693 // Check if the PHI operand is not dependent on the PHI itself. Every
1694 // recurrence is a cyclic net of PHIs in the data flow, and has an
1695 // equivalent Cycle in the control flow. One of those PHIs must be in the
1696 // header of that control flow Cycle. This is independent of the choice of
1697 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1698 // every Cycle header; if such a node is marked unknown, this will
1699 // eventually propagate through the whole net of PHIs in the recurrence.
1700 const auto *CI =
1701 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1702 F: *PHI->getFunction());
1703 if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1704 auto BaseOI = It->getSecond();
1705 BaseOI.addToAll(Inc: Offset.getZExtValue());
1706 if (IsFirstPHIUser || BaseOI == UsrOI) {
1707 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1708 << " in " << *Usr << "\n");
1709 return HandlePassthroughUser(Usr, CurPtr, Follow);
1710 }
1711
1712 LLVM_DEBUG(
1713 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1714 << *CurPtr << " in " << *PHI << "\n");
1715 UsrOI.setUnknown();
1716 Follow = true;
1717 return true;
1718 }
1719
1720 UsrOI.merge(R: PtrOI);
1721 Follow = true;
1722 return true;
1723 }
1724
1725 if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1726 // If the access is to a pointer that may or may not be the associated
1727 // value, e.g. due to a PHI, we cannot assume it will be read.
1728 AccessKind AK = AccessKind::AK_R;
1729 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1730 AK = AccessKind(AK | AccessKind::AK_MUST);
1731 else
1732 AK = AccessKind(AK | AccessKind::AK_MAY);
1733 if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1734 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1735 Ty&: *LoadI->getType()))
1736 return false;
1737
1738 auto IsAssumption = [](Instruction &I) {
1739 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1740 return II->isAssumeLikeIntrinsic();
1741 return false;
1742 };
1743
1744 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1745 // Check if the assumption and the load are executed together without
1746 // memory modification.
1747 do {
1748 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1749 return true;
1750 FromI = FromI->getNextNode();
1751 } while (FromI && FromI != ToI);
1752 return false;
1753 };
1754
1755 BasicBlock *BB = LoadI->getParent();
1756 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1757 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1758 return false;
1759 BasicBlock *IntrBB = IntrI.getParent();
1760 if (IntrI.getParent() == BB) {
1761 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1762 return false;
1763 } else {
1764 auto PredIt = pred_begin(BB: IntrBB);
1765 if (PredIt == pred_end(BB: IntrBB))
1766 return false;
1767 if ((*PredIt) != BB)
1768 return false;
1769 if (++PredIt != pred_end(BB: IntrBB))
1770 return false;
1771 for (auto *SuccBB : successors(BB)) {
1772 if (SuccBB == IntrBB)
1773 continue;
1774 if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1775 continue;
1776 return false;
1777 }
1778 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1779 return false;
1780 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1781 return false;
1782 }
1783 return true;
1784 };
1785
1786 std::pair<Value *, IntrinsicInst *> Assumption;
1787 for (const Use &LoadU : LoadI->uses()) {
1788 if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1789 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1790 continue;
1791 for (const Use &CmpU : CmpI->uses()) {
1792 if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1793 if (!IsValidAssume(*IntrI))
1794 continue;
1795 int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1796 Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1797 break;
1798 }
1799 }
1800 }
1801 if (Assumption.first)
1802 break;
1803 }
1804
1805 // Check if we found an assumption associated with this load.
1806 if (!Assumption.first || !Assumption.second)
1807 return true;
1808
1809 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1810 << *Assumption.second << ": " << *LoadI
1811 << " == " << *Assumption.first << "\n");
1812 bool UsedAssumedInformation = false;
1813 std::optional<Value *> Content = nullptr;
1814 if (Assumption.first)
1815 Content =
1816 A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1817 UsedAssumedInformation, S: AA::Interprocedural);
1818 return handleAccess(
1819 A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1820 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1821 }
1822
1823 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1824 ArrayRef<Value *> OtherOps, AccessKind AK) {
1825 for (auto *OtherOp : OtherOps) {
1826 if (OtherOp == CurPtr) {
1827 LLVM_DEBUG(
1828 dbgs()
1829 << "[AAPointerInfo] Escaping use in store like instruction " << I
1830 << "\n");
1831 return false;
1832 }
1833 }
1834
1835 // If the access is to a pointer that may or may not be the associated
1836 // value, e.g. due to a PHI, we cannot assume it will be written.
1837 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1838 AK = AccessKind(AK | AccessKind::AK_MUST);
1839 else
1840 AK = AccessKind(AK | AccessKind::AK_MAY);
1841 bool UsedAssumedInformation = false;
1842 std::optional<Value *> Content = nullptr;
1843 if (ValueOp)
1844 Content = A.getAssumedSimplified(
1845 V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1846 return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1847 Changed, Ty&: ValueTy);
1848 };
1849
1850 if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1851 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1852 *StoreI->getValueOperand()->getType(),
1853 {StoreI->getValueOperand()}, AccessKind::AK_W);
1854 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1855 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1856 {RMWI->getValOperand()}, AccessKind::AK_RW);
1857 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1858 return HandleStoreLike(
1859 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1860 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1861 AccessKind::AK_RW);
1862
1863 if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1864 if (CB->isLifetimeStartOrEnd())
1865 return true;
1866 const auto *TLI =
1867 A.getInfoCache().getTargetLibraryInfoForFunction(F: *CB->getFunction());
1868 if (getFreedOperand(CB, TLI) == U)
1869 return true;
1870 if (CB->isArgOperand(U: &U)) {
1871 unsigned ArgNo = CB->getArgOperandNo(U: &U);
1872 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1873 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1874 DepClass: DepClassTy::REQUIRED);
1875 if (!CSArgPI)
1876 return false;
1877 bool IsArgMustAcc = (getUnderlyingObject(V: CurPtr) == &AssociatedValue);
1878 Changed = translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB,
1879 IsMustAcc: IsArgMustAcc) |
1880 Changed;
1881 if (!CSArgPI->reachesReturn())
1882 return isValidState();
1883
1884 Function *Callee = CB->getCalledFunction();
1885 if (!Callee || Callee->arg_size() <= ArgNo)
1886 return false;
1887 bool UsedAssumedInformation = false;
1888 auto ReturnedValue = A.getAssumedSimplified(
1889 IRP: IRPosition::returned(F: *Callee), AA: *this, UsedAssumedInformation,
1890 S: AA::ValueScope::Intraprocedural);
1891 auto *ReturnedArg =
1892 dyn_cast_or_null<Argument>(Val: ReturnedValue.value_or(u: nullptr));
1893 auto *Arg = Callee->getArg(i: ArgNo);
1894 if (ReturnedArg && Arg != ReturnedArg)
1895 return true;
1896 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1897 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1898 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
1899 if (!CSRetPI)
1900 return false;
1901 OffsetInfo OI = OffsetInfoMap[CurPtr];
1902 CSArgPI->addReturnedOffsetsTo(OI);
1903 Changed =
1904 translateAndAddState(A, OtherAA: *CSRetPI, Offsets: OI, CB&: *CB, IsMustAcc: IsRetMustAcc) | Changed;
1905 return isValidState();
1906 }
1907 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1908 << "\n");
1909 return false;
1910 }
1911
1912 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1913 return false;
1914 };
1915 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1916 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1917 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1918 if (OffsetInfoMap.count(Val: NewU)) {
1919 LLVM_DEBUG({
1920 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1921 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1922 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1923 << "\n";
1924 }
1925 });
1926 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1927 }
1928 bool Unused;
1929 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1930 };
1931 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1932 /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1933 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1934 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1935 return indicatePessimisticFixpoint();
1936 }
1937
1938 LLVM_DEBUG({
1939 dbgs() << "Accesses by bin after update:\n";
1940 dumpState(dbgs());
1941 });
1942
1943 return Changed;
1944}
1945
1946struct AAPointerInfoReturned final : AAPointerInfoImpl {
1947 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1948 : AAPointerInfoImpl(IRP, A) {}
1949
1950 /// See AbstractAttribute::updateImpl(...).
1951 ChangeStatus updateImpl(Attributor &A) override {
1952 return indicatePessimisticFixpoint();
1953 }
1954
1955 /// See AbstractAttribute::trackStatistics()
1956 void trackStatistics() const override {
1957 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1958 }
1959};
1960
1961struct AAPointerInfoArgument final : AAPointerInfoFloating {
1962 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1963 : AAPointerInfoFloating(IRP, A) {}
1964
1965 /// See AbstractAttribute::trackStatistics()
1966 void trackStatistics() const override {
1967 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1968 }
1969};
1970
1971struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1972 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1973 : AAPointerInfoFloating(IRP, A) {}
1974
1975 /// See AbstractAttribute::updateImpl(...).
1976 ChangeStatus updateImpl(Attributor &A) override {
1977 using namespace AA::PointerInfo;
1978 // We handle memory intrinsics explicitly, at least the first (=
1979 // destination) and second (=source) arguments as we know how they are
1980 // accessed.
1981 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
1982 int64_t LengthVal = AA::RangeTy::Unknown;
1983 if (auto Length = MI->getLengthInBytes())
1984 LengthVal = Length->getSExtValue();
1985 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1986 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1987 if (ArgNo > 1) {
1988 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1989 << *MI << "\n");
1990 return indicatePessimisticFixpoint();
1991 } else {
1992 auto Kind =
1993 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1994 Changed =
1995 Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
1996 }
1997 LLVM_DEBUG({
1998 dbgs() << "Accesses by bin after update:\n";
1999 dumpState(dbgs());
2000 });
2001
2002 return Changed;
2003 }
2004
2005 // TODO: Once we have call site specific value information we can provide
2006 // call site specific liveness information and then it makes
2007 // sense to specialize attributes for call sites arguments instead of
2008 // redirecting requests to the callee argument.
2009 Argument *Arg = getAssociatedArgument();
2010 if (Arg) {
2011 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2012 auto *ArgAA =
2013 A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
2014 if (ArgAA && ArgAA->getState().isValidState())
2015 return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
2016 CB&: *cast<CallBase>(Val: getCtxI()));
2017 if (!Arg->getParent()->isDeclaration())
2018 return indicatePessimisticFixpoint();
2019 }
2020
2021 bool IsKnownNoCapture;
2022 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2023 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
2024 return indicatePessimisticFixpoint();
2025
2026 bool IsKnown = false;
2027 if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
2028 return ChangeStatus::UNCHANGED;
2029 bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
2030 auto Kind =
2031 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2032 return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
2033 Ty: nullptr);
2034 }
2035
2036 /// See AbstractAttribute::trackStatistics()
2037 void trackStatistics() const override {
2038 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2039 }
2040};
2041
2042struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2043 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2044 : AAPointerInfoFloating(IRP, A) {}
2045
2046 /// See AbstractAttribute::trackStatistics()
2047 void trackStatistics() const override {
2048 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2049 }
2050};
2051} // namespace
2052
2053/// -----------------------NoUnwind Function Attribute--------------------------
2054
2055namespace {
2056struct AANoUnwindImpl : AANoUnwind {
2057 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2058
2059 /// See AbstractAttribute::initialize(...).
2060 void initialize(Attributor &A) override {
2061 bool IsKnown;
2062 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2063 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2064 (void)IsKnown;
2065 }
2066
2067 const std::string getAsStr(Attributor *A) const override {
2068 return getAssumed() ? "nounwind" : "may-unwind";
2069 }
2070
2071 /// See AbstractAttribute::updateImpl(...).
2072 ChangeStatus updateImpl(Attributor &A) override {
2073 auto Opcodes = {
2074 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2075 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2076 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2077
2078 auto CheckForNoUnwind = [&](Instruction &I) {
2079 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2080 return true;
2081
2082 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2083 bool IsKnownNoUnwind;
2084 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2085 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2086 IsKnown&: IsKnownNoUnwind);
2087 }
2088 return false;
2089 };
2090
2091 bool UsedAssumedInformation = false;
2092 if (!A.checkForAllInstructions(Pred: CheckForNoUnwind, QueryingAA: *this, Opcodes,
2093 UsedAssumedInformation))
2094 return indicatePessimisticFixpoint();
2095
2096 return ChangeStatus::UNCHANGED;
2097 }
2098};
2099
2100struct AANoUnwindFunction final : public AANoUnwindImpl {
2101 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2102 : AANoUnwindImpl(IRP, A) {}
2103
2104 /// See AbstractAttribute::trackStatistics()
2105 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2106};
2107
2108/// NoUnwind attribute deduction for a call sites.
2109struct AANoUnwindCallSite final
2110 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2111 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2112 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2113
2114 /// See AbstractAttribute::trackStatistics()
2115 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2116};
2117} // namespace
2118
2119/// ------------------------ NoSync Function Attribute -------------------------
2120
2121bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2122 switch (CB.getIntrinsicID()) {
2123 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2124 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2125 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2126 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2127 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2128 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2129 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2130 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2131 return true;
2132 case Intrinsic::amdgcn_s_barrier:
2133 if (ExecutedAligned)
2134 return true;
2135 break;
2136 default:
2137 break;
2138 }
2139 return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2140}
2141
2142bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2143 if (!I->isAtomic())
2144 return false;
2145
2146 if (auto *FI = dyn_cast<FenceInst>(Val: I))
2147 // All legal orderings for fence are stronger than monotonic.
2148 return FI->getSyncScopeID() != SyncScope::SingleThread;
2149 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2150 // Unordered is not a legal ordering for cmpxchg.
2151 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2152 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2153 }
2154
2155 AtomicOrdering Ordering;
2156 switch (I->getOpcode()) {
2157 case Instruction::AtomicRMW:
2158 Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2159 break;
2160 case Instruction::Store:
2161 Ordering = cast<StoreInst>(Val: I)->getOrdering();
2162 break;
2163 case Instruction::Load:
2164 Ordering = cast<LoadInst>(Val: I)->getOrdering();
2165 break;
2166 default:
2167 llvm_unreachable(
2168 "New atomic operations need to be known in the attributor.");
2169 }
2170
2171 return (Ordering != AtomicOrdering::Unordered &&
2172 Ordering != AtomicOrdering::Monotonic);
2173}
2174
2175namespace {
2176struct AANoSyncImpl : AANoSync {
2177 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2178
2179 /// See AbstractAttribute::initialize(...).
2180 void initialize(Attributor &A) override {
2181 bool IsKnown;
2182 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2183 DepClassTy::NONE, IsKnown));
2184 (void)IsKnown;
2185 }
2186
2187 const std::string getAsStr(Attributor *A) const override {
2188 return getAssumed() ? "nosync" : "may-sync";
2189 }
2190
2191 /// See AbstractAttribute::updateImpl(...).
2192 ChangeStatus updateImpl(Attributor &A) override;
2193};
2194
2195ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2196
2197 auto CheckRWInstForNoSync = [&](Instruction &I) {
2198 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2199 };
2200
2201 auto CheckForNoSync = [&](Instruction &I) {
2202 // At this point we handled all read/write effects and they are all
2203 // nosync, so they can be skipped.
2204 if (I.mayReadOrWriteMemory())
2205 return true;
2206
2207 bool IsKnown;
2208 CallBase &CB = cast<CallBase>(Val&: I);
2209 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2210 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL,
2211 IsKnown))
2212 return true;
2213
2214 // non-convergent and readnone imply nosync.
2215 return !CB.isConvergent();
2216 };
2217
2218 bool UsedAssumedInformation = false;
2219 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInstForNoSync, QueryingAA&: *this,
2220 UsedAssumedInformation) ||
2221 !A.checkForAllCallLikeInstructions(Pred: CheckForNoSync, QueryingAA: *this,
2222 UsedAssumedInformation))
2223 return indicatePessimisticFixpoint();
2224
2225 return ChangeStatus::UNCHANGED;
2226}
2227
2228struct AANoSyncFunction final : public AANoSyncImpl {
2229 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2230 : AANoSyncImpl(IRP, A) {}
2231
2232 /// See AbstractAttribute::trackStatistics()
2233 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2234};
2235
2236/// NoSync attribute deduction for a call sites.
2237struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2238 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2239 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2240
2241 /// See AbstractAttribute::trackStatistics()
2242 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2243};
2244} // namespace
2245
2246/// ------------------------ No-Free Attributes ----------------------------
2247
2248namespace {
2249struct AANoFreeImpl : public AANoFree {
2250 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2251
2252 /// See AbstractAttribute::initialize(...).
2253 void initialize(Attributor &A) override {
2254 bool IsKnown;
2255 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2256 DepClassTy::NONE, IsKnown));
2257 (void)IsKnown;
2258 }
2259
2260 /// See AbstractAttribute::updateImpl(...).
2261 ChangeStatus updateImpl(Attributor &A) override {
2262 auto CheckForNoFree = [&](Instruction &I) {
2263 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
2264 bool IsKnown;
2265 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2266 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2267 IsKnown);
2268 }
2269 // Make sure that synchronization cannot establish happens-before with a
2270 // free on another thread.
2271 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2272 };
2273
2274 bool UsedAssumedInformation = false;
2275 if (!A.checkForAllReadWriteInstructions(Pred: CheckForNoFree, QueryingAA&: *this,
2276 UsedAssumedInformation) ||
2277 !A.checkForAllCallLikeInstructions(Pred: CheckForNoFree, QueryingAA: *this,
2278 UsedAssumedInformation))
2279 return indicatePessimisticFixpoint();
2280
2281 return ChangeStatus::UNCHANGED;
2282 }
2283
2284 /// See AbstractAttribute::getAsStr().
2285 const std::string getAsStr(Attributor *A) const override {
2286 return getAssumed() ? "nofree" : "may-free";
2287 }
2288};
2289
2290struct AANoFreeFunction final : public AANoFreeImpl {
2291 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2292 : AANoFreeImpl(IRP, A) {}
2293
2294 /// See AbstractAttribute::trackStatistics()
2295 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2296};
2297
2298/// NoFree attribute deduction for a call sites.
2299struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2300 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2301 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2302
2303 /// See AbstractAttribute::trackStatistics()
2304 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2305};
2306
2307/// NoFree attribute for floating values.
2308struct AANoFreeFloating : AANoFreeImpl {
2309 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2310 : AANoFreeImpl(IRP, A) {}
2311
2312 /// See AbstractAttribute::trackStatistics()
2313 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2314
2315 /// See Abstract Attribute::updateImpl(...).
2316 ChangeStatus updateImpl(Attributor &A) override {
2317 const IRPosition &IRP = getIRPosition();
2318
2319 bool IsKnown;
2320 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this,
2321 IRP: IRPosition::function_scope(IRP),
2322 DepClass: DepClassTy::OPTIONAL, IsKnown))
2323 return ChangeStatus::UNCHANGED;
2324
2325 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2326 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2327 Instruction *UserI = cast<Instruction>(Val: U.getUser());
2328 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2329 if (CB->isBundleOperand(U: &U))
2330 return false;
2331 if (!CB->isArgOperand(U: &U))
2332 return true;
2333 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2334
2335 // Even if the argument is nofree, we still need to check for nocapture,
2336 // as the call may capture the argument without freeing it, and the
2337 // captured argument is freed later.
2338 bool IsKnown;
2339 if (!AA::hasAssumedIRAttr<Attribute::NoFree>(
2340 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2341 DepClass: DepClassTy::REQUIRED, IsKnown))
2342 return false;
2343
2344 const AANoCapture *NoCaptureAA = nullptr;
2345 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2346 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2347 DepClass: DepClassTy::REQUIRED, IsKnown,
2348 /*IgnoreSubsumingPositions=*/false, AAPtr: &NoCaptureAA)) {
2349 if (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
2350 Follow = true;
2351 return true;
2352 }
2353 return false;
2354 }
2355
2356 return true;
2357 }
2358
2359 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
2360 if (!capturesAnyProvenance(CC: CI))
2361 return true;
2362 if (capturesAnyProvenance(CC: CI.ResultCC)) {
2363 Follow = true;
2364 return true;
2365 }
2366
2367 if (isa<ReturnInst>(Val: UserI) && getIRPosition().isArgumentPosition())
2368 return true;
2369
2370 // Capturing user.
2371 return false;
2372 };
2373 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: AssociatedValue))
2374 return indicatePessimisticFixpoint();
2375
2376 return ChangeStatus::UNCHANGED;
2377 }
2378};
2379
2380/// NoFree attribute for a call site argument.
2381struct AANoFreeArgument final : AANoFreeFloating {
2382 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2383 : AANoFreeFloating(IRP, A) {}
2384
2385 /// See AbstractAttribute::trackStatistics()
2386 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2387};
2388
2389/// NoFree attribute for call site arguments.
2390struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2391 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2392 : AANoFreeFloating(IRP, A) {}
2393
2394 /// See AbstractAttribute::updateImpl(...).
2395 ChangeStatus updateImpl(Attributor &A) override {
2396 // TODO: Once we have call site specific value information we can provide
2397 // call site specific liveness information and then it makes
2398 // sense to specialize attributes for call sites arguments instead of
2399 // redirecting requests to the callee argument.
2400 Argument *Arg = getAssociatedArgument();
2401 if (!Arg)
2402 return indicatePessimisticFixpoint();
2403 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2404 bool IsKnown;
2405 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this, IRP: ArgPos,
2406 DepClass: DepClassTy::REQUIRED, IsKnown))
2407 return ChangeStatus::UNCHANGED;
2408 return indicatePessimisticFixpoint();
2409 }
2410
2411 /// See AbstractAttribute::trackStatistics()
2412 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2413};
2414
2415/// NoFree attribute for function return value.
2416struct AANoFreeReturned final : AANoFreeFloating {
2417 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2418 : AANoFreeFloating(IRP, A) {
2419 llvm_unreachable("NoFree is not applicable to function returns!");
2420 }
2421
2422 /// See AbstractAttribute::initialize(...).
2423 void initialize(Attributor &A) override {
2424 llvm_unreachable("NoFree is not applicable to function returns!");
2425 }
2426
2427 /// See AbstractAttribute::updateImpl(...).
2428 ChangeStatus updateImpl(Attributor &A) override {
2429 llvm_unreachable("NoFree is not applicable to function returns!");
2430 }
2431
2432 /// See AbstractAttribute::trackStatistics()
2433 void trackStatistics() const override {}
2434};
2435
2436/// NoFree attribute deduction for a call site return value.
2437struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2438 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2439 : AANoFreeFloating(IRP, A) {}
2440
2441 ChangeStatus manifest(Attributor &A) override {
2442 return ChangeStatus::UNCHANGED;
2443 }
2444 /// See AbstractAttribute::trackStatistics()
2445 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2446};
2447} // namespace
2448
2449/// ------------------------ NonNull Argument Attribute ------------------------
2450
2451bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2452 Attribute::AttrKind ImpliedAttributeKind,
2453 bool IgnoreSubsumingPositions) {
2454 SmallVector<Attribute::AttrKind, 2> AttrKinds;
2455 AttrKinds.push_back(Elt: Attribute::NonNull);
2456 if (!NullPointerIsDefined(F: IRP.getAnchorScope(),
2457 AS: IRP.getAssociatedType()->getPointerAddressSpace()))
2458 AttrKinds.push_back(Elt: Attribute::Dereferenceable);
2459 if (A.hasAttr(IRP, AKs: AttrKinds, IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NonNull))
2460 return true;
2461
2462 DominatorTree *DT = nullptr;
2463 AssumptionCache *AC = nullptr;
2464 InformationCache &InfoCache = A.getInfoCache();
2465 if (const Function *Fn = IRP.getAnchorScope()) {
2466 if (!Fn->isDeclaration()) {
2467 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2468 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2469 }
2470 }
2471
2472 SmallVector<AA::ValueAndContext> Worklist;
2473 if (IRP.getPositionKind() != IRP_RETURNED) {
2474 Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2475 } else {
2476 bool UsedAssumedInformation = false;
2477 if (!A.checkForAllInstructions(
2478 Pred: [&](Instruction &I) {
2479 Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2480 return true;
2481 },
2482 Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2483 UsedAssumedInformation, CheckBBLivenessOnly: false, /*CheckPotentiallyDead=*/true))
2484 return false;
2485 }
2486
2487 if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2488 return !isKnownNonZero(
2489 V: VAC.getValue(),
2490 Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2491 }))
2492 return false;
2493
2494 A.manifestAttrs(IRP, DeducedAttrs: {Attribute::get(Context&: IRP.getAnchorValue().getContext(),
2495 Kind: Attribute::NonNull)});
2496 return true;
2497}
2498
2499namespace {
2500static int64_t getKnownNonNullAndDerefBytesForUse(
2501 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2502 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2503 TrackUse = false;
2504
2505 const Value *UseV = U->get();
2506 if (!UseV->getType()->isPointerTy())
2507 return 0;
2508
2509 // We need to follow common pointer manipulation uses to the accesses they
2510 // feed into. We can try to be smart to avoid looking through things we do not
2511 // like for now, e.g., non-inbounds GEPs.
2512 if (isa<CastInst>(Val: I)) {
2513 TrackUse = true;
2514 return 0;
2515 }
2516
2517 if (isa<GetElementPtrInst>(Val: I)) {
2518 TrackUse = true;
2519 return 0;
2520 }
2521
2522 Type *PtrTy = UseV->getType();
2523 const Function *F = I->getFunction();
2524 bool NullPointerIsDefined =
2525 F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2526 const DataLayout &DL = A.getInfoCache().getDL();
2527 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2528 if (CB->isBundleOperand(U)) {
2529 if (RetainedKnowledge RK = getKnowledgeFromUse(
2530 U, AttrKinds: {Attribute::NonNull, Attribute::Dereferenceable})) {
2531 IsNonNull |=
2532 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2533 return RK.ArgValue;
2534 }
2535 return 0;
2536 }
2537
2538 if (CB->isCallee(U)) {
2539 IsNonNull |= !NullPointerIsDefined;
2540 return 0;
2541 }
2542
2543 unsigned ArgNo = CB->getArgOperandNo(U);
2544 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2545 // As long as we only use known information there is no need to track
2546 // dependences here.
2547 bool IsKnownNonNull;
2548 AA::hasAssumedIRAttr<Attribute::NonNull>(A, QueryingAA: &QueryingAA, IRP,
2549 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
2550 IsNonNull |= IsKnownNonNull;
2551 auto *DerefAA =
2552 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2553 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2554 }
2555
2556 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2557 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2558 Loc->Size.isScalable() || I->isVolatile())
2559 return 0;
2560
2561 int64_t Offset;
2562 const Value *Base =
2563 getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2564 if (Base && Base == &AssociatedValue) {
2565 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2566 IsNonNull |= !NullPointerIsDefined;
2567 return std::max(a: int64_t(0), b: DerefBytes);
2568 }
2569
2570 /// Corner case when an offset is 0.
2571 Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2572 /*AllowNonInbounds*/ true);
2573 if (Base && Base == &AssociatedValue && Offset == 0) {
2574 int64_t DerefBytes = Loc->Size.getValue();
2575 IsNonNull |= !NullPointerIsDefined;
2576 return std::max(a: int64_t(0), b: DerefBytes);
2577 }
2578
2579 return 0;
2580}
2581
2582struct AANonNullImpl : AANonNull {
2583 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2584
2585 /// See AbstractAttribute::initialize(...).
2586 void initialize(Attributor &A) override {
2587 Value &V = *getAssociatedValue().stripPointerCasts();
2588 if (isa<ConstantPointerNull>(Val: V)) {
2589 indicatePessimisticFixpoint();
2590 return;
2591 }
2592
2593 if (Instruction *CtxI = getCtxI())
2594 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
2595 }
2596
2597 /// See followUsesInMBEC
2598 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2599 AANonNull::StateType &State) {
2600 bool IsNonNull = false;
2601 bool TrackUse = false;
2602 getKnownNonNullAndDerefBytesForUse(A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I,
2603 IsNonNull, TrackUse);
2604 State.setKnown(IsNonNull);
2605 return TrackUse;
2606 }
2607
2608 /// See AbstractAttribute::getAsStr().
2609 const std::string getAsStr(Attributor *A) const override {
2610 return getAssumed() ? "nonnull" : "may-null";
2611 }
2612};
2613
2614/// NonNull attribute for a floating value.
2615struct AANonNullFloating : public AANonNullImpl {
2616 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2617 : AANonNullImpl(IRP, A) {}
2618
2619 /// See AbstractAttribute::updateImpl(...).
2620 ChangeStatus updateImpl(Attributor &A) override {
2621 auto CheckIRP = [&](const IRPosition &IRP) {
2622 bool IsKnownNonNull;
2623 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2624 A, QueryingAA: *this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
2625 };
2626
2627 bool Stripped;
2628 bool UsedAssumedInformation = false;
2629 Value *AssociatedValue = &getAssociatedValue();
2630 SmallVector<AA::ValueAndContext> Values;
2631 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2632 S: AA::AnyScope, UsedAssumedInformation))
2633 Stripped = false;
2634 else
2635 Stripped =
2636 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2637
2638 if (!Stripped) {
2639 bool IsKnown;
2640 if (auto *PHI = dyn_cast<PHINode>(Val: AssociatedValue))
2641 if (llvm::all_of(Range: PHI->incoming_values(), P: [&](Value *Op) {
2642 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2643 A, QueryingAA: this, IRP: IRPosition::value(V: *Op), DepClass: DepClassTy::OPTIONAL,
2644 IsKnown);
2645 }))
2646 return ChangeStatus::UNCHANGED;
2647 if (auto *Select = dyn_cast<SelectInst>(Val: AssociatedValue))
2648 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2649 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getFalseValue()),
2650 DepClass: DepClassTy::OPTIONAL, IsKnown) &&
2651 AA::hasAssumedIRAttr<Attribute::NonNull>(
2652 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getTrueValue()),
2653 DepClass: DepClassTy::OPTIONAL, IsKnown))
2654 return ChangeStatus::UNCHANGED;
2655
2656 // If we haven't stripped anything we might still be able to use a
2657 // different AA, but only if the IRP changes. Effectively when we
2658 // interpret this not as a call site value but as a floating/argument
2659 // value.
2660 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2661 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2662 return indicatePessimisticFixpoint();
2663 return ChangeStatus::UNCHANGED;
2664 }
2665
2666 for (const auto &VAC : Values)
2667 if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2668 return indicatePessimisticFixpoint();
2669
2670 return ChangeStatus::UNCHANGED;
2671 }
2672
2673 /// See AbstractAttribute::trackStatistics()
2674 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2675};
2676
2677/// NonNull attribute for function return value.
2678struct AANonNullReturned final
2679 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2680 false, AANonNull::IRAttributeKind, false> {
2681 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2682 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2683 false, Attribute::NonNull, false>(IRP, A) {
2684 }
2685
2686 /// See AbstractAttribute::getAsStr().
2687 const std::string getAsStr(Attributor *A) const override {
2688 return getAssumed() ? "nonnull" : "may-null";
2689 }
2690
2691 /// See AbstractAttribute::trackStatistics()
2692 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2693};
2694
2695/// NonNull attribute for function argument.
2696struct AANonNullArgument final
2697 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2698 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2699 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2700
2701 /// See AbstractAttribute::trackStatistics()
2702 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2703};
2704
2705struct AANonNullCallSiteArgument final : AANonNullFloating {
2706 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2707 : AANonNullFloating(IRP, A) {}
2708
2709 /// See AbstractAttribute::trackStatistics()
2710 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2711};
2712
2713/// NonNull attribute for a call site return position.
2714struct AANonNullCallSiteReturned final
2715 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2716 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2717 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2718
2719 /// See AbstractAttribute::trackStatistics()
2720 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2721};
2722} // namespace
2723
2724/// ------------------------ Must-Progress Attributes --------------------------
2725namespace {
2726struct AAMustProgressImpl : public AAMustProgress {
2727 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2728 : AAMustProgress(IRP, A) {}
2729
2730 /// See AbstractAttribute::initialize(...).
2731 void initialize(Attributor &A) override {
2732 bool IsKnown;
2733 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2734 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2735 (void)IsKnown;
2736 }
2737
2738 /// See AbstractAttribute::getAsStr()
2739 const std::string getAsStr(Attributor *A) const override {
2740 return getAssumed() ? "mustprogress" : "may-not-progress";
2741 }
2742};
2743
2744struct AAMustProgressFunction final : AAMustProgressImpl {
2745 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2746 : AAMustProgressImpl(IRP, A) {}
2747
2748 /// See AbstractAttribute::updateImpl(...).
2749 ChangeStatus updateImpl(Attributor &A) override {
2750 bool IsKnown;
2751 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2752 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown)) {
2753 if (IsKnown)
2754 return indicateOptimisticFixpoint();
2755 return ChangeStatus::UNCHANGED;
2756 }
2757
2758 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2759 IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2760 bool IsKnownMustProgress;
2761 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2762 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress,
2763 /* IgnoreSubsumingPositions */ true);
2764 };
2765
2766 bool AllCallSitesKnown = true;
2767 if (!A.checkForAllCallSites(Pred: CheckForMustProgress, QueryingAA: *this,
2768 /* RequireAllCallSites */ true,
2769 UsedAssumedInformation&: AllCallSitesKnown))
2770 return indicatePessimisticFixpoint();
2771
2772 return ChangeStatus::UNCHANGED;
2773 }
2774
2775 /// See AbstractAttribute::trackStatistics()
2776 void trackStatistics() const override {
2777 STATS_DECLTRACK_FN_ATTR(mustprogress)
2778 }
2779};
2780
2781/// MustProgress attribute deduction for a call sites.
2782struct AAMustProgressCallSite final : AAMustProgressImpl {
2783 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2784 : AAMustProgressImpl(IRP, A) {}
2785
2786 /// See AbstractAttribute::updateImpl(...).
2787 ChangeStatus updateImpl(Attributor &A) override {
2788 // TODO: Once we have call site specific value information we can provide
2789 // call site specific liveness information and then it makes
2790 // sense to specialize attributes for call sites arguments instead of
2791 // redirecting requests to the callee argument.
2792 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2793 bool IsKnownMustProgress;
2794 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2795 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress))
2796 return indicatePessimisticFixpoint();
2797 return ChangeStatus::UNCHANGED;
2798 }
2799
2800 /// See AbstractAttribute::trackStatistics()
2801 void trackStatistics() const override {
2802 STATS_DECLTRACK_CS_ATTR(mustprogress);
2803 }
2804};
2805} // namespace
2806
2807/// ------------------------ No-Recurse Attributes ----------------------------
2808
2809namespace {
2810struct AANoRecurseImpl : public AANoRecurse {
2811 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2812
2813 /// See AbstractAttribute::initialize(...).
2814 void initialize(Attributor &A) override {
2815 bool IsKnown;
2816 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2817 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2818 (void)IsKnown;
2819 }
2820
2821 /// See AbstractAttribute::getAsStr()
2822 const std::string getAsStr(Attributor *A) const override {
2823 return getAssumed() ? "norecurse" : "may-recurse";
2824 }
2825};
2826
2827struct AANoRecurseFunction final : AANoRecurseImpl {
2828 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2829 : AANoRecurseImpl(IRP, A) {}
2830
2831 /// See AbstractAttribute::updateImpl(...).
2832 ChangeStatus updateImpl(Attributor &A) override {
2833
2834 // If all live call sites are known to be no-recurse, we are as well.
2835 auto CallSitePred = [&](AbstractCallSite ACS) {
2836 bool IsKnownNoRecurse;
2837 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2838 A, QueryingAA: this,
2839 IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
2840 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoRecurse))
2841 return false;
2842 return IsKnownNoRecurse;
2843 };
2844 bool UsedAssumedInformation = false;
2845 if (A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
2846 UsedAssumedInformation)) {
2847 // If we know all call sites and all are known no-recurse, we are done.
2848 // If all known call sites, which might not be all that exist, are known
2849 // to be no-recurse, we are not done but we can continue to assume
2850 // no-recurse. If one of the call sites we have not visited will become
2851 // live, another update is triggered.
2852 if (!UsedAssumedInformation)
2853 indicateOptimisticFixpoint();
2854 return ChangeStatus::UNCHANGED;
2855 }
2856
2857 const AAInterFnReachability *EdgeReachability =
2858 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: getIRPosition(),
2859 DepClass: DepClassTy::REQUIRED);
2860 if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2861 return indicatePessimisticFixpoint();
2862 return ChangeStatus::UNCHANGED;
2863 }
2864
2865 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2866};
2867
2868/// NoRecurse attribute deduction for a call sites.
2869struct AANoRecurseCallSite final
2870 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2871 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2872 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2873
2874 /// See AbstractAttribute::trackStatistics()
2875 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2876};
2877} // namespace
2878
2879/// ------------------------ No-Convergent Attribute --------------------------
2880
2881namespace {
2882struct AANonConvergentImpl : public AANonConvergent {
2883 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2884 : AANonConvergent(IRP, A) {}
2885
2886 /// See AbstractAttribute::getAsStr()
2887 const std::string getAsStr(Attributor *A) const override {
2888 return getAssumed() ? "non-convergent" : "may-be-convergent";
2889 }
2890};
2891
2892struct AANonConvergentFunction final : AANonConvergentImpl {
2893 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2894 : AANonConvergentImpl(IRP, A) {}
2895
2896 /// See AbstractAttribute::updateImpl(...).
2897 ChangeStatus updateImpl(Attributor &A) override {
2898 // If all function calls are known to not be convergent, we are not
2899 // convergent.
2900 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2901 CallBase &CB = cast<CallBase>(Val&: Inst);
2902 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2903 if (!Callee || Callee->isIntrinsic()) {
2904 return false;
2905 }
2906 if (Callee->isDeclaration()) {
2907 return !Callee->hasFnAttribute(Kind: Attribute::Convergent);
2908 }
2909 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2910 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2911 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2912 };
2913
2914 bool UsedAssumedInformation = false;
2915 if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2916 UsedAssumedInformation)) {
2917 return indicatePessimisticFixpoint();
2918 }
2919 return ChangeStatus::UNCHANGED;
2920 }
2921
2922 ChangeStatus manifest(Attributor &A) override {
2923 if (isKnownNotConvergent() &&
2924 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::Convergent)) {
2925 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::Convergent});
2926 return ChangeStatus::CHANGED;
2927 }
2928 return ChangeStatus::UNCHANGED;
2929 }
2930
2931 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2932};
2933} // namespace
2934
2935/// -------------------- Undefined-Behavior Attributes ------------------------
2936
2937namespace {
2938struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2939 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2940 : AAUndefinedBehavior(IRP, A) {}
2941
2942 struct UBInfo {
2943 enum Kind {
2944 NullPtrAccess,
2945 UndefPtrAccess,
2946 UndefBranchCondition,
2947 UndefReturnValue,
2948 NullReturnViolatesNonNull,
2949 UndefCallArgument,
2950 NullArgViolatesNonNull,
2951 };
2952
2953 Kind K;
2954 std::optional<unsigned> ArgNo;
2955
2956 UBInfo(Kind K) : K(K), ArgNo(std::nullopt) {}
2957
2958 UBInfo(Kind K, std::optional<unsigned> ArgNo) : K(K), ArgNo(ArgNo) {}
2959 };
2960
2961 /// See AbstractAttribute::updateImpl(...).
2962 // through a pointer (i.e. also branches etc.)
2963 ChangeStatus updateImpl(Attributor &A) override {
2964 const size_t UBPrevSize = KnownUBInsts.size();
2965 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2966
2967 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2968 // Volatile accesses on null are not necessarily UB.
2969 if (I.isVolatile())
2970 return true;
2971
2972 // Skip instructions that are already saved.
2973 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
2974 return true;
2975
2976 // If we reach here, we know we have an instruction
2977 // that accesses memory through a pointer operand,
2978 // for which getPointerOperand() should give it to us.
2979 Value *PtrOp =
2980 const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2981 assert(PtrOp &&
2982 "Expected pointer operand of memory accessing instruction");
2983
2984 // Either we stopped and the appropriate action was taken,
2985 // or we got back a simplified value to continue.
2986 std::optional<Value *> SimplifiedPtrOp =
2987 stopOnUndefOrAssumed(A, V: PtrOp, I: &I, K: UBInfo::UndefPtrAccess);
2988 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2989 return true;
2990 const Value *PtrOpVal = *SimplifiedPtrOp;
2991
2992 // A memory access through a pointer is considered UB
2993 // only if the pointer has constant null value.
2994 // TODO: Expand it to not only check constant values.
2995 if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2996 AssumedNoUBInsts.insert(Ptr: &I);
2997 return true;
2998 }
2999 const Type *PtrTy = PtrOpVal->getType();
3000
3001 // Because we only consider instructions inside functions,
3002 // assume that a parent function exists.
3003 const Function *F = I.getFunction();
3004
3005 // A memory access using constant null pointer is only considered UB
3006 // if null pointer is _not_ defined for the target platform.
3007 if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
3008 AssumedNoUBInsts.insert(Ptr: &I);
3009 else
3010 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullPtrAccess);
3011 return true;
3012 };
3013
3014 auto InspectBrInstForUB = [&](Instruction &I) {
3015 // A conditional branch instruction is considered UB if it has `undef`
3016 // condition.
3017
3018 // Skip instructions that are already saved.
3019 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
3020 return true;
3021
3022 // We know we have a branch instruction.
3023 auto *BrInst = cast<CondBrInst>(Val: &I);
3024
3025 // Either we stopped and the appropriate action was taken,
3026 // or we got back a simplified value to continue.
3027 std::optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(
3028 A, V: BrInst->getCondition(), I: BrInst, K: UBInfo::UndefBranchCondition);
3029 if (!SimplifiedCond || !*SimplifiedCond)
3030 return true;
3031 AssumedNoUBInsts.insert(Ptr: &I);
3032 return true;
3033 };
3034
3035 auto InspectCallSiteForUB = [&](Instruction &I) {
3036 // Check whether a callsite always cause UB or not
3037
3038 // Skip instructions that are already saved.
3039 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
3040 return true;
3041
3042 // Check nonnull and noundef argument attribute violation for each
3043 // callsite.
3044 CallBase &CB = cast<CallBase>(Val&: I);
3045 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
3046 if (!Callee)
3047 return true;
3048 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3049 // If current argument is known to be simplified to null pointer and the
3050 // corresponding argument position is known to have nonnull attribute,
3051 // the argument is poison. Furthermore, if the argument is poison and
3052 // the position is known to have noundef attriubte, this callsite is
3053 // considered UB.
3054 if (idx >= Callee->arg_size())
3055 break;
3056 Value *ArgVal = CB.getArgOperand(i: idx);
3057 if (!ArgVal)
3058 continue;
3059 // Here, we handle three cases.
3060 // (1) Not having a value means it is dead. (we can replace the value
3061 // with undef)
3062 // (2) Simplified to undef. The argument violate noundef attriubte.
3063 // (3) Simplified to null pointer where known to be nonnull.
3064 // The argument is a poison value and violate noundef attribute.
3065 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
3066 bool IsKnownNoUndef;
3067 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3068 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3069 if (!IsKnownNoUndef)
3070 continue;
3071 bool UsedAssumedInformation = false;
3072 std::optional<Value *> SimplifiedVal =
3073 A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
3074 UsedAssumedInformation, S: AA::Interprocedural);
3075 if (UsedAssumedInformation)
3076 continue;
3077 if (SimplifiedVal && !*SimplifiedVal)
3078 return true;
3079 if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3080 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo(UBInfo::UndefCallArgument, idx));
3081 continue;
3082 }
3083 if (!ArgVal->getType()->isPointerTy() ||
3084 !isa<ConstantPointerNull>(Val: **SimplifiedVal))
3085 continue;
3086 bool IsKnownNonNull;
3087 AA::hasAssumedIRAttr<Attribute::NonNull>(
3088 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
3089 if (IsKnownNonNull)
3090 KnownUBInsts.try_emplace(Key: &I,
3091 Args: UBInfo(UBInfo::NullArgViolatesNonNull, idx));
3092 }
3093 return true;
3094 };
3095
3096 auto InspectReturnInstForUB = [&](Instruction &I) {
3097 auto &RI = cast<ReturnInst>(Val&: I);
3098 // Either we stopped and the appropriate action was taken,
3099 // or we got back a simplified return value to continue.
3100 std::optional<Value *> SimplifiedRetValue = stopOnUndefOrAssumed(
3101 A, V: RI.getReturnValue(), I: &I, K: UBInfo::UndefReturnValue);
3102 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3103 return true;
3104
3105 // Check if a return instruction always cause UB or not
3106 // Note: It is guaranteed that the returned position of the anchor
3107 // scope has noundef attribute when this is called.
3108 // We also ensure the return position is not "assumed dead"
3109 // because the returned value was then potentially simplified to
3110 // `undef` in AAReturnedValues without removing the `noundef`
3111 // attribute yet.
3112
3113 // When the returned position has noundef attriubte, UB occurs in the
3114 // following cases.
3115 // (1) Returned value is known to be undef.
3116 // (2) The value is known to be a null pointer and the returned
3117 // position has nonnull attribute (because the returned value is
3118 // poison).
3119 if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3120 bool IsKnownNonNull;
3121 AA::hasAssumedIRAttr<Attribute::NonNull>(
3122 A, QueryingAA: this, IRP: IRPosition::returned(F: *getAnchorScope()), DepClass: DepClassTy::NONE,
3123 IsKnown&: IsKnownNonNull);
3124 if (IsKnownNonNull)
3125 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullReturnViolatesNonNull);
3126 }
3127
3128 return true;
3129 };
3130
3131 bool UsedAssumedInformation = false;
3132 A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3133 Opcodes: {Instruction::Load, Instruction::Store,
3134 Instruction::AtomicCmpXchg,
3135 Instruction::AtomicRMW},
3136 UsedAssumedInformation,
3137 /* CheckBBLivenessOnly */ true);
3138 A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::CondBr},
3139 UsedAssumedInformation,
3140 /* CheckBBLivenessOnly */ true);
3141 A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3142 UsedAssumedInformation);
3143
3144 // If the returned position of the anchor scope has noundef attriubte, check
3145 // all returned instructions.
3146 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3147 const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3148 if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3149 bool IsKnownNoUndef;
3150 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3151 A, QueryingAA: this, IRP: ReturnIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3152 if (IsKnownNoUndef)
3153 A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3154 Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3155 /* CheckBBLivenessOnly */ true);
3156 }
3157 }
3158
3159 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3160 UBPrevSize != KnownUBInsts.size())
3161 return ChangeStatus::CHANGED;
3162 return ChangeStatus::UNCHANGED;
3163 }
3164
3165 bool isKnownToCauseUB(Instruction *I) const override {
3166 return KnownUBInsts.count(Key: I);
3167 }
3168
3169 bool isAssumedToCauseUB(Instruction *I) const override {
3170 // In simple words, if an instruction is not in the assumed to _not_
3171 // cause UB, then it is assumed UB (that includes those
3172 // in the KnownUBInsts set). The rest is boilerplate
3173 // is to ensure that it is one of the instructions we test
3174 // for UB.
3175
3176 switch (I->getOpcode()) {
3177 case Instruction::Load:
3178 case Instruction::Store:
3179 case Instruction::AtomicCmpXchg:
3180 case Instruction::AtomicRMW:
3181 case Instruction::CondBr:
3182 return !AssumedNoUBInsts.count(Ptr: I);
3183 default:
3184 return false;
3185 }
3186 return false;
3187 }
3188
3189 /// Emit an optimization remark explaining why \p I is known to cause UB,
3190 /// per \p Info, right before it is replaced with 'unreachable'.
3191 static void emitUBRemark(Attributor &A, Instruction *I, const UBInfo &Info) {
3192 auto Remark = [&](OptimizationRemark OR) {
3193 switch (Info.K) {
3194 case UBInfo::NullPtrAccess:
3195 case UBInfo::UndefPtrAccess: {
3196 return OR << "Memory access through a pointer known to be "
3197 << ore::NV("Pointer",
3198 getPointerOperand(I, /*AllowVolatile*/ true))
3199 << " is undefined behavior; replacing with 'unreachable'.";
3200 }
3201 case UBInfo::UndefBranchCondition:
3202 return OR << "Branch condition known to be "
3203 << ore::NV("Condition", cast<CondBrInst>(Val: I)->getCondition())
3204 << " is undefined behavior; replacing with 'unreachable'.";
3205 case UBInfo::UndefReturnValue:
3206 case UBInfo::NullReturnViolatesNonNull:
3207 return OR << "Value returned known to be "
3208 << ore::NV("ReturnValue",
3209 cast<ReturnInst>(Val: I)->getReturnValue())
3210 << " is undefined behavior; replacing with 'unreachable'.";
3211 case UBInfo::UndefCallArgument:
3212 case UBInfo::NullArgViolatesNonNull: {
3213 bool IsUndef = Info.K == UBInfo::UndefCallArgument;
3214 CallBase &CB = *cast<CallBase>(Val: I);
3215 OR << "Argument " << ore::NV("ArgNo", *Info.ArgNo)
3216 << " passed to parameter of ";
3217 if (auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand()))
3218 OR << ore::NV("Callee", Callee);
3219 else
3220 OR << "the callee";
3221 return OR << " known to be "
3222 << ore::NV("Argument", IsUndef ? "undef" : "null")
3223 << " is undefined behavior; replacing with 'unreachable'.";
3224 }
3225 }
3226 llvm_unreachable("Unknown UBInfo::Kind");
3227 };
3228 A.emitRemark<OptimizationRemark>(I, RemarkName: "UndefinedBehavior", RemarkCB&: Remark);
3229 }
3230
3231 ChangeStatus manifest(Attributor &A) override {
3232 if (KnownUBInsts.empty())
3233 return ChangeStatus::UNCHANGED;
3234 for (const auto &[I, Info] : KnownUBInsts) {
3235 emitUBRemark(A, I, Info);
3236 A.changeToUnreachableAfterManifest(I);
3237 }
3238 return ChangeStatus::CHANGED;
3239 }
3240
3241 /// See AbstractAttribute::getAsStr()
3242 const std::string getAsStr(Attributor *A) const override {
3243 return getAssumed() ? "undefined-behavior" : "no-ub";
3244 }
3245
3246 /// Note: The correctness of this analysis depends on the fact that the
3247 /// following 2 sets will stop changing after some point.
3248 /// "Change" here means that their size changes.
3249 /// The size of each set is monotonically increasing
3250 /// (we only add items to them) and it is upper bounded by the number of
3251 /// instructions in the processed function (we can never save more
3252 /// elements in either set than this number). Hence, at some point,
3253 /// they will stop increasing.
3254 /// Consequently, at some point, both sets will have stopped
3255 /// changing, effectively making the analysis reach a fixpoint.
3256
3257 /// Note: These 2 sets are disjoint and an instruction can be considered
3258 /// one of 3 things:
3259 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3260 /// the KnownUBInsts set.
3261 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3262 /// has a reason to assume it).
3263 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3264 /// could not find a reason to assume or prove that it can cause UB,
3265 /// hence it assumes it doesn't. We have a set for these instructions
3266 /// so that we don't reprocess them in every update.
3267 /// Note however that instructions in this set may cause UB.
3268
3269protected:
3270 /// A map from all live instructions _known_ to cause UB to the reason why,
3271 /// used to build actionable optimization remarks in manifest().
3272 MapVector<Instruction *, UBInfo> KnownUBInsts;
3273
3274private:
3275 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3276 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3277
3278 // Should be called on updates in which if we're processing an instruction
3279 // \p I that depends on a value \p V, one of the following has to happen:
3280 // - If the value is assumed, then stop.
3281 // - If the value is known but undef, then consider it UB for \p K.
3282 // - Otherwise, do specific processing with the simplified value.
3283 // We return std::nullopt in the first 2 cases to signify that an appropriate
3284 // action was taken and the caller should stop.
3285 // Otherwise, we return the simplified value that the caller should
3286 // use for specific processing.
3287 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3288 Instruction *I, UBInfo::Kind K) {
3289 bool UsedAssumedInformation = false;
3290 std::optional<Value *> SimplifiedV =
3291 A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3292 UsedAssumedInformation, S: AA::Interprocedural);
3293 if (!UsedAssumedInformation) {
3294 // Don't depend on assumed values.
3295 if (!SimplifiedV) {
3296 // If it is known (which we tested above) but it doesn't have a value,
3297 // then we can assume `undef` and hence the instruction is UB.
3298 KnownUBInsts.try_emplace(Key: I, Args&: K);
3299 return std::nullopt;
3300 }
3301 if (!*SimplifiedV)
3302 return nullptr;
3303 V = *SimplifiedV;
3304 }
3305 if (isa<UndefValue>(Val: V)) {
3306 KnownUBInsts.try_emplace(Key: I, Args&: K);
3307 return std::nullopt;
3308 }
3309 return V;
3310 }
3311};
3312
3313struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3314 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3315 : AAUndefinedBehaviorImpl(IRP, A) {}
3316
3317 /// See AbstractAttribute::trackStatistics()
3318 void trackStatistics() const override {
3319 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3320 "Number of instructions known to have UB");
3321 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3322 KnownUBInsts.size();
3323 }
3324};
3325} // namespace
3326
3327/// ------------------------ Will-Return Attributes ----------------------------
3328
3329namespace {
3330// Helper function that checks whether a function has any cycle which we don't
3331// know if it is bounded or not.
3332// Loops with maximum trip count are considered bounded, any other cycle not.
3333static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3334 ScalarEvolution *SE =
3335 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3336 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3337 // If either SCEV or LoopInfo is not available for the function then we assume
3338 // any cycle to be unbounded cycle.
3339 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3340 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3341 if (!SE || !LI) {
3342 for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3343 if (SCCI.hasCycle())
3344 return true;
3345 return false;
3346 }
3347
3348 // If there's irreducible control, the function may contain non-loop cycles.
3349 if (mayContainIrreducibleControl(F, LI))
3350 return true;
3351
3352 // Any loop that does not have a max trip count is considered unbounded cycle.
3353 for (auto *L : LI->getLoopsInPreorder()) {
3354 if (!SE->getSmallConstantMaxTripCount(L))
3355 return true;
3356 }
3357 return false;
3358}
3359
3360struct AAWillReturnImpl : public AAWillReturn {
3361 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3362 : AAWillReturn(IRP, A) {}
3363
3364 /// See AbstractAttribute::initialize(...).
3365 void initialize(Attributor &A) override {
3366 bool IsKnown;
3367 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3368 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3369 (void)IsKnown;
3370 }
3371
3372 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3373 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3374 if (!A.hasAttr(IRP: getIRPosition(), AKs: {Attribute::MustProgress}))
3375 return false;
3376
3377 bool IsKnown;
3378 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3379 return IsKnown || !KnownOnly;
3380 return false;
3381 }
3382
3383 /// See AbstractAttribute::updateImpl(...).
3384 ChangeStatus updateImpl(Attributor &A) override {
3385 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3386 return ChangeStatus::UNCHANGED;
3387
3388 auto CheckForWillReturn = [&](Instruction &I) {
3389 IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3390 bool IsKnown;
3391 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3392 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown)) {
3393 if (IsKnown)
3394 return true;
3395 } else {
3396 return false;
3397 }
3398 bool IsKnownNoRecurse;
3399 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3400 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoRecurse);
3401 };
3402
3403 bool UsedAssumedInformation = false;
3404 if (!A.checkForAllCallLikeInstructions(Pred: CheckForWillReturn, QueryingAA: *this,
3405 UsedAssumedInformation))
3406 return indicatePessimisticFixpoint();
3407
3408 auto CheckForVolatile = [&](Instruction &I) {
3409 // Volatile operations are not willreturn.
3410 return !I.isVolatile();
3411 };
3412 if (!A.checkForAllInstructions(Pred: CheckForVolatile, QueryingAA: *this,
3413 Opcodes: {Instruction::Load, Instruction::Store,
3414 Instruction::AtomicCmpXchg,
3415 Instruction::AtomicRMW},
3416 UsedAssumedInformation))
3417 return indicatePessimisticFixpoint();
3418
3419 return ChangeStatus::UNCHANGED;
3420 }
3421
3422 /// See AbstractAttribute::getAsStr()
3423 const std::string getAsStr(Attributor *A) const override {
3424 return getAssumed() ? "willreturn" : "may-noreturn";
3425 }
3426};
3427
3428struct AAWillReturnFunction final : AAWillReturnImpl {
3429 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3430 : AAWillReturnImpl(IRP, A) {}
3431
3432 /// See AbstractAttribute::initialize(...).
3433 void initialize(Attributor &A) override {
3434 AAWillReturnImpl::initialize(A);
3435
3436 Function *F = getAnchorScope();
3437 assert(F && "Did expect an anchor function");
3438 if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3439 indicatePessimisticFixpoint();
3440 }
3441
3442 /// See AbstractAttribute::trackStatistics()
3443 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3444};
3445
3446/// WillReturn attribute deduction for a call sites.
3447struct AAWillReturnCallSite final
3448 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3449 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3450 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3451
3452 /// See AbstractAttribute::updateImpl(...).
3453 ChangeStatus updateImpl(Attributor &A) override {
3454 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3455 return ChangeStatus::UNCHANGED;
3456
3457 return AACalleeToCallSite::updateImpl(A);
3458 }
3459
3460 /// See AbstractAttribute::trackStatistics()
3461 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3462};
3463} // namespace
3464
3465/// -------------------AAIntraFnReachability Attribute--------------------------
3466
3467/// All information associated with a reachability query. This boilerplate code
3468/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3469/// different \p ToTy values.
3470template <typename ToTy> struct ReachabilityQueryInfo {
3471 enum class Reachable {
3472 No,
3473 Yes,
3474 };
3475
3476 /// Start here,
3477 const Instruction *From = nullptr;
3478 /// reach this place,
3479 const ToTy *To = nullptr;
3480 /// without going through any of these instructions,
3481 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3482 /// and remember if it worked:
3483 Reachable Result = Reachable::No;
3484
3485 /// Precomputed hash for this RQI.
3486 unsigned Hash = 0;
3487
3488 unsigned computeHashValue() const {
3489 assert(Hash == 0 && "Computed hash twice!");
3490 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3491 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3492 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3493 detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3494 b: InstSetDMI::getHashValue(BES: ExclusionSet));
3495 }
3496
3497 ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3498 : From(From), To(To) {}
3499
3500 /// Constructor replacement to ensure unique and stable sets are used for the
3501 /// cache.
3502 ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3503 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3504 : From(&From), To(&To), ExclusionSet(ES) {
3505
3506 if (!ES || ES->empty()) {
3507 ExclusionSet = nullptr;
3508 } else if (MakeUnique) {
3509 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3510 }
3511 }
3512
3513 ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3514 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3515};
3516
3517namespace llvm {
3518template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3519 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3520 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3521
3522 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3523 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3524 }
3525 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3526 const ReachabilityQueryInfo<ToTy> *RHS) {
3527 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3528 return false;
3529 return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3530 }
3531};
3532
3533} // namespace llvm
3534
3535namespace {
3536
3537template <typename BaseTy, typename ToTy>
3538struct CachedReachabilityAA : public BaseTy {
3539 using RQITy = ReachabilityQueryInfo<ToTy>;
3540
3541 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3542
3543 /// See AbstractAttribute::isQueryAA.
3544 bool isQueryAA() const override { return true; }
3545
3546 /// See AbstractAttribute::updateImpl(...).
3547 ChangeStatus updateImpl(Attributor &A) override {
3548 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3549 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3550 RQITy *RQI = QueryVector[u];
3551 if (RQI->Result == RQITy::Reachable::No &&
3552 isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3553 Changed = ChangeStatus::CHANGED;
3554 }
3555 return Changed;
3556 }
3557
3558 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3559 bool IsTemporaryRQI) = 0;
3560
3561 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3562 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3563 RQI.Result = Result;
3564
3565 // Remove the temporary RQI from the cache.
3566 if (IsTemporaryRQI)
3567 QueryCache.erase(&RQI);
3568
3569 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3570 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3571 // this query. 2) We did not use the exclusion set, potentially because
3572 // there is none.
3573 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3574 RQITy PlainRQI(RQI.From, RQI.To);
3575 if (!QueryCache.count(&PlainRQI)) {
3576 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3577 RQIPtr->Result = Result;
3578 QueryVector.push_back(RQIPtr);
3579 QueryCache.insert(RQIPtr);
3580 }
3581 }
3582
3583 // Check if we need to insert a new permanent RQI with the exclusion set.
3584 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3585 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3586 "Did not expect empty set!");
3587 RQITy *RQIPtr = new (A.Allocator)
3588 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3589 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3590 RQIPtr->Result = Result;
3591 assert(!QueryCache.count(RQIPtr));
3592 QueryVector.push_back(RQIPtr);
3593 QueryCache.insert(RQIPtr);
3594 }
3595
3596 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3597 A.registerForUpdate(AA&: *this);
3598 return Result == RQITy::Reachable::Yes;
3599 }
3600
3601 const std::string getAsStr(Attributor *A) const override {
3602 // TODO: Return the number of reachable queries.
3603 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3604 }
3605
3606 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3607 typename RQITy::Reachable &Result) {
3608 if (!this->getState().isValidState()) {
3609 Result = RQITy::Reachable::Yes;
3610 return true;
3611 }
3612
3613 // If we have an exclusion set we might be able to find our answer by
3614 // ignoring it first.
3615 if (StackRQI.ExclusionSet) {
3616 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3617 auto It = QueryCache.find(&PlainRQI);
3618 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3619 Result = RQITy::Reachable::No;
3620 return true;
3621 }
3622 }
3623
3624 auto It = QueryCache.find(&StackRQI);
3625 if (It != QueryCache.end()) {
3626 Result = (*It)->Result;
3627 return true;
3628 }
3629
3630 // Insert a temporary for recursive queries. We will replace it with a
3631 // permanent entry later.
3632 QueryCache.insert(&StackRQI);
3633 return false;
3634 }
3635
3636private:
3637 SmallVector<RQITy *> QueryVector;
3638 DenseSet<RQITy *> QueryCache;
3639};
3640
3641struct AAIntraFnReachabilityFunction final
3642 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3643 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3644 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3645 : Base(IRP, A) {
3646 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3647 F: *IRP.getAssociatedFunction());
3648 }
3649
3650 bool isAssumedReachable(
3651 Attributor &A, const Instruction &From, const Instruction &To,
3652 const AA::InstExclusionSetTy *ExclusionSet) const override {
3653 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3654 if (&From == &To)
3655 return true;
3656
3657 RQITy StackRQI(A, From, To, ExclusionSet, false);
3658 RQITy::Reachable Result;
3659 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3660 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3661 /*IsTemporaryRQI=*/true);
3662 return Result == RQITy::Reachable::Yes;
3663 }
3664
3665 ChangeStatus updateImpl(Attributor &A) override {
3666 // We only depend on liveness. DeadEdges is all we care about, check if any
3667 // of them changed.
3668 auto *LivenessAA =
3669 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3670 if (LivenessAA &&
3671 llvm::all_of(Range&: DeadEdges,
3672 P: [&](const auto &DeadEdge) {
3673 return LivenessAA->isEdgeDead(From: DeadEdge.first,
3674 To: DeadEdge.second);
3675 }) &&
3676 llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3677 return LivenessAA->isAssumedDead(BB);
3678 })) {
3679 return ChangeStatus::UNCHANGED;
3680 }
3681 DeadEdges.clear();
3682 DeadBlocks.clear();
3683 return Base::updateImpl(A);
3684 }
3685
3686 bool isReachableImpl(Attributor &A, RQITy &RQI,
3687 bool IsTemporaryRQI) override {
3688 const Instruction *Origin = RQI.From;
3689 bool UsedExclusionSet = false;
3690
3691 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3692 const AA::InstExclusionSetTy *ExclusionSet) {
3693 const Instruction *IP = &From;
3694 while (IP && IP != &To) {
3695 if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3696 UsedExclusionSet = true;
3697 break;
3698 }
3699 IP = IP->getNextNode();
3700 }
3701 return IP == &To;
3702 };
3703
3704 const BasicBlock *FromBB = RQI.From->getParent();
3705 const BasicBlock *ToBB = RQI.To->getParent();
3706 assert(FromBB->getParent() == ToBB->getParent() &&
3707 "Not an intra-procedural query!");
3708
3709 // Check intra-block reachability, however, other reaching paths are still
3710 // possible.
3711 if (FromBB == ToBB &&
3712 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3713 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3714 IsTemporaryRQI);
3715
3716 // Check if reaching the ToBB block is sufficient or if even that would not
3717 // ensure reaching the target. In the latter case we are done.
3718 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3719 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3720 IsTemporaryRQI);
3721
3722 const Function *Fn = FromBB->getParent();
3723 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3724 if (RQI.ExclusionSet)
3725 for (auto *I : *RQI.ExclusionSet)
3726 if (I->getFunction() == Fn)
3727 ExclusionBlocks.insert(Ptr: I->getParent());
3728
3729 // Check if we make it out of the FromBB block at all.
3730 if (ExclusionBlocks.count(Ptr: FromBB) &&
3731 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3732 RQI.ExclusionSet))
3733 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3734
3735 auto *LivenessAA =
3736 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3737 if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3738 DeadBlocks.insert(V: ToBB);
3739 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3740 IsTemporaryRQI);
3741 }
3742
3743 SmallPtrSet<const BasicBlock *, 16> Visited;
3744 SmallVector<const BasicBlock *, 16> Worklist;
3745 Worklist.push_back(Elt: FromBB);
3746
3747 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3748 while (!Worklist.empty()) {
3749 const BasicBlock *BB = Worklist.pop_back_val();
3750 if (!Visited.insert(Ptr: BB).second)
3751 continue;
3752 for (const BasicBlock *SuccBB : successors(BB)) {
3753 if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3754 LocalDeadEdges.insert(V: {BB, SuccBB});
3755 continue;
3756 }
3757 // We checked before if we just need to reach the ToBB block.
3758 if (SuccBB == ToBB)
3759 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3760 IsTemporaryRQI);
3761 if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3762 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3763 IsTemporaryRQI);
3764
3765 if (ExclusionBlocks.count(Ptr: SuccBB)) {
3766 UsedExclusionSet = true;
3767 continue;
3768 }
3769 Worklist.push_back(Elt: SuccBB);
3770 }
3771 }
3772
3773 DeadEdges.insert_range(R&: LocalDeadEdges);
3774 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3775 IsTemporaryRQI);
3776 }
3777
3778 /// See AbstractAttribute::trackStatistics()
3779 void trackStatistics() const override {}
3780
3781private:
3782 // Set of assumed dead blocks we used in the last query. If any changes we
3783 // update the state.
3784 DenseSet<const BasicBlock *> DeadBlocks;
3785
3786 // Set of assumed dead edges we used in the last query. If any changes we
3787 // update the state.
3788 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3789
3790 /// The dominator tree of the function to short-circuit reasoning.
3791 const DominatorTree *DT = nullptr;
3792};
3793} // namespace
3794
3795/// ------------------------ NoAlias Argument Attribute ------------------------
3796
3797bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3798 Attribute::AttrKind ImpliedAttributeKind,
3799 bool IgnoreSubsumingPositions) {
3800 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3801 "Unexpected attribute kind");
3802 Value *Val = &IRP.getAssociatedValue();
3803 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3804 if (isa<AllocaInst>(Val))
3805 return true;
3806 } else {
3807 IgnoreSubsumingPositions = true;
3808 }
3809
3810 if (isa<UndefValue>(Val))
3811 return true;
3812
3813 if (isa<ConstantPointerNull>(Val) &&
3814 !NullPointerIsDefined(F: IRP.getAnchorScope(),
3815 AS: Val->getType()->getPointerAddressSpace()))
3816 return true;
3817
3818 if (A.hasAttr(IRP, AKs: {Attribute::ByVal, Attribute::NoAlias},
3819 IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NoAlias))
3820 return true;
3821
3822 return false;
3823}
3824
3825namespace {
3826struct AANoAliasImpl : AANoAlias {
3827 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3828 assert(getAssociatedType()->isPointerTy() &&
3829 "Noalias is a pointer attribute");
3830 }
3831
3832 const std::string getAsStr(Attributor *A) const override {
3833 return getAssumed() ? "noalias" : "may-alias";
3834 }
3835};
3836
3837/// NoAlias attribute for a floating value.
3838struct AANoAliasFloating final : AANoAliasImpl {
3839 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3840 : AANoAliasImpl(IRP, A) {}
3841
3842 /// See AbstractAttribute::updateImpl(...).
3843 ChangeStatus updateImpl(Attributor &A) override {
3844 // TODO: Implement this.
3845 return indicatePessimisticFixpoint();
3846 }
3847
3848 /// See AbstractAttribute::trackStatistics()
3849 void trackStatistics() const override {
3850 STATS_DECLTRACK_FLOATING_ATTR(noalias)
3851 }
3852};
3853
3854/// NoAlias attribute for an argument.
3855struct AANoAliasArgument final
3856 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3857 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3858 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3859
3860 /// See AbstractAttribute::update(...).
3861 ChangeStatus updateImpl(Attributor &A) override {
3862 // We have to make sure no-alias on the argument does not break
3863 // synchronization when this is a callback argument, see also [1] below.
3864 // If synchronization cannot be affected, we delegate to the base updateImpl
3865 // function, otherwise we give up for now.
3866
3867 // If the function is no-sync, no-alias cannot break synchronization.
3868 bool IsKnownNoSycn;
3869 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3870 A, QueryingAA: this, IRP: IRPosition::function_scope(IRP: getIRPosition()),
3871 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn))
3872 return Base::updateImpl(A);
3873
3874 // If the argument is read-only, no-alias cannot break synchronization.
3875 bool IsKnown;
3876 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3877 return Base::updateImpl(A);
3878
3879 // If the argument is never passed through callbacks, no-alias cannot break
3880 // synchronization.
3881 bool UsedAssumedInformation = false;
3882 if (A.checkForAllCallSites(
3883 Pred: [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, QueryingAA: *this,
3884 RequireAllCallSites: true, UsedAssumedInformation))
3885 return Base::updateImpl(A);
3886
3887 // TODO: add no-alias but make sure it doesn't break synchronization by
3888 // introducing fake uses. See:
3889 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3890 // International Workshop on OpenMP 2018,
3891 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3892
3893 return indicatePessimisticFixpoint();
3894 }
3895
3896 /// See AbstractAttribute::trackStatistics()
3897 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3898};
3899
3900struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3901 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3902 : AANoAliasImpl(IRP, A) {}
3903
3904 /// Determine if the underlying value may alias with the call site argument
3905 /// \p OtherArgNo of \p ICS (= the underlying call site).
3906 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3907 const AAMemoryBehavior &MemBehaviorAA,
3908 const CallBase &CB, unsigned OtherArgNo) {
3909 // We do not need to worry about aliasing with the underlying IRP.
3910 if (this->getCalleeArgNo() == (int)OtherArgNo)
3911 return false;
3912
3913 // If it is not a pointer or pointer vector we do not alias.
3914 const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3915 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3916 return false;
3917
3918 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3919 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClass: DepClassTy::NONE);
3920
3921 // If the argument is readnone, there is no read-write aliasing.
3922 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3923 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3924 return false;
3925 }
3926
3927 // If the argument is readonly and the underlying value is readonly, there
3928 // is no read-write aliasing.
3929 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3930 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3931 IsReadOnly) {
3932 A.recordDependence(FromAA: MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3933 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3934 return false;
3935 }
3936
3937 // We have to utilize actual alias analysis queries so we need the object.
3938 if (!AAR)
3939 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3940 F: *getAnchorScope());
3941
3942 // Try to rule it out at the call site.
3943 bool IsAliasing = !AAR || !AAR->isNoAlias(V1: &getAssociatedValue(), V2: ArgOp);
3944 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3945 "callsite arguments: "
3946 << getAssociatedValue() << " " << *ArgOp << " => "
3947 << (IsAliasing ? "" : "no-") << "alias \n");
3948
3949 return IsAliasing;
3950 }
3951
3952 bool isKnownNoAliasDueToNoAliasPreservation(
3953 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3954 // We can deduce "noalias" if the following conditions hold.
3955 // (i) Associated value is assumed to be noalias in the definition.
3956 // (ii) Associated value is assumed to be no-capture in all the uses
3957 // possibly executed before this callsite.
3958 // (iii) There is no other pointer argument which could alias with the
3959 // value.
3960
3961 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3962 const Function *ScopeFn = VIRP.getAnchorScope();
3963 // Check whether the value is captured in the scope using AANoCapture.
3964 // Look at CFG and check only uses possibly executed before this
3965 // callsite.
3966 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3967 Instruction *UserI = cast<Instruction>(Val: U.getUser());
3968
3969 // If UserI is the curr instruction and there is a single potential use of
3970 // the value in UserI we allow the use.
3971 // TODO: We should inspect the operands and allow those that cannot alias
3972 // with the value.
3973 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3974 return true;
3975
3976 if (ScopeFn) {
3977 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3978 if (CB->isArgOperand(U: &U)) {
3979
3980 unsigned ArgNo = CB->getArgOperandNo(U: &U);
3981
3982 bool IsKnownNoCapture;
3983 if (AA::hasAssumedIRAttr<Attribute::Captures>(
3984 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
3985 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
3986 return true;
3987 }
3988 }
3989
3990 if (!AA::isPotentiallyReachable(
3991 A, FromI: *UserI, ToI: *getCtxI(), QueryingAA: *this, /* ExclusionSet */ nullptr,
3992 GoBackwardsCB: [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3993 return true;
3994 }
3995
3996 // TODO: We should track the capturing uses in AANoCapture but the problem
3997 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3998 // a value in the module slice.
3999 // TODO(captures): Make this more precise.
4000 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
4001 if (capturesNothing(CC: CI))
4002 return true;
4003 if (CI.isPassthrough()) {
4004 Follow = true;
4005 return true;
4006 }
4007 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
4008 return false;
4009 };
4010
4011 bool IsKnownNoCapture;
4012 const AANoCapture *NoCaptureAA = nullptr;
4013 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4014 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false, AAPtr: &NoCaptureAA);
4015 if (!IsAssumedNoCapture &&
4016 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
4017 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
4018 LLVM_DEBUG(
4019 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
4020 << " cannot be noalias as it is potentially captured\n");
4021 return false;
4022 }
4023 }
4024 if (NoCaptureAA)
4025 A.recordDependence(FromAA: *NoCaptureAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4026
4027 // Check there is no other pointer argument which could alias with the
4028 // value passed at this call site.
4029 // TODO: AbstractCallSite
4030 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
4031 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
4032 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
4033 return false;
4034
4035 return true;
4036 }
4037
4038 /// See AbstractAttribute::updateImpl(...).
4039 ChangeStatus updateImpl(Attributor &A) override {
4040 // If the argument is readnone we are done as there are no accesses via the
4041 // argument.
4042 auto *MemBehaviorAA =
4043 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
4044 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
4045 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4046 return ChangeStatus::UNCHANGED;
4047 }
4048
4049 bool IsKnownNoAlias;
4050 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
4051 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4052 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
4053 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
4054 << " is not no-alias at the definition\n");
4055 return indicatePessimisticFixpoint();
4056 }
4057
4058 AAResults *AAR = nullptr;
4059 if (MemBehaviorAA &&
4060 isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
4061 LLVM_DEBUG(
4062 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
4063 return ChangeStatus::UNCHANGED;
4064 }
4065
4066 return indicatePessimisticFixpoint();
4067 }
4068
4069 /// See AbstractAttribute::trackStatistics()
4070 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4071};
4072
4073/// NoAlias attribute for function return value.
4074struct AANoAliasReturned final : AANoAliasImpl {
4075 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4076 : AANoAliasImpl(IRP, A) {}
4077
4078 /// See AbstractAttribute::updateImpl(...).
4079 ChangeStatus updateImpl(Attributor &A) override {
4080
4081 auto CheckReturnValue = [&](Value &RV) -> bool {
4082 if (Constant *C = dyn_cast<Constant>(Val: &RV))
4083 if (C->isNullValue() || isa<UndefValue>(Val: C))
4084 return true;
4085
4086 /// For now, we can only deduce noalias if we have call sites.
4087 /// FIXME: add more support.
4088 if (!isa<CallBase>(Val: &RV))
4089 return false;
4090
4091 const IRPosition &RVPos = IRPosition::value(V: RV);
4092 bool IsKnownNoAlias;
4093 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4094 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias))
4095 return false;
4096
4097 bool IsKnownNoCapture;
4098 const AANoCapture *NoCaptureAA = nullptr;
4099 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4100 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
4101 AAPtr: &NoCaptureAA);
4102 return IsAssumedNoCapture ||
4103 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4104 };
4105
4106 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA: *this))
4107 return indicatePessimisticFixpoint();
4108
4109 return ChangeStatus::UNCHANGED;
4110 }
4111
4112 /// See AbstractAttribute::trackStatistics()
4113 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4114};
4115
4116/// NoAlias attribute deduction for a call site return value.
4117struct AANoAliasCallSiteReturned final
4118 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4119 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4120 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4121
4122 /// See AbstractAttribute::trackStatistics()
4123 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4124};
4125} // namespace
4126
4127/// -------------------AAIsDead Function Attribute-----------------------
4128
4129namespace {
4130struct AAIsDeadValueImpl : public AAIsDead {
4131 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4132
4133 /// See AAIsDead::isAssumedDead().
4134 bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4135
4136 /// See AAIsDead::isKnownDead().
4137 bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4138
4139 /// See AAIsDead::isAssumedDead(BasicBlock *).
4140 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4141
4142 /// See AAIsDead::isKnownDead(BasicBlock *).
4143 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4144
4145 /// See AAIsDead::isAssumedDead(Instruction *I).
4146 bool isAssumedDead(const Instruction *I) const override {
4147 return I == getCtxI() && isAssumedDead();
4148 }
4149
4150 /// See AAIsDead::isKnownDead(Instruction *I).
4151 bool isKnownDead(const Instruction *I) const override {
4152 return isAssumedDead(I) && isKnownDead();
4153 }
4154
4155 /// See AbstractAttribute::getAsStr().
4156 const std::string getAsStr(Attributor *A) const override {
4157 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4158 }
4159
4160 /// Check if all uses are assumed dead.
4161 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4162 // Callers might not check the type, void has no uses.
4163 if (V.getType()->isVoidTy() || V.use_empty())
4164 return true;
4165
4166 // If we replace a value with a constant there are no uses left afterwards.
4167 if (!isa<Constant>(Val: V)) {
4168 if (auto *I = dyn_cast<Instruction>(Val: &V))
4169 if (!A.isRunOn(Fn&: *I->getFunction()))
4170 return false;
4171 bool UsedAssumedInformation = false;
4172 std::optional<Constant *> C =
4173 A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4174 if (!C || *C)
4175 return true;
4176 }
4177
4178 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4179 // Explicitly set the dependence class to required because we want a long
4180 // chain of N dependent instructions to be considered live as soon as one is
4181 // without going through N update cycles. This is not required for
4182 // correctness.
4183 return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4184 LivenessDepClass: DepClassTy::REQUIRED,
4185 /* IgnoreDroppableUses */ false);
4186 }
4187
4188 /// Determine if \p I is assumed to be side-effect free.
4189 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4190 if (!I || wouldInstructionBeTriviallyDead(I))
4191 return true;
4192
4193 if (!I->isTerminator() && !I->mayHaveSideEffects())
4194 return true;
4195
4196 auto *CB = dyn_cast<CallBase>(Val: I);
4197 if (!CB || isa<IntrinsicInst>(Val: CB))
4198 return false;
4199
4200 const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4201
4202 bool IsKnownNoUnwind;
4203 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4204 A, QueryingAA: this, IRP: CallIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind))
4205 return false;
4206
4207 bool IsKnown;
4208 return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4209 }
4210};
4211
4212struct AAIsDeadFloating : public AAIsDeadValueImpl {
4213 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4214 : AAIsDeadValueImpl(IRP, A) {}
4215
4216 /// See AbstractAttribute::initialize(...).
4217 void initialize(Attributor &A) override {
4218 AAIsDeadValueImpl::initialize(A);
4219
4220 if (isa<UndefValue>(Val: getAssociatedValue())) {
4221 indicatePessimisticFixpoint();
4222 return;
4223 }
4224
4225 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4226 if (!isAssumedSideEffectFree(A, I)) {
4227 if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4228 indicatePessimisticFixpoint();
4229 else
4230 removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4231 }
4232 }
4233
4234 bool isDeadFence(Attributor &A, FenceInst &FI) {
4235 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4236 IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4237 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4238 return false;
4239 A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4240 return true;
4241 }
4242
4243 bool isDeadStore(Attributor &A, StoreInst &SI,
4244 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4245 // Lang ref now states volatile store is not UB/dead, let's skip them.
4246 if (SI.isVolatile())
4247 return false;
4248
4249 // If we are collecting assumes to be deleted we are in the manifest stage.
4250 // It's problematic to collect the potential copies again now so we use the
4251 // cached ones.
4252 bool UsedAssumedInformation = false;
4253 if (!AssumeOnlyInst) {
4254 PotentialCopies.clear();
4255 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4256 UsedAssumedInformation)) {
4257 LLVM_DEBUG(
4258 dbgs()
4259 << "[AAIsDead] Could not determine potential copies of store!\n");
4260 return false;
4261 }
4262 }
4263 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4264 << " potential copies.\n");
4265
4266 InformationCache &InfoCache = A.getInfoCache();
4267 return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4268 if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4269 UsedAssumedInformation))
4270 return true;
4271 if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4272 if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4273 auto &UserI = cast<Instruction>(Val&: *U.getUser());
4274 if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4275 if (AssumeOnlyInst)
4276 AssumeOnlyInst->insert(X: &UserI);
4277 return true;
4278 }
4279 return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4280 })) {
4281 return true;
4282 }
4283 }
4284 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4285 << " is assumed live!\n");
4286 return false;
4287 });
4288 }
4289
4290 /// See AbstractAttribute::getAsStr().
4291 const std::string getAsStr(Attributor *A) const override {
4292 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4293 if (isa_and_nonnull<StoreInst>(Val: I))
4294 if (isValidState())
4295 return "assumed-dead-store";
4296 if (isa_and_nonnull<FenceInst>(Val: I))
4297 if (isValidState())
4298 return "assumed-dead-fence";
4299 return AAIsDeadValueImpl::getAsStr(A);
4300 }
4301
4302 /// See AbstractAttribute::updateImpl(...).
4303 ChangeStatus updateImpl(Attributor &A) override {
4304 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4305 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4306 if (!isDeadStore(A, SI&: *SI))
4307 return indicatePessimisticFixpoint();
4308 } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4309 if (!isDeadFence(A, FI&: *FI))
4310 return indicatePessimisticFixpoint();
4311 } else {
4312 if (!isAssumedSideEffectFree(A, I))
4313 return indicatePessimisticFixpoint();
4314 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4315 return indicatePessimisticFixpoint();
4316 }
4317 return ChangeStatus::UNCHANGED;
4318 }
4319
4320 bool isRemovableStore() const override {
4321 return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4322 }
4323
4324 /// See AbstractAttribute::manifest(...).
4325 ChangeStatus manifest(Attributor &A) override {
4326 Value &V = getAssociatedValue();
4327 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4328 // If we get here we basically know the users are all dead. We check if
4329 // isAssumedSideEffectFree returns true here again because it might not be
4330 // the case and only the users are dead but the instruction (=call) is
4331 // still needed.
4332 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4333 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4334 bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4335 (void)IsDead;
4336 assert(IsDead && "Store was assumed to be dead!");
4337 A.deleteAfterManifest(I&: *I);
4338 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4339 Instruction *AOI = AssumeOnlyInst[i];
4340 for (auto *Usr : AOI->users())
4341 AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4342 A.deleteAfterManifest(I&: *AOI);
4343 }
4344 return ChangeStatus::CHANGED;
4345 }
4346 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4347 assert(isDeadFence(A, *FI));
4348 A.deleteAfterManifest(I&: *FI);
4349 return ChangeStatus::CHANGED;
4350 }
4351 if (isAssumedSideEffectFree(A, I) && !I->isTerminator()) {
4352 A.deleteAfterManifest(I&: *I);
4353 return ChangeStatus::CHANGED;
4354 }
4355 }
4356 return ChangeStatus::UNCHANGED;
4357 }
4358
4359 /// See AbstractAttribute::trackStatistics()
4360 void trackStatistics() const override {
4361 STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4362 }
4363
4364private:
4365 // The potential copies of a dead store, used for deletion during manifest.
4366 SmallSetVector<Value *, 4> PotentialCopies;
4367};
4368
4369struct AAIsDeadArgument : public AAIsDeadFloating {
4370 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4371 : AAIsDeadFloating(IRP, A) {}
4372
4373 /// See AbstractAttribute::manifest(...).
4374 ChangeStatus manifest(Attributor &A) override {
4375 Argument &Arg = *getAssociatedArgument();
4376 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4377 if (A.registerFunctionSignatureRewrite(
4378 Arg, /* ReplacementTypes */ {},
4379 CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4380 ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4381 return ChangeStatus::CHANGED;
4382 }
4383 return ChangeStatus::UNCHANGED;
4384 }
4385
4386 /// See AbstractAttribute::trackStatistics()
4387 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4388};
4389
4390struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4391 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4392 : AAIsDeadValueImpl(IRP, A) {}
4393
4394 /// See AbstractAttribute::initialize(...).
4395 void initialize(Attributor &A) override {
4396 AAIsDeadValueImpl::initialize(A);
4397 if (isa<UndefValue>(Val: getAssociatedValue()))
4398 indicatePessimisticFixpoint();
4399 }
4400
4401 /// See AbstractAttribute::updateImpl(...).
4402 ChangeStatus updateImpl(Attributor &A) override {
4403 // TODO: Once we have call site specific value information we can provide
4404 // call site specific liveness information and then it makes
4405 // sense to specialize attributes for call sites arguments instead of
4406 // redirecting requests to the callee argument.
4407 Argument *Arg = getAssociatedArgument();
4408 if (!Arg)
4409 return indicatePessimisticFixpoint();
4410 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4411 auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4412 if (!ArgAA)
4413 return indicatePessimisticFixpoint();
4414 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4415 }
4416
4417 /// See AbstractAttribute::manifest(...).
4418 ChangeStatus manifest(Attributor &A) override {
4419 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4420 Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4421 assert(!isa<UndefValue>(U.get()) &&
4422 "Expected undef values to be filtered out!");
4423 UndefValue &UV = *UndefValue::get(T: U->getType());
4424 if (A.changeUseAfterManifest(U, NV&: UV))
4425 return ChangeStatus::CHANGED;
4426 return ChangeStatus::UNCHANGED;
4427 }
4428
4429 /// See AbstractAttribute::trackStatistics()
4430 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4431};
4432
4433struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4434 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4435 : AAIsDeadFloating(IRP, A) {}
4436
4437 /// See AAIsDead::isAssumedDead().
4438 bool isAssumedDead() const override {
4439 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4440 }
4441
4442 /// See AbstractAttribute::initialize(...).
4443 void initialize(Attributor &A) override {
4444 AAIsDeadFloating::initialize(A);
4445 if (isa<UndefValue>(Val: getAssociatedValue())) {
4446 indicatePessimisticFixpoint();
4447 return;
4448 }
4449
4450 // We track this separately as a secondary state.
4451 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4452 }
4453
4454 /// See AbstractAttribute::updateImpl(...).
4455 ChangeStatus updateImpl(Attributor &A) override {
4456 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4457 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4458 IsAssumedSideEffectFree = false;
4459 Changed = ChangeStatus::CHANGED;
4460 }
4461 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4462 return indicatePessimisticFixpoint();
4463 return Changed;
4464 }
4465
4466 /// See AbstractAttribute::trackStatistics()
4467 void trackStatistics() const override {
4468 if (IsAssumedSideEffectFree)
4469 STATS_DECLTRACK_CSRET_ATTR(IsDead)
4470 else
4471 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4472 }
4473
4474 /// See AbstractAttribute::getAsStr().
4475 const std::string getAsStr(Attributor *A) const override {
4476 return isAssumedDead()
4477 ? "assumed-dead"
4478 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4479 }
4480
4481private:
4482 bool IsAssumedSideEffectFree = true;
4483};
4484
4485struct AAIsDeadReturned : public AAIsDeadValueImpl {
4486 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4487 : AAIsDeadValueImpl(IRP, A) {}
4488
4489 /// See AbstractAttribute::updateImpl(...).
4490 ChangeStatus updateImpl(Attributor &A) override {
4491
4492 bool UsedAssumedInformation = false;
4493 A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4494 Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4495
4496 auto PredForCallSite = [&](AbstractCallSite ACS) {
4497 if (ACS.isCallbackCall() || !ACS.getInstruction())
4498 return false;
4499 return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4500 };
4501
4502 if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4503 UsedAssumedInformation))
4504 return indicatePessimisticFixpoint();
4505
4506 return ChangeStatus::UNCHANGED;
4507 }
4508
4509 /// See AbstractAttribute::manifest(...).
4510 ChangeStatus manifest(Attributor &A) override {
4511 // TODO: Rewrite the signature to return void?
4512 bool AnyChange = false;
4513 UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4514 auto RetInstPred = [&](Instruction &I) {
4515 ReturnInst &RI = cast<ReturnInst>(Val&: I);
4516 if (!isa<UndefValue>(Val: RI.getReturnValue()))
4517 AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4518 return true;
4519 };
4520 bool UsedAssumedInformation = false;
4521 A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4522 UsedAssumedInformation);
4523 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4524 }
4525
4526 /// See AbstractAttribute::trackStatistics()
4527 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4528};
4529
4530struct AAIsDeadFunction : public AAIsDead {
4531 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4532
4533 /// See AbstractAttribute::initialize(...).
4534 void initialize(Attributor &A) override {
4535 Function *F = getAnchorScope();
4536 assert(F && "Did expect an anchor function");
4537 if (!isAssumedDeadInternalFunction(A)) {
4538 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4539 assumeLive(A, BB: F->getEntryBlock());
4540 }
4541 }
4542
4543 bool isAssumedDeadInternalFunction(Attributor &A) {
4544 if (!getAnchorScope()->hasLocalLinkage())
4545 return false;
4546 bool UsedAssumedInformation = false;
4547 return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4548 RequireAllCallSites: true, UsedAssumedInformation);
4549 }
4550
4551 /// See AbstractAttribute::getAsStr().
4552 const std::string getAsStr(Attributor *A) const override {
4553 return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4554 std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4555 std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4556 std::to_string(val: KnownDeadEnds.size()) + "]";
4557 }
4558
4559 /// See AbstractAttribute::manifest(...).
4560 ChangeStatus manifest(Attributor &A) override {
4561 assert(getState().isValidState() &&
4562 "Attempted to manifest an invalid state!");
4563
4564 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4565 Function &F = *getAnchorScope();
4566
4567 if (AssumedLiveBlocks.empty()) {
4568 A.deleteAfterManifest(F);
4569 return ChangeStatus::CHANGED;
4570 }
4571
4572 // Flag to determine if we can change an invoke to a call assuming the
4573 // callee is nounwind. This is not possible if the personality of the
4574 // function allows to catch asynchronous exceptions.
4575 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4576
4577 KnownDeadEnds.set_union(ToBeExploredFrom);
4578 for (const Instruction *DeadEndI : KnownDeadEnds) {
4579 auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4580 if (!CB)
4581 continue;
4582 bool IsKnownNoReturn;
4583 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4584 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL,
4585 IsKnown&: IsKnownNoReturn);
4586 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4587 continue;
4588
4589 if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4590 A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4591 else
4592 A.changeToUnreachableAfterManifest(
4593 I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4594 HasChanged = ChangeStatus::CHANGED;
4595 }
4596
4597 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4598 for (BasicBlock &BB : F)
4599 if (!AssumedLiveBlocks.count(V: &BB)) {
4600 A.deleteAfterManifest(BB);
4601 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4602 HasChanged = ChangeStatus::CHANGED;
4603 }
4604
4605 return HasChanged;
4606 }
4607
4608 /// See AbstractAttribute::updateImpl(...).
4609 ChangeStatus updateImpl(Attributor &A) override;
4610
4611 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4612 assert(From->getParent() == getAnchorScope() &&
4613 To->getParent() == getAnchorScope() &&
4614 "Used AAIsDead of the wrong function");
4615 return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4616 }
4617
4618 /// See AbstractAttribute::trackStatistics()
4619 void trackStatistics() const override {}
4620
4621 /// Returns true if the function is assumed dead.
4622 bool isAssumedDead() const override { return false; }
4623
4624 /// See AAIsDead::isKnownDead().
4625 bool isKnownDead() const override { return false; }
4626
4627 /// See AAIsDead::isAssumedDead(BasicBlock *).
4628 bool isAssumedDead(const BasicBlock *BB) const override {
4629 assert(BB->getParent() == getAnchorScope() &&
4630 "BB must be in the same anchor scope function.");
4631
4632 if (!getAssumed())
4633 return false;
4634 return !AssumedLiveBlocks.count(V: BB);
4635 }
4636
4637 /// See AAIsDead::isKnownDead(BasicBlock *).
4638 bool isKnownDead(const BasicBlock *BB) const override {
4639 return getKnown() && isAssumedDead(BB);
4640 }
4641
4642 /// See AAIsDead::isAssumed(Instruction *I).
4643 bool isAssumedDead(const Instruction *I) const override {
4644 assert(I->getParent()->getParent() == getAnchorScope() &&
4645 "Instruction must be in the same anchor scope function.");
4646
4647 if (!getAssumed())
4648 return false;
4649
4650 // If it is not in AssumedLiveBlocks then it for sure dead.
4651 // Otherwise, it can still be after noreturn call in a live block.
4652 if (!AssumedLiveBlocks.count(V: I->getParent()))
4653 return true;
4654
4655 // If it is not after a liveness barrier it is live.
4656 const Instruction *PrevI = I->getPrevNode();
4657 while (PrevI) {
4658 if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4659 return true;
4660 PrevI = PrevI->getPrevNode();
4661 }
4662 return false;
4663 }
4664
4665 /// See AAIsDead::isKnownDead(Instruction *I).
4666 bool isKnownDead(const Instruction *I) const override {
4667 return getKnown() && isAssumedDead(I);
4668 }
4669
4670 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4671 /// that internal function called from \p BB should now be looked at.
4672 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4673 if (!AssumedLiveBlocks.insert(V: &BB).second)
4674 return false;
4675
4676 // We assume that all of BB is (probably) live now and if there are calls to
4677 // internal functions we will assume that those are now live as well. This
4678 // is a performance optimization for blocks with calls to a lot of internal
4679 // functions. It can however cause dead functions to be treated as live.
4680 for (const Instruction &I : BB)
4681 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4682 if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4683 if (F->hasLocalLinkage())
4684 A.markLiveInternalFunction(F: *F);
4685 return true;
4686 }
4687
4688 /// Collection of instructions that need to be explored again, e.g., we
4689 /// did assume they do not transfer control to (one of their) successors.
4690 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4691
4692 /// Collection of instructions that are known to not transfer control.
4693 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4694
4695 /// Collection of all assumed live edges
4696 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4697
4698 /// Collection of all assumed live BasicBlocks.
4699 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4700};
4701
4702static bool
4703identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4704 AbstractAttribute &AA,
4705 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4706 const IRPosition &IPos = IRPosition::callsite_function(CB);
4707
4708 bool IsKnownNoReturn;
4709 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4710 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoReturn))
4711 return !IsKnownNoReturn;
4712 if (CB.isTerminator())
4713 AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4714 else
4715 AliveSuccessors.push_back(Elt: CB.getNextNode());
4716 return false;
4717}
4718
4719static bool
4720identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4721 AbstractAttribute &AA,
4722 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4723 bool UsedAssumedInformation =
4724 identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4725
4726 // First, determine if we can change an invoke to a call assuming the
4727 // callee is nounwind. This is not possible if the personality of the
4728 // function allows to catch asynchronous exceptions.
4729 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4730 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4731 } else {
4732 const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4733
4734 bool IsKnownNoUnwind;
4735 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4736 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
4737 UsedAssumedInformation |= !IsKnownNoUnwind;
4738 } else {
4739 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4740 }
4741 }
4742 return UsedAssumedInformation;
4743}
4744
4745static bool
4746identifyAliveSuccessors(Attributor &, const UncondBrInst &BI,
4747 AbstractAttribute &,
4748 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4749 AliveSuccessors.push_back(Elt: &BI.getSuccessor()->front());
4750 return false;
4751}
4752
4753static bool
4754identifyAliveSuccessors(Attributor &A, const CondBrInst &BI,
4755 AbstractAttribute &AA,
4756 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4757 bool UsedAssumedInformation = false;
4758 std::optional<Constant *> C =
4759 A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4760 if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4761 // No value yet, assume both edges are dead.
4762 } else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4763 const BasicBlock *SuccBB =
4764 BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4765 AliveSuccessors.push_back(Elt: &SuccBB->front());
4766 } else {
4767 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4768 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4769 UsedAssumedInformation = false;
4770 }
4771 return UsedAssumedInformation;
4772}
4773
4774static bool
4775identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4776 AbstractAttribute &AA,
4777 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4778 bool UsedAssumedInformation = false;
4779 SmallVector<AA::ValueAndContext> Values;
4780 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4781 Values, S: AA::AnyScope,
4782 UsedAssumedInformation)) {
4783 // Something went wrong, assume all successors are live.
4784 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4785 AliveSuccessors.push_back(Elt: &SuccBB->front());
4786 return false;
4787 }
4788
4789 if (Values.empty() ||
4790 (Values.size() == 1 &&
4791 isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4792 // No valid value yet, assume all edges are dead.
4793 return UsedAssumedInformation;
4794 }
4795
4796 Type &Ty = *SI.getCondition()->getType();
4797 SmallPtrSet<ConstantInt *, 8> Constants;
4798 auto CheckForConstantInt = [&](Value *V) {
4799 if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4800 Constants.insert(Ptr: CI);
4801 return true;
4802 }
4803 return false;
4804 };
4805
4806 if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4807 return CheckForConstantInt(VAC.getValue());
4808 })) {
4809 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4810 AliveSuccessors.push_back(Elt: &SuccBB->front());
4811 return UsedAssumedInformation;
4812 }
4813
4814 unsigned MatchedCases = 0;
4815 for (const auto &CaseIt : SI.cases()) {
4816 if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4817 ++MatchedCases;
4818 AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4819 }
4820 }
4821
4822 // If all potential values have been matched, we will not visit the default
4823 // case.
4824 if (MatchedCases < Constants.size())
4825 AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4826 return UsedAssumedInformation;
4827}
4828
4829ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4830 ChangeStatus Change = ChangeStatus::UNCHANGED;
4831
4832 if (AssumedLiveBlocks.empty()) {
4833 if (isAssumedDeadInternalFunction(A))
4834 return ChangeStatus::UNCHANGED;
4835
4836 Function *F = getAnchorScope();
4837 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4838 assumeLive(A, BB: F->getEntryBlock());
4839 Change = ChangeStatus::CHANGED;
4840 }
4841
4842 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4843 << getAnchorScope()->size() << "] BBs and "
4844 << ToBeExploredFrom.size() << " exploration points and "
4845 << KnownDeadEnds.size() << " known dead ends\n");
4846
4847 // Copy and clear the list of instructions we need to explore from. It is
4848 // refilled with instructions the next update has to look at.
4849 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4850 ToBeExploredFrom.end());
4851 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4852
4853 SmallVector<const Instruction *, 8> AliveSuccessors;
4854 while (!Worklist.empty()) {
4855 const Instruction *I = Worklist.pop_back_val();
4856 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4857
4858 // Fast forward for uninteresting instructions. We could look for UB here
4859 // though.
4860 while (!I->isTerminator() && !isa<CallBase>(Val: I))
4861 I = I->getNextNode();
4862
4863 AliveSuccessors.clear();
4864
4865 bool UsedAssumedInformation = false;
4866 switch (I->getOpcode()) {
4867 // TODO: look for (assumed) UB to backwards propagate "deadness".
4868 default:
4869 assert(I->isTerminator() &&
4870 "Expected non-terminators to be handled already!");
4871 for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4872 AliveSuccessors.push_back(Elt: &SuccBB->front());
4873 break;
4874 case Instruction::Call:
4875 UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4876 AA&: *this, AliveSuccessors);
4877 break;
4878 case Instruction::Invoke:
4879 UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4880 AA&: *this, AliveSuccessors);
4881 break;
4882 case Instruction::UncondBr:
4883 UsedAssumedInformation = identifyAliveSuccessors(
4884 A, BI: cast<UncondBrInst>(Val: *I), *this, AliveSuccessors);
4885 break;
4886 case Instruction::CondBr:
4887 UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<CondBrInst>(Val: *I),
4888 AA&: *this, AliveSuccessors);
4889 break;
4890 case Instruction::Switch:
4891 UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4892 AA&: *this, AliveSuccessors);
4893 break;
4894 }
4895
4896 if (UsedAssumedInformation) {
4897 NewToBeExploredFrom.insert(X: I);
4898 } else if (AliveSuccessors.empty() ||
4899 (I->isTerminator() &&
4900 AliveSuccessors.size() < I->getNumSuccessors())) {
4901 if (KnownDeadEnds.insert(X: I))
4902 Change = ChangeStatus::CHANGED;
4903 }
4904
4905 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4906 << AliveSuccessors.size() << " UsedAssumedInformation: "
4907 << UsedAssumedInformation << "\n");
4908
4909 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4910 if (!I->isTerminator()) {
4911 assert(AliveSuccessors.size() == 1 &&
4912 "Non-terminator expected to have a single successor!");
4913 Worklist.push_back(Elt: AliveSuccessor);
4914 } else {
4915 // record the assumed live edge
4916 auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4917 if (AssumedLiveEdges.insert(V: Edge).second)
4918 Change = ChangeStatus::CHANGED;
4919 if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4920 Worklist.push_back(Elt: AliveSuccessor);
4921 }
4922 }
4923 }
4924
4925 // Check if the content of ToBeExploredFrom changed, ignore the order.
4926 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4927 llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4928 return !ToBeExploredFrom.count(key: I);
4929 })) {
4930 Change = ChangeStatus::CHANGED;
4931 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4932 }
4933
4934 // If we know everything is live there is no need to query for liveness.
4935 // Instead, indicating a pessimistic fixpoint will cause the state to be
4936 // "invalid" and all queries to be answered conservatively without lookups.
4937 // To be in this state we have to (1) finished the exploration and (3) not
4938 // discovered any non-trivial dead end and (2) not ruled unreachable code
4939 // dead.
4940 if (ToBeExploredFrom.empty() &&
4941 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4942 llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4943 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4944 }))
4945 return indicatePessimisticFixpoint();
4946 return Change;
4947}
4948
4949/// Liveness information for a call sites.
4950struct AAIsDeadCallSite final : AAIsDeadFunction {
4951 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4952 : AAIsDeadFunction(IRP, A) {}
4953
4954 /// See AbstractAttribute::initialize(...).
4955 void initialize(Attributor &A) override {
4956 // TODO: Once we have call site specific value information we can provide
4957 // call site specific liveness information and then it makes
4958 // sense to specialize attributes for call sites instead of
4959 // redirecting requests to the callee.
4960 llvm_unreachable("Abstract attributes for liveness are not "
4961 "supported for call sites yet!");
4962 }
4963
4964 /// See AbstractAttribute::updateImpl(...).
4965 ChangeStatus updateImpl(Attributor &A) override {
4966 return indicatePessimisticFixpoint();
4967 }
4968
4969 /// See AbstractAttribute::trackStatistics()
4970 void trackStatistics() const override {}
4971};
4972} // namespace
4973
4974/// -------------------- Dereferenceable Argument Attribute --------------------
4975
4976namespace {
4977struct AADereferenceableImpl : AADereferenceable {
4978 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4979 : AADereferenceable(IRP, A) {}
4980 using StateType = DerefState;
4981
4982 /// See AbstractAttribute::initialize(...).
4983 void initialize(Attributor &A) override {
4984 Value &V = *getAssociatedValue().stripPointerCasts();
4985 SmallVector<Attribute, 4> Attrs;
4986 A.getAttrs(IRP: getIRPosition(),
4987 AKs: {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4988 Attrs, /* IgnoreSubsumingPositions */ false);
4989 for (const Attribute &Attr : Attrs)
4990 takeKnownDerefBytesMaximum(Bytes: Attr.getValueAsInt());
4991
4992 // Ensure we initialize the non-null AA (if necessary).
4993 bool IsKnownNonNull;
4994 AA::hasAssumedIRAttr<Attribute::NonNull>(
4995 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
4996
4997 bool CanBeNull;
4998 takeKnownDerefBytesMaximum(Bytes: V.getPointerDereferenceableBytes(
4999 DL: A.getDataLayout(), CanBeNull, /*CanBeFreed=*/nullptr));
5000
5001 if (Instruction *CtxI = getCtxI())
5002 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5003 }
5004
5005 /// See AbstractAttribute::getState()
5006 /// {
5007 StateType &getState() override { return *this; }
5008 const StateType &getState() const override { return *this; }
5009 /// }
5010
5011 /// Helper function for collecting accessed bytes in must-be-executed-context
5012 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
5013 DerefState &State) {
5014 const Value *UseV = U->get();
5015 if (!UseV->getType()->isPointerTy())
5016 return;
5017
5018 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
5019 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
5020 return;
5021
5022 int64_t Offset;
5023 const Value *Base = GetPointerBaseWithConstantOffset(
5024 Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
5025 if (Base && Base == &getAssociatedValue())
5026 State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
5027 }
5028
5029 /// See followUsesInMBEC
5030 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5031 AADereferenceable::StateType &State) {
5032 bool IsNonNull = false;
5033 bool TrackUse = false;
5034 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
5035 A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I, IsNonNull, TrackUse);
5036 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
5037 << " for instruction " << *I << "\n");
5038
5039 addAccessedBytesForUse(A, U, I, State);
5040 State.takeKnownDerefBytesMaximum(Bytes: DerefBytes);
5041 return TrackUse;
5042 }
5043
5044 /// See AbstractAttribute::manifest(...).
5045 ChangeStatus manifest(Attributor &A) override {
5046 ChangeStatus Change = AADereferenceable::manifest(A);
5047 bool IsKnownNonNull;
5048 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5049 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5050 if (IsAssumedNonNull &&
5051 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::DereferenceableOrNull)) {
5052 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::DereferenceableOrNull});
5053 return ChangeStatus::CHANGED;
5054 }
5055 return Change;
5056 }
5057
5058 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5059 SmallVectorImpl<Attribute> &Attrs) const override {
5060 // TODO: Add *_globally support
5061 bool IsKnownNonNull;
5062 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5063 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5064 if (IsAssumedNonNull)
5065 Attrs.emplace_back(Args: Attribute::getWithDereferenceableBytes(
5066 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5067 else
5068 Attrs.emplace_back(Args: Attribute::getWithDereferenceableOrNullBytes(
5069 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5070 }
5071
5072 /// See AbstractAttribute::getAsStr().
5073 const std::string getAsStr(Attributor *A) const override {
5074 if (!getAssumedDereferenceableBytes())
5075 return "unknown-dereferenceable";
5076 bool IsKnownNonNull;
5077 bool IsAssumedNonNull = false;
5078 if (A)
5079 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5080 A&: *A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5081 return std::string("dereferenceable") +
5082 (IsAssumedNonNull ? "" : "_or_null") +
5083 (isAssumedGlobal() ? "_globally" : "") + "<" +
5084 std::to_string(val: getKnownDereferenceableBytes()) + "-" +
5085 std::to_string(val: getAssumedDereferenceableBytes()) + ">" +
5086 (!A ? " [non-null is unknown]" : "");
5087 }
5088};
5089
5090/// Dereferenceable attribute for a floating value.
5091struct AADereferenceableFloating : AADereferenceableImpl {
5092 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5093 : AADereferenceableImpl(IRP, A) {}
5094
5095 /// See AbstractAttribute::updateImpl(...).
5096 ChangeStatus updateImpl(Attributor &A) override {
5097 bool Stripped;
5098 bool UsedAssumedInformation = false;
5099 SmallVector<AA::ValueAndContext> Values;
5100 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5101 S: AA::AnyScope, UsedAssumedInformation)) {
5102 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5103 Stripped = false;
5104 } else {
5105 Stripped = Values.size() != 1 ||
5106 Values.front().getValue() != &getAssociatedValue();
5107 }
5108
5109 const DataLayout &DL = A.getDataLayout();
5110 DerefState T;
5111
5112 auto VisitValueCB = [&](const Value &V) -> bool {
5113 unsigned IdxWidth =
5114 DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
5115 APInt Offset(IdxWidth, 0);
5116 const Value *Base = stripAndAccumulateOffsets(
5117 A, QueryingAA: *this, Val: &V, DL, Offset, /* GetMinOffset */ false,
5118 /* AllowNonInbounds */ true);
5119
5120 const auto *AA = A.getAAFor<AADereferenceable>(
5121 QueryingAA: *this, IRP: IRPosition::value(V: *Base), DepClass: DepClassTy::REQUIRED);
5122 int64_t DerefBytes = 0;
5123 if (!AA || (!Stripped && this == AA)) {
5124 // Use IR information if we did not strip anything.
5125 // TODO: track globally.
5126 bool CanBeNull;
5127 DerefBytes = Base->getPointerDereferenceableBytes(
5128 DL, CanBeNull, /*CanBeFreed=*/nullptr);
5129 T.GlobalState.indicatePessimisticFixpoint();
5130 } else {
5131 const DerefState &DS = AA->getState();
5132 DerefBytes = DS.DerefBytesState.getAssumed();
5133 T.GlobalState &= DS.GlobalState;
5134 }
5135
5136 // For now we do not try to "increase" dereferenceability due to negative
5137 // indices as we first have to come up with code to deal with loops and
5138 // for overflows of the dereferenceable bytes.
5139 int64_t OffsetSExt = Offset.getSExtValue();
5140 if (OffsetSExt < 0)
5141 OffsetSExt = 0;
5142
5143 T.takeAssumedDerefBytesMinimum(
5144 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5145
5146 if (this == AA) {
5147 if (!Stripped) {
5148 // If nothing was stripped IR information is all we got.
5149 T.takeKnownDerefBytesMaximum(
5150 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5151 T.indicatePessimisticFixpoint();
5152 } else if (OffsetSExt > 0) {
5153 // If something was stripped but there is circular reasoning we look
5154 // for the offset. If it is positive we basically decrease the
5155 // dereferenceable bytes in a circular loop now, which will simply
5156 // drive them down to the known value in a very slow way which we
5157 // can accelerate.
5158 T.indicatePessimisticFixpoint();
5159 }
5160 }
5161
5162 return T.isValidState();
5163 };
5164
5165 for (const auto &VAC : Values)
5166 if (!VisitValueCB(*VAC.getValue()))
5167 return indicatePessimisticFixpoint();
5168
5169 return clampStateAndIndicateChange(S&: getState(), R: T);
5170 }
5171
5172 /// See AbstractAttribute::trackStatistics()
5173 void trackStatistics() const override {
5174 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5175 }
5176};
5177
5178/// Dereferenceable attribute for a return value.
5179struct AADereferenceableReturned final
5180 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5181 using Base =
5182 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5183 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5184 : Base(IRP, A) {}
5185
5186 /// See AbstractAttribute::trackStatistics()
5187 void trackStatistics() const override {
5188 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5189 }
5190};
5191
5192/// Dereferenceable attribute for an argument
5193struct AADereferenceableArgument final
5194 : AAArgumentFromCallSiteArguments<AADereferenceable,
5195 AADereferenceableImpl> {
5196 using Base =
5197 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5198 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5199 : Base(IRP, A) {}
5200
5201 /// See AbstractAttribute::trackStatistics()
5202 void trackStatistics() const override {
5203 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5204 }
5205};
5206
5207/// Dereferenceable attribute for a call site argument.
5208struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5209 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5210 : AADereferenceableFloating(IRP, A) {}
5211
5212 /// See AbstractAttribute::trackStatistics()
5213 void trackStatistics() const override {
5214 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5215 }
5216};
5217
5218/// Dereferenceable attribute deduction for a call site return value.
5219struct AADereferenceableCallSiteReturned final
5220 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5221 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5222 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5223 : Base(IRP, A) {}
5224
5225 /// See AbstractAttribute::trackStatistics()
5226 void trackStatistics() const override {
5227 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5228 }
5229};
5230} // namespace
5231
5232// ------------------------ Align Argument Attribute ------------------------
5233
5234namespace {
5235
5236static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5237 Value &AssociatedValue, const Use *U,
5238 const Instruction *I, bool &TrackUse) {
5239 // We need to follow common pointer manipulation uses to the accesses they
5240 // feed into.
5241 if (isa<CastInst>(Val: I)) {
5242 // Follow all but ptr2int casts.
5243 TrackUse = !isa<PtrToIntInst>(Val: I);
5244 return 0;
5245 }
5246 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5247 if (GEP->hasAllConstantIndices())
5248 TrackUse = true;
5249 return 0;
5250 }
5251 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I))
5252 switch (II->getIntrinsicID()) {
5253 case Intrinsic::ptrmask: {
5254 // Is it appropriate to pull attribute in initialization?
5255 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5256 QueryingAA, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::NONE);
5257 const auto *AlignAA = A.getAAFor<AAAlign>(
5258 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5259 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5260 unsigned ShiftValue = std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5261 b: Value::MaxAlignmentExponent);
5262 Align ConstAlign(UINT64_C(1) << ShiftValue);
5263 if (ConstAlign >= AlignAA->getKnownAlign())
5264 return Align(1).value();
5265 }
5266 if (AlignAA)
5267 return AlignAA->getKnownAlign().value();
5268 break;
5269 }
5270 case Intrinsic::amdgcn_make_buffer_rsrc: {
5271 const auto *AlignAA = A.getAAFor<AAAlign>(
5272 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5273 if (AlignAA)
5274 return AlignAA->getKnownAlign().value();
5275 break;
5276 }
5277 default:
5278 break;
5279 }
5280
5281 MaybeAlign MA;
5282 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5283 if (CB->isBundleOperand(U) || CB->isCallee(U))
5284 return 0;
5285
5286 unsigned ArgNo = CB->getArgOperandNo(U);
5287 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5288 // As long as we only use known information there is no need to track
5289 // dependences here.
5290 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
5291 if (AlignAA)
5292 MA = MaybeAlign(AlignAA->getKnownAlign());
5293 }
5294
5295 const DataLayout &DL = A.getDataLayout();
5296 const Value *UseV = U->get();
5297 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5298 if (SI->getPointerOperand() == UseV)
5299 MA = SI->getAlign();
5300 } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5301 if (LI->getPointerOperand() == UseV)
5302 MA = LI->getAlign();
5303 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5304 if (AI->getPointerOperand() == UseV)
5305 MA = AI->getAlign();
5306 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5307 if (AI->getPointerOperand() == UseV)
5308 MA = AI->getAlign();
5309 }
5310
5311 if (!MA || *MA <= QueryingAA.getKnownAlign())
5312 return 0;
5313
5314 unsigned Alignment = MA->value();
5315 int64_t Offset;
5316
5317 if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5318 if (Base == &AssociatedValue) {
5319 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5320 // So we can say that the maximum power of two which is a divisor of
5321 // gcd(Offset, Alignment) is an alignment.
5322
5323 uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5324 Alignment = llvm::bit_floor(Value: gcd);
5325 }
5326 }
5327
5328 return Alignment;
5329}
5330
5331struct AAAlignImpl : AAAlign {
5332 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5333
5334 /// See AbstractAttribute::initialize(...).
5335 void initialize(Attributor &A) override {
5336 SmallVector<Attribute, 4> Attrs;
5337 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::Alignment}, Attrs);
5338 for (const Attribute &Attr : Attrs)
5339 takeKnownMaximum(Value: Attr.getValueAsInt());
5340
5341 Value &V = *getAssociatedValue().stripPointerCasts();
5342 takeKnownMaximum(Value: V.getPointerAlignment(DL: A.getDataLayout()).value());
5343
5344 if (Instruction *CtxI = getCtxI())
5345 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5346 }
5347
5348 /// See AbstractAttribute::manifest(...).
5349 ChangeStatus manifest(Attributor &A) override {
5350 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5351
5352 // Check for users that allow alignment annotations.
5353 Value &AssociatedValue = getAssociatedValue();
5354 if (isa<ConstantData>(Val: AssociatedValue))
5355 return ChangeStatus::UNCHANGED;
5356
5357 for (const Use &U : AssociatedValue.uses()) {
5358 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
5359 if (SI->getPointerOperand() == &AssociatedValue)
5360 if (SI->getAlign() < getAssumedAlign()) {
5361 STATS_DECLTRACK(AAAlign, Store,
5362 "Number of times alignment added to a store");
5363 SI->setAlignment(getAssumedAlign());
5364 InstrChanged = ChangeStatus::CHANGED;
5365 }
5366 } else if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
5367 if (LI->getPointerOperand() == &AssociatedValue)
5368 if (LI->getAlign() < getAssumedAlign()) {
5369 LI->setAlignment(getAssumedAlign());
5370 STATS_DECLTRACK(AAAlign, Load,
5371 "Number of times alignment added to a load");
5372 InstrChanged = ChangeStatus::CHANGED;
5373 }
5374 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: U.getUser())) {
5375 if (RMW->getPointerOperand() == &AssociatedValue) {
5376 if (RMW->getAlign() < getAssumedAlign()) {
5377 STATS_DECLTRACK(AAAlign, AtomicRMW,
5378 "Number of times alignment added to atomicrmw");
5379
5380 RMW->setAlignment(getAssumedAlign());
5381 InstrChanged = ChangeStatus::CHANGED;
5382 }
5383 }
5384 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: U.getUser())) {
5385 if (CAS->getPointerOperand() == &AssociatedValue) {
5386 if (CAS->getAlign() < getAssumedAlign()) {
5387 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5388 "Number of times alignment added to cmpxchg");
5389 CAS->setAlignment(getAssumedAlign());
5390 InstrChanged = ChangeStatus::CHANGED;
5391 }
5392 }
5393 }
5394 }
5395
5396 ChangeStatus Changed = AAAlign::manifest(A);
5397
5398 Align InheritAlign =
5399 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5400 if (InheritAlign >= getAssumedAlign())
5401 return InstrChanged;
5402 return Changed | InstrChanged;
5403 }
5404
5405 // TODO: Provide a helper to determine the implied ABI alignment and check in
5406 // the existing manifest method and a new one for AAAlignImpl that value
5407 // to avoid making the alignment explicit if it did not improve.
5408
5409 /// See AbstractAttribute::getDeducedAttributes
5410 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5411 SmallVectorImpl<Attribute> &Attrs) const override {
5412 if (getAssumedAlign() > 1)
5413 Attrs.emplace_back(
5414 Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5415 }
5416
5417 /// See followUsesInMBEC
5418 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5419 AAAlign::StateType &State) {
5420 bool TrackUse = false;
5421
5422 unsigned int KnownAlign =
5423 getKnownAlignForUse(A, QueryingAA&: *this, AssociatedValue&: getAssociatedValue(), U, I, TrackUse);
5424 State.takeKnownMaximum(Value: KnownAlign);
5425
5426 return TrackUse;
5427 }
5428
5429 /// See AbstractAttribute::getAsStr().
5430 const std::string getAsStr(Attributor *A) const override {
5431 return "align<" + std::to_string(val: getKnownAlign().value()) + "-" +
5432 std::to_string(val: getAssumedAlign().value()) + ">";
5433 }
5434};
5435
5436/// Align attribute for a floating value.
5437struct AAAlignFloating : AAAlignImpl {
5438 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5439
5440 /// See AbstractAttribute::updateImpl(...).
5441 ChangeStatus updateImpl(Attributor &A) override {
5442 const DataLayout &DL = A.getDataLayout();
5443
5444 bool Stripped;
5445 bool UsedAssumedInformation = false;
5446 SmallVector<AA::ValueAndContext> Values;
5447 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5448 S: AA::AnyScope, UsedAssumedInformation)) {
5449 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5450 Stripped = false;
5451 } else {
5452 Stripped = Values.size() != 1 ||
5453 Values.front().getValue() != &getAssociatedValue();
5454 }
5455
5456 StateType T;
5457 auto VisitValueCB = [&](Value &V) -> bool {
5458 if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5459 return true;
5460 const auto *AA = A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V),
5461 DepClass: DepClassTy::REQUIRED);
5462 if (!AA || (!Stripped && this == AA)) {
5463 int64_t Offset;
5464 unsigned Alignment = 1;
5465 if (const Value *Base =
5466 GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5467 // TODO: Use AAAlign for the base too.
5468 Align PA = Base->getPointerAlignment(DL);
5469 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5470 // So we can say that the maximum power of two which is a divisor of
5471 // gcd(Offset, Alignment) is an alignment.
5472
5473 uint32_t gcd =
5474 std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5475 Alignment = llvm::bit_floor(Value: gcd);
5476 } else {
5477 Alignment = V.getPointerAlignment(DL).value();
5478 }
5479 // Use only IR information if we did not strip anything.
5480 T.takeKnownMaximum(Value: Alignment);
5481 T.indicatePessimisticFixpoint();
5482 } else {
5483 // Use abstract attribute information.
5484 const AAAlign::StateType &DS = AA->getState();
5485 T ^= DS;
5486 }
5487 return T.isValidState();
5488 };
5489
5490 for (const auto &VAC : Values) {
5491 if (!VisitValueCB(*VAC.getValue()))
5492 return indicatePessimisticFixpoint();
5493 }
5494
5495 // TODO: If we know we visited all incoming values, thus no are assumed
5496 // dead, we can take the known information from the state T.
5497 return clampStateAndIndicateChange(S&: getState(), R: T);
5498 }
5499
5500 /// See AbstractAttribute::trackStatistics()
5501 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5502};
5503
5504/// Align attribute for function return value.
5505struct AAAlignReturned final
5506 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5507 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5508 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5509
5510 /// See AbstractAttribute::trackStatistics()
5511 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5512};
5513
5514/// Align attribute for function argument.
5515struct AAAlignArgument final
5516 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5517 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5518 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5519
5520 /// See AbstractAttribute::manifest(...).
5521 ChangeStatus manifest(Attributor &A) override {
5522 // If the associated argument is involved in a must-tail call we give up
5523 // because we would need to keep the argument alignments of caller and
5524 // callee in-sync. Just does not seem worth the trouble right now.
5525 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5526 return ChangeStatus::UNCHANGED;
5527 return Base::manifest(A);
5528 }
5529
5530 /// See AbstractAttribute::trackStatistics()
5531 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5532};
5533
5534struct AAAlignCallSiteArgument final : AAAlignFloating {
5535 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5536 : AAAlignFloating(IRP, A) {}
5537
5538 /// See AbstractAttribute::manifest(...).
5539 ChangeStatus manifest(Attributor &A) override {
5540 // If the associated argument is involved in a must-tail call we give up
5541 // because we would need to keep the argument alignments of caller and
5542 // callee in-sync. Just does not seem worth the trouble right now.
5543 if (Argument *Arg = getAssociatedArgument())
5544 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5545 return ChangeStatus::UNCHANGED;
5546 ChangeStatus Changed = AAAlignImpl::manifest(A);
5547 Align InheritAlign =
5548 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5549 if (InheritAlign >= getAssumedAlign())
5550 Changed = ChangeStatus::UNCHANGED;
5551 return Changed;
5552 }
5553
5554 /// See AbstractAttribute::updateImpl(Attributor &A).
5555 ChangeStatus updateImpl(Attributor &A) override {
5556 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5557 if (Argument *Arg = getAssociatedArgument()) {
5558 // We only take known information from the argument
5559 // so we do not need to track a dependence.
5560 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5561 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::NONE);
5562 if (ArgAlignAA)
5563 takeKnownMaximum(Value: ArgAlignAA->getKnownAlign().value());
5564 }
5565 return Changed;
5566 }
5567
5568 /// See AbstractAttribute::trackStatistics()
5569 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5570};
5571
5572/// Align attribute deduction for a call site return value.
5573struct AAAlignCallSiteReturned final
5574 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5575 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5576 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5577 : Base(IRP, A) {}
5578
5579 ChangeStatus updateImpl(Attributor &A) override {
5580 Instruction *I = getIRPosition().getCtxI();
5581 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
5582 switch (II->getIntrinsicID()) {
5583 case Intrinsic::ptrmask: {
5584 Align Alignment;
5585 bool Valid = false;
5586
5587 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5588 QueryingAA: *this, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::REQUIRED);
5589 if (ConstVals && ConstVals->isValidState()) {
5590 unsigned ShiftValue =
5591 std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5592 b: Value::MaxAlignmentExponent);
5593 Alignment = Align(UINT64_C(1) << ShiftValue);
5594 Valid = true;
5595 }
5596
5597 const auto *AlignAA =
5598 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5599 DepClass: DepClassTy::REQUIRED);
5600 if (AlignAA) {
5601 Alignment = std::max(a: AlignAA->getAssumedAlign(), b: Alignment);
5602 Valid = true;
5603 }
5604
5605 if (Valid)
5606 return clampStateAndIndicateChange<StateType>(
5607 S&: this->getState(),
5608 R: std::min(a: this->getAssumedAlign(), b: Alignment).value());
5609 break;
5610 }
5611 // FIXME: Should introduce target specific sub-attributes and letting
5612 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5613 // specific intrinsics.
5614 case Intrinsic::amdgcn_make_buffer_rsrc: {
5615 const auto *AlignAA =
5616 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5617 DepClass: DepClassTy::REQUIRED);
5618 if (AlignAA)
5619 return clampStateAndIndicateChange<StateType>(
5620 S&: this->getState(), R: AlignAA->getAssumedAlign().value());
5621 break;
5622 }
5623 default:
5624 break;
5625 }
5626 }
5627 return Base::updateImpl(A);
5628 };
5629 /// See AbstractAttribute::trackStatistics()
5630 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5631};
5632} // namespace
5633
5634/// ------------------ Function No-Return Attribute ----------------------------
5635namespace {
5636struct AANoReturnImpl : public AANoReturn {
5637 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5638
5639 /// See AbstractAttribute::initialize(...).
5640 void initialize(Attributor &A) override {
5641 bool IsKnown;
5642 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5643 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5644 (void)IsKnown;
5645 }
5646
5647 /// See AbstractAttribute::getAsStr().
5648 const std::string getAsStr(Attributor *A) const override {
5649 return getAssumed() ? "noreturn" : "may-return";
5650 }
5651
5652 /// See AbstractAttribute::updateImpl(Attributor &A).
5653 ChangeStatus updateImpl(Attributor &A) override {
5654 auto CheckForNoReturn = [](Instruction &) { return false; };
5655 bool UsedAssumedInformation = false;
5656 if (!A.checkForAllInstructions(Pred: CheckForNoReturn, QueryingAA: *this,
5657 Opcodes: {(unsigned)Instruction::Ret},
5658 UsedAssumedInformation))
5659 return indicatePessimisticFixpoint();
5660 return ChangeStatus::UNCHANGED;
5661 }
5662};
5663
5664struct AANoReturnFunction final : AANoReturnImpl {
5665 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5666 : AANoReturnImpl(IRP, A) {}
5667
5668 /// See AbstractAttribute::trackStatistics()
5669 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5670};
5671
5672/// NoReturn attribute deduction for a call sites.
5673struct AANoReturnCallSite final
5674 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5675 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5676 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5677
5678 /// See AbstractAttribute::trackStatistics()
5679 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5680};
5681} // namespace
5682
5683/// ----------------------- Instance Info ---------------------------------
5684
5685namespace {
5686/// A class to hold the state of for no-capture attributes.
5687struct AAInstanceInfoImpl : public AAInstanceInfo {
5688 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5689 : AAInstanceInfo(IRP, A) {}
5690
5691 /// See AbstractAttribute::initialize(...).
5692 void initialize(Attributor &A) override {
5693 Value &V = getAssociatedValue();
5694 if (auto *C = dyn_cast<Constant>(Val: &V)) {
5695 if (C->isThreadDependent())
5696 indicatePessimisticFixpoint();
5697 else
5698 indicateOptimisticFixpoint();
5699 return;
5700 }
5701 if (auto *CB = dyn_cast<CallBase>(Val: &V))
5702 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5703 !CB->mayReadFromMemory()) {
5704 indicateOptimisticFixpoint();
5705 return;
5706 }
5707 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5708 const auto *CI =
5709 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5710 F: *I->getFunction());
5711 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5712 indicatePessimisticFixpoint();
5713 return;
5714 }
5715 }
5716 }
5717
5718 /// See AbstractAttribute::updateImpl(...).
5719 ChangeStatus updateImpl(Attributor &A) override {
5720 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5721
5722 Value &V = getAssociatedValue();
5723 const Function *Scope = nullptr;
5724 if (auto *I = dyn_cast<Instruction>(Val: &V))
5725 Scope = I->getFunction();
5726 if (auto *A = dyn_cast<Argument>(Val: &V)) {
5727 Scope = A->getParent();
5728 if (!Scope->hasLocalLinkage())
5729 return Changed;
5730 }
5731 if (!Scope)
5732 return indicateOptimisticFixpoint();
5733
5734 bool IsKnownNoRecurse;
5735 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5736 A, QueryingAA: this, IRP: IRPosition::function(F: *Scope), DepClass: DepClassTy::OPTIONAL,
5737 IsKnown&: IsKnownNoRecurse))
5738 return Changed;
5739
5740 auto UsePred = [&](const Use &U, bool &Follow) {
5741 const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5742 if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5743 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5744 Follow = true;
5745 return true;
5746 }
5747 if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5748 (isa<StoreInst>(Val: UserI) &&
5749 cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5750 return true;
5751 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5752 // This check is not guaranteeing uniqueness but for now that we cannot
5753 // end up with two versions of \p U thinking it was one.
5754 auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5755 if (!Callee || !Callee->hasLocalLinkage())
5756 return true;
5757 if (!CB->isArgOperand(U: &U))
5758 return false;
5759 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5760 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5761 DepClass: DepClassTy::OPTIONAL);
5762 if (!ArgInstanceInfoAA ||
5763 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5764 return false;
5765 // If this call base might reach the scope again we might forward the
5766 // argument back here. This is very conservative.
5767 if (AA::isPotentiallyReachable(
5768 A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5769 GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5770 return false;
5771 return true;
5772 }
5773 return false;
5774 };
5775
5776 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5777 if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5778 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5779 if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5780 AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5781 return true;
5782 }
5783 return false;
5784 };
5785
5786 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5787 LivenessDepClass: DepClassTy::OPTIONAL,
5788 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5789 return indicatePessimisticFixpoint();
5790
5791 return Changed;
5792 }
5793
5794 /// See AbstractState::getAsStr().
5795 const std::string getAsStr(Attributor *A) const override {
5796 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5797 }
5798
5799 /// See AbstractAttribute::trackStatistics()
5800 void trackStatistics() const override {}
5801};
5802
5803/// InstanceInfo attribute for floating values.
5804struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5805 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5806 : AAInstanceInfoImpl(IRP, A) {}
5807};
5808
5809/// NoCapture attribute for function arguments.
5810struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5811 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5812 : AAInstanceInfoFloating(IRP, A) {}
5813};
5814
5815/// InstanceInfo attribute for call site arguments.
5816struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5817 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5818 : AAInstanceInfoImpl(IRP, A) {}
5819
5820 /// See AbstractAttribute::updateImpl(...).
5821 ChangeStatus updateImpl(Attributor &A) override {
5822 // TODO: Once we have call site specific value information we can provide
5823 // call site specific liveness information and then it makes
5824 // sense to specialize attributes for call sites arguments instead of
5825 // redirecting requests to the callee argument.
5826 Argument *Arg = getAssociatedArgument();
5827 if (!Arg)
5828 return indicatePessimisticFixpoint();
5829 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5830 auto *ArgAA =
5831 A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5832 if (!ArgAA)
5833 return indicatePessimisticFixpoint();
5834 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5835 }
5836};
5837
5838/// InstanceInfo attribute for function return value.
5839struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5840 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5841 : AAInstanceInfoImpl(IRP, A) {
5842 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5843 }
5844
5845 /// See AbstractAttribute::initialize(...).
5846 void initialize(Attributor &A) override {
5847 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5848 }
5849
5850 /// See AbstractAttribute::updateImpl(...).
5851 ChangeStatus updateImpl(Attributor &A) override {
5852 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5853 }
5854};
5855
5856/// InstanceInfo attribute deduction for a call site return value.
5857struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5858 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5859 : AAInstanceInfoFloating(IRP, A) {}
5860};
5861} // namespace
5862
5863/// ----------------------- Variable Capturing ---------------------------------
5864bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5865 Attribute::AttrKind ImpliedAttributeKind,
5866 bool IgnoreSubsumingPositions) {
5867 assert(ImpliedAttributeKind == Attribute::Captures &&
5868 "Unexpected attribute kind");
5869 Value &V = IRP.getAssociatedValue();
5870 if (!isa<Constant>(Val: V) && !IRP.isArgumentPosition())
5871 return V.use_empty();
5872
5873 // You cannot "capture" null in the default address space.
5874 //
5875 // FIXME: This should use NullPointerIsDefined to account for the function
5876 // attribute.
5877 if (isa<UndefValue>(Val: V) || (isa<ConstantPointerNull>(Val: V) &&
5878 V.getType()->getPointerAddressSpace() == 0)) {
5879 return true;
5880 }
5881
5882 SmallVector<Attribute, 1> Attrs;
5883 A.getAttrs(IRP, AKs: {Attribute::Captures}, Attrs,
5884 /* IgnoreSubsumingPositions */ true);
5885 for (const Attribute &Attr : Attrs)
5886 if (capturesNothing(CC: Attr.getCaptureInfo()))
5887 return true;
5888
5889 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5890 if (Argument *Arg = IRP.getAssociatedArgument()) {
5891 SmallVector<Attribute, 1> Attrs;
5892 A.getAttrs(IRP: IRPosition::argument(Arg: *Arg),
5893 AKs: {Attribute::Captures, Attribute::ByVal}, Attrs,
5894 /* IgnoreSubsumingPositions */ true);
5895 bool ArgNoCapture = any_of(Range&: Attrs, P: [](Attribute Attr) {
5896 return Attr.getKindAsEnum() == Attribute::ByVal ||
5897 capturesNothing(CC: Attr.getCaptureInfo());
5898 });
5899 if (ArgNoCapture) {
5900 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(
5901 Context&: V.getContext(), CI: CaptureInfo::none()));
5902 return true;
5903 }
5904 }
5905
5906 if (const Function *F = IRP.getAssociatedFunction()) {
5907 // Check what state the associated function can actually capture.
5908 AANoCapture::StateType State;
5909 determineFunctionCaptureCapabilities(IRP, F: *F, State);
5910 if (State.isKnown(BitsEncoding: NO_CAPTURE)) {
5911 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(Context&: V.getContext(),
5912 CI: CaptureInfo::none()));
5913 return true;
5914 }
5915 }
5916
5917 return false;
5918}
5919
5920/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5921/// depending on the ability of the function associated with \p IRP to capture
5922/// state in memory and through "returning/throwing", respectively.
5923void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5924 const Function &F,
5925 BitIntegerState &State) {
5926 // TODO: Once we have memory behavior attributes we should use them here.
5927
5928 // If we know we cannot communicate or write to memory, we do not care about
5929 // ptr2int anymore.
5930 bool ReadOnly = F.onlyReadsMemory();
5931 bool NoThrow = F.doesNotThrow();
5932 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5933 if (ReadOnly && NoThrow && IsVoidReturn) {
5934 State.addKnownBits(Bits: NO_CAPTURE);
5935 return;
5936 }
5937
5938 // A function cannot capture state in memory if it only reads memory, it can
5939 // however return/throw state and the state might be influenced by the
5940 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5941 if (ReadOnly)
5942 State.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5943
5944 // A function cannot communicate state back if it does not through
5945 // exceptions and doesn not return values.
5946 if (NoThrow && IsVoidReturn)
5947 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5948
5949 // Check existing "returned" attributes.
5950 int ArgNo = IRP.getCalleeArgNo();
5951 if (!NoThrow || ArgNo < 0 ||
5952 !F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned))
5953 return;
5954
5955 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5956 if (F.hasParamAttribute(ArgNo: U, Kind: Attribute::Returned)) {
5957 if (U == unsigned(ArgNo))
5958 State.removeAssumedBits(BitsEncoding: NOT_CAPTURED_IN_RET);
5959 else if (ReadOnly)
5960 State.addKnownBits(Bits: NO_CAPTURE);
5961 else
5962 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5963 break;
5964 }
5965}
5966
5967namespace {
5968/// A class to hold the state of for no-capture attributes.
5969struct AANoCaptureImpl : public AANoCapture {
5970 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5971
5972 /// See AbstractAttribute::initialize(...).
5973 void initialize(Attributor &A) override {
5974 bool IsKnown;
5975 assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
5976 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5977 (void)IsKnown;
5978 }
5979
5980 /// See AbstractAttribute::updateImpl(...).
5981 ChangeStatus updateImpl(Attributor &A) override;
5982
5983 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5984 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5985 SmallVectorImpl<Attribute> &Attrs) const override {
5986 if (!isAssumedNoCaptureMaybeReturned())
5987 return;
5988
5989 if (isArgumentPosition()) {
5990 if (isAssumedNoCapture())
5991 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: Attribute::Captures));
5992 else if (ManifestInternal)
5993 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
5994 }
5995 }
5996
5997 /// See AbstractState::getAsStr().
5998 const std::string getAsStr(Attributor *A) const override {
5999 if (isKnownNoCapture())
6000 return "known not-captured";
6001 if (isAssumedNoCapture())
6002 return "assumed not-captured";
6003 if (isKnownNoCaptureMaybeReturned())
6004 return "known not-captured-maybe-returned";
6005 if (isAssumedNoCaptureMaybeReturned())
6006 return "assumed not-captured-maybe-returned";
6007 return "assumed-captured";
6008 }
6009
6010 /// Check the use \p U and update \p State accordingly. Return true if we
6011 /// should continue to update the state.
6012 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
6013 bool &Follow) {
6014 Instruction *UInst = cast<Instruction>(Val: U.getUser());
6015 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
6016 << *UInst << "\n");
6017
6018 // Deal with ptr2int by following uses.
6019 if (isa<PtrToIntInst>(Val: UInst)) {
6020 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
6021 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6022 /* Return */ CapturedInRet: true);
6023 }
6024
6025 // For stores we already checked if we can follow them, if they make it
6026 // here we give up.
6027 if (isa<StoreInst>(Val: UInst))
6028 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6029 /* Return */ CapturedInRet: true);
6030
6031 // Explicitly catch return instructions.
6032 if (isa<ReturnInst>(Val: UInst)) {
6033 if (UInst->getFunction() == getAnchorScope())
6034 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6035 /* Return */ CapturedInRet: true);
6036 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6037 /* Return */ CapturedInRet: true);
6038 }
6039
6040 // For now we only use special logic for call sites. However, the tracker
6041 // itself knows about a lot of other non-capturing cases already.
6042 auto *CB = dyn_cast<CallBase>(Val: UInst);
6043 if (!CB || !CB->isArgOperand(U: &U))
6044 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6045 /* Return */ CapturedInRet: true);
6046
6047 unsigned ArgNo = CB->getArgOperandNo(U: &U);
6048 const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
6049 // If we have a abstract no-capture attribute for the argument we can use
6050 // it to justify a non-capture attribute here. This allows recursion!
6051 bool IsKnownNoCapture;
6052 const AANoCapture *ArgNoCaptureAA = nullptr;
6053 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
6054 A, QueryingAA: this, IRP: CSArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6055 AAPtr: &ArgNoCaptureAA);
6056 if (IsAssumedNoCapture)
6057 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6058 /* Return */ CapturedInRet: false);
6059 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
6060 Follow = true;
6061 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6062 /* Return */ CapturedInRet: false);
6063 }
6064
6065 // Lastly, we could not find a reason no-capture can be assumed so we don't.
6066 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6067 /* Return */ CapturedInRet: true);
6068 }
6069
6070 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
6071 /// \p CapturedInRet, then return true if we should continue updating the
6072 /// state.
6073 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
6074 bool CapturedInInt, bool CapturedInRet) {
6075 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
6076 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
6077 if (CapturedInMem)
6078 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_MEM);
6079 if (CapturedInInt)
6080 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_INT);
6081 if (CapturedInRet)
6082 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_RET);
6083 return State.isAssumed(BitsEncoding: AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
6084 }
6085};
6086
6087ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
6088 const IRPosition &IRP = getIRPosition();
6089 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
6090 : &IRP.getAssociatedValue();
6091 if (!V)
6092 return indicatePessimisticFixpoint();
6093
6094 const Function *F =
6095 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
6096
6097 // TODO: Is the checkForAllUses below useful for constants?
6098 if (!F)
6099 return indicatePessimisticFixpoint();
6100
6101 AANoCapture::StateType T;
6102 const IRPosition &FnPos = IRPosition::function(F: *F);
6103
6104 // Readonly means we cannot capture through memory.
6105 bool IsKnown;
6106 if (AA::isAssumedReadOnly(A, IRP: FnPos, QueryingAA: *this, IsKnown)) {
6107 T.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6108 if (IsKnown)
6109 addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6110 }
6111
6112 // Make sure all returned values are different than the underlying value.
6113 // TODO: we could do this in a more sophisticated way inside
6114 // AAReturnedValues, e.g., track all values that escape through returns
6115 // directly somehow.
6116 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
6117 SmallVector<AA::ValueAndContext> Values;
6118 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *F), AA: this, Values,
6119 S: AA::ValueScope::Intraprocedural,
6120 UsedAssumedInformation))
6121 return false;
6122 bool SeenConstant = false;
6123 for (const AA::ValueAndContext &VAC : Values) {
6124 if (isa<Constant>(Val: VAC.getValue())) {
6125 if (SeenConstant)
6126 return false;
6127 SeenConstant = true;
6128 } else if (!isa<Argument>(Val: VAC.getValue()) ||
6129 VAC.getValue() == getAssociatedArgument())
6130 return false;
6131 }
6132 return true;
6133 };
6134
6135 bool IsKnownNoUnwind;
6136 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
6137 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
6138 bool IsVoidTy = F->getReturnType()->isVoidTy();
6139 bool UsedAssumedInformation = false;
6140 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
6141 T.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6142 if (T.isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6143 return ChangeStatus::UNCHANGED;
6144 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
6145 addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6146 if (isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6147 return indicateOptimisticFixpoint();
6148 }
6149 }
6150 }
6151
6152 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
6153 // TODO(captures): Make this more precise.
6154 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6155 if (capturesNothing(CC: CI))
6156 return true;
6157 if (CI.isPassthrough()) {
6158 Follow = true;
6159 return true;
6160 }
6161 return checkUse(A, State&: T, U, Follow);
6162 };
6163
6164 if (!A.checkForAllUses(Pred: UseCheck, QueryingAA: *this, V: *V))
6165 return indicatePessimisticFixpoint();
6166
6167 AANoCapture::StateType &S = getState();
6168 auto Assumed = S.getAssumed();
6169 S.intersectAssumedBits(BitsEncoding: T.getAssumed());
6170 if (!isAssumedNoCaptureMaybeReturned())
6171 return indicatePessimisticFixpoint();
6172 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6173 : ChangeStatus::CHANGED;
6174}
6175
6176/// NoCapture attribute for function arguments.
6177struct AANoCaptureArgument final : AANoCaptureImpl {
6178 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6179 : AANoCaptureImpl(IRP, A) {}
6180
6181 /// See AbstractAttribute::trackStatistics()
6182 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6183};
6184
6185/// NoCapture attribute for call site arguments.
6186struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6187 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6188 : AANoCaptureImpl(IRP, A) {}
6189
6190 /// See AbstractAttribute::updateImpl(...).
6191 ChangeStatus updateImpl(Attributor &A) override {
6192 // TODO: Once we have call site specific value information we can provide
6193 // call site specific liveness information and then it makes
6194 // sense to specialize attributes for call sites arguments instead of
6195 // redirecting requests to the callee argument.
6196 Argument *Arg = getAssociatedArgument();
6197 if (!Arg)
6198 return indicatePessimisticFixpoint();
6199 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6200 bool IsKnownNoCapture;
6201 const AANoCapture *ArgAA = nullptr;
6202 if (AA::hasAssumedIRAttr<Attribute::Captures>(
6203 A, QueryingAA: this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6204 AAPtr: &ArgAA))
6205 return ChangeStatus::UNCHANGED;
6206 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6207 return indicatePessimisticFixpoint();
6208 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
6209 }
6210
6211 /// See AbstractAttribute::trackStatistics()
6212 void trackStatistics() const override {
6213 STATS_DECLTRACK_CSARG_ATTR(nocapture)
6214 };
6215};
6216
6217/// NoCapture attribute for floating values.
6218struct AANoCaptureFloating final : AANoCaptureImpl {
6219 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6220 : AANoCaptureImpl(IRP, A) {}
6221
6222 /// See AbstractAttribute::trackStatistics()
6223 void trackStatistics() const override {
6224 STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6225 }
6226};
6227
6228/// NoCapture attribute for function return value.
6229struct AANoCaptureReturned final : AANoCaptureImpl {
6230 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6231 : AANoCaptureImpl(IRP, A) {
6232 llvm_unreachable("NoCapture is not applicable to function returns!");
6233 }
6234
6235 /// See AbstractAttribute::initialize(...).
6236 void initialize(Attributor &A) override {
6237 llvm_unreachable("NoCapture is not applicable to function returns!");
6238 }
6239
6240 /// See AbstractAttribute::updateImpl(...).
6241 ChangeStatus updateImpl(Attributor &A) override {
6242 llvm_unreachable("NoCapture is not applicable to function returns!");
6243 }
6244
6245 /// See AbstractAttribute::trackStatistics()
6246 void trackStatistics() const override {}
6247};
6248
6249/// NoCapture attribute deduction for a call site return value.
6250struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6251 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6252 : AANoCaptureImpl(IRP, A) {}
6253
6254 /// See AbstractAttribute::initialize(...).
6255 void initialize(Attributor &A) override {
6256 const Function *F = getAnchorScope();
6257 // Check what state the associated function can actually capture.
6258 determineFunctionCaptureCapabilities(IRP: getIRPosition(), F: *F, State&: *this);
6259 }
6260
6261 /// See AbstractAttribute::trackStatistics()
6262 void trackStatistics() const override {
6263 STATS_DECLTRACK_CSRET_ATTR(nocapture)
6264 }
6265};
6266} // namespace
6267
6268/// ------------------ Value Simplify Attribute ----------------------------
6269
6270bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6271 // FIXME: Add a typecast support.
6272 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6273 A: SimplifiedAssociatedValue, B: Other, Ty);
6274 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6275 return false;
6276
6277 LLVM_DEBUG({
6278 if (SimplifiedAssociatedValue)
6279 dbgs() << "[ValueSimplify] is assumed to be "
6280 << **SimplifiedAssociatedValue << "\n";
6281 else
6282 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6283 });
6284 return true;
6285}
6286
6287namespace {
6288struct AAValueSimplifyImpl : AAValueSimplify {
6289 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6290 : AAValueSimplify(IRP, A) {}
6291
6292 /// See AbstractAttribute::initialize(...).
6293 void initialize(Attributor &A) override {
6294 if (getAssociatedValue().getType()->isVoidTy())
6295 indicatePessimisticFixpoint();
6296 if (A.hasSimplificationCallback(IRP: getIRPosition()))
6297 indicatePessimisticFixpoint();
6298 }
6299
6300 /// See AbstractAttribute::getAsStr().
6301 const std::string getAsStr(Attributor *A) const override {
6302 LLVM_DEBUG({
6303 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6304 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6305 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6306 });
6307 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6308 : "not-simple";
6309 }
6310
6311 /// See AbstractAttribute::trackStatistics()
6312 void trackStatistics() const override {}
6313
6314 /// See AAValueSimplify::getAssumedSimplifiedValue()
6315 std::optional<Value *>
6316 getAssumedSimplifiedValue(Attributor &A) const override {
6317 return SimplifiedAssociatedValue;
6318 }
6319
6320 /// Ensure the return value is \p V with type \p Ty, if not possible return
6321 /// nullptr. If \p Check is true we will only verify such an operation would
6322 /// suceed and return a non-nullptr value if that is the case. No IR is
6323 /// generated or modified.
6324 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6325 bool Check) {
6326 if (auto *TypedV = AA::getWithType(V, Ty))
6327 return TypedV;
6328 if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6329 return Check ? &V
6330 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6331 S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6332 return nullptr;
6333 }
6334
6335 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6336 /// If \p Check is true we will only verify such an operation would suceed and
6337 /// return a non-nullptr value if that is the case. No IR is generated or
6338 /// modified.
6339 static Value *reproduceInst(Attributor &A,
6340 const AbstractAttribute &QueryingAA,
6341 Instruction &I, Type &Ty, Instruction *CtxI,
6342 bool Check, ValueToValueMapTy &VMap) {
6343 assert(CtxI && "Cannot reproduce an instruction without context!");
6344 if (Check && (I.mayReadFromMemory() ||
6345 !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6346 /* TLI */ DT: nullptr)))
6347 return nullptr;
6348 for (Value *Op : I.operands()) {
6349 Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6350 if (!NewOp) {
6351 assert(Check && "Manifest of new value unexpectedly failed!");
6352 return nullptr;
6353 }
6354 if (!Check)
6355 VMap[Op] = NewOp;
6356 }
6357 if (Check)
6358 return &I;
6359
6360 Instruction *CloneI = I.clone();
6361 // TODO: Try to salvage debug information here.
6362 CloneI->setDebugLoc(DebugLoc());
6363 VMap[&I] = CloneI;
6364 CloneI->insertBefore(InsertPos: CtxI->getIterator());
6365 RemapInstruction(I: CloneI, VM&: VMap);
6366 return CloneI;
6367 }
6368
6369 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6370 /// If \p Check is true we will only verify such an operation would suceed and
6371 /// return a non-nullptr value if that is the case. No IR is generated or
6372 /// modified.
6373 static Value *reproduceValue(Attributor &A,
6374 const AbstractAttribute &QueryingAA, Value &V,
6375 Type &Ty, Instruction *CtxI, bool Check,
6376 ValueToValueMapTy &VMap) {
6377 if (const auto &NewV = VMap.lookup(Val: &V))
6378 return NewV;
6379 bool UsedAssumedInformation = false;
6380 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6381 V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6382 if (!SimpleV.has_value())
6383 return PoisonValue::get(T: &Ty);
6384 Value *EffectiveV = &V;
6385 if (*SimpleV)
6386 EffectiveV = *SimpleV;
6387 if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6388 return C;
6389 if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6390 InfoCache&: A.getInfoCache()))
6391 return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6392 if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6393 if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6394 return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6395 return nullptr;
6396 }
6397
6398 /// Return a value we can use as replacement for the associated one, or
6399 /// nullptr if we don't have one that makes sense.
6400 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6401 Value *NewV = SimplifiedAssociatedValue
6402 ? *SimplifiedAssociatedValue
6403 : UndefValue::get(T: getAssociatedType());
6404 if (NewV && NewV != &getAssociatedValue()) {
6405 ValueToValueMapTy VMap;
6406 // First verify we can reprduce the value with the required type at the
6407 // context location before we actually start modifying the IR.
6408 if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6409 /* CheckOnly */ Check: true, VMap))
6410 return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6411 /* CheckOnly */ Check: false, VMap);
6412 }
6413 return nullptr;
6414 }
6415
6416 /// Helper function for querying AAValueSimplify and updating candidate.
6417 /// \param IRP The value position we are trying to unify with SimplifiedValue
6418 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6419 const IRPosition &IRP, bool Simplify = true) {
6420 bool UsedAssumedInformation = false;
6421 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6422 if (Simplify)
6423 QueryingValueSimplified = A.getAssumedSimplified(
6424 IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6425 return unionAssumed(Other: QueryingValueSimplified);
6426 }
6427
6428 /// Returns a candidate is found or not
6429 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6430 if (!getAssociatedValue().getType()->isIntegerTy())
6431 return false;
6432
6433 // This will also pass the call base context.
6434 const auto *AA =
6435 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6436 if (!AA)
6437 return false;
6438
6439 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6440
6441 if (!COpt) {
6442 SimplifiedAssociatedValue = std::nullopt;
6443 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6444 return true;
6445 }
6446 if (auto *C = *COpt) {
6447 SimplifiedAssociatedValue = C;
6448 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6449 return true;
6450 }
6451 return false;
6452 }
6453
6454 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6455 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6456 return true;
6457 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6458 return true;
6459 return false;
6460 }
6461
6462 /// See AbstractAttribute::manifest(...).
6463 ChangeStatus manifest(Attributor &A) override {
6464 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6465 for (auto &U : getAssociatedValue().uses()) {
6466 // Check if we need to adjust the insertion point to make sure the IR is
6467 // valid.
6468 Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6469 if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6470 IP = PHI->getIncomingBlock(U)->getTerminator();
6471 if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6472 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6473 << " -> " << *NewV << " :: " << *this << "\n");
6474 if (A.changeUseAfterManifest(U, NV&: *NewV))
6475 Changed = ChangeStatus::CHANGED;
6476 }
6477 }
6478
6479 return Changed | AAValueSimplify::manifest(A);
6480 }
6481
6482 /// See AbstractState::indicatePessimisticFixpoint(...).
6483 ChangeStatus indicatePessimisticFixpoint() override {
6484 SimplifiedAssociatedValue = &getAssociatedValue();
6485 return AAValueSimplify::indicatePessimisticFixpoint();
6486 }
6487};
6488
6489struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6490 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6491 : AAValueSimplifyImpl(IRP, A) {}
6492
6493 void initialize(Attributor &A) override {
6494 AAValueSimplifyImpl::initialize(A);
6495 if (A.hasAttr(IRP: getIRPosition(),
6496 AKs: {Attribute::InAlloca, Attribute::Preallocated,
6497 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6498 /* IgnoreSubsumingPositions */ true))
6499 indicatePessimisticFixpoint();
6500 }
6501
6502 /// See AbstractAttribute::updateImpl(...).
6503 ChangeStatus updateImpl(Attributor &A) override {
6504 // Byval is only replacable if it is readonly otherwise we would write into
6505 // the replaced value and not the copy that byval creates implicitly.
6506 Argument *Arg = getAssociatedArgument();
6507 if (Arg->hasByValAttr()) {
6508 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6509 // there is no race by not copying a constant byval.
6510 bool IsKnown;
6511 if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6512 return indicatePessimisticFixpoint();
6513 }
6514
6515 auto Before = SimplifiedAssociatedValue;
6516
6517 auto PredForCallSite = [&](AbstractCallSite ACS) {
6518 const IRPosition &ACSArgPos =
6519 IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6520 // Check if a coresponding argument was found or if it is on not
6521 // associated (which can happen for callback calls).
6522 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6523 return false;
6524
6525 // Simplify the argument operand explicitly and check if the result is
6526 // valid in the current scope. This avoids refering to simplified values
6527 // in other functions, e.g., we don't want to say a an argument in a
6528 // static function is actually an argument in a different function.
6529 bool UsedAssumedInformation = false;
6530 std::optional<Constant *> SimpleArgOp =
6531 A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6532 if (!SimpleArgOp)
6533 return true;
6534 if (!*SimpleArgOp)
6535 return false;
6536 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6537 return false;
6538 return unionAssumed(Other: *SimpleArgOp);
6539 };
6540
6541 // Generate a answer specific to a call site context.
6542 bool Success;
6543 bool UsedAssumedInformation = false;
6544 if (hasCallBaseContext() &&
6545 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6546 Success = PredForCallSite(
6547 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6548 else
6549 Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6550 UsedAssumedInformation);
6551
6552 if (!Success)
6553 if (!askSimplifiedValueForOtherAAs(A))
6554 return indicatePessimisticFixpoint();
6555
6556 // If a candidate was found in this update, return CHANGED.
6557 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6558 : ChangeStatus ::CHANGED;
6559 }
6560
6561 /// See AbstractAttribute::trackStatistics()
6562 void trackStatistics() const override {
6563 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6564 }
6565};
6566
6567struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6568 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6569 : AAValueSimplifyImpl(IRP, A) {}
6570
6571 /// See AAValueSimplify::getAssumedSimplifiedValue()
6572 std::optional<Value *>
6573 getAssumedSimplifiedValue(Attributor &A) const override {
6574 if (!isValidState())
6575 return nullptr;
6576 return SimplifiedAssociatedValue;
6577 }
6578
6579 /// See AbstractAttribute::updateImpl(...).
6580 ChangeStatus updateImpl(Attributor &A) override {
6581 auto Before = SimplifiedAssociatedValue;
6582
6583 auto ReturnInstCB = [&](Instruction &I) {
6584 auto &RI = cast<ReturnInst>(Val&: I);
6585 return checkAndUpdate(
6586 A, QueryingAA: *this,
6587 IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6588 };
6589
6590 bool UsedAssumedInformation = false;
6591 if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6592 UsedAssumedInformation))
6593 if (!askSimplifiedValueForOtherAAs(A))
6594 return indicatePessimisticFixpoint();
6595
6596 // If a candidate was found in this update, return CHANGED.
6597 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6598 : ChangeStatus ::CHANGED;
6599 }
6600
6601 ChangeStatus manifest(Attributor &A) override {
6602 // We queried AAValueSimplify for the returned values so they will be
6603 // replaced if a simplified form was found. Nothing to do here.
6604 return ChangeStatus::UNCHANGED;
6605 }
6606
6607 /// See AbstractAttribute::trackStatistics()
6608 void trackStatistics() const override {
6609 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6610 }
6611};
6612
6613struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6614 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6615 : AAValueSimplifyImpl(IRP, A) {}
6616
6617 /// See AbstractAttribute::initialize(...).
6618 void initialize(Attributor &A) override {
6619 AAValueSimplifyImpl::initialize(A);
6620 Value &V = getAnchorValue();
6621
6622 // TODO: add other stuffs
6623 if (isa<Constant>(Val: V))
6624 indicatePessimisticFixpoint();
6625 }
6626
6627 /// See AbstractAttribute::updateImpl(...).
6628 ChangeStatus updateImpl(Attributor &A) override {
6629 auto Before = SimplifiedAssociatedValue;
6630 if (!askSimplifiedValueForOtherAAs(A))
6631 return indicatePessimisticFixpoint();
6632
6633 // If a candidate was found in this update, return CHANGED.
6634 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6635 : ChangeStatus ::CHANGED;
6636 }
6637
6638 /// See AbstractAttribute::trackStatistics()
6639 void trackStatistics() const override {
6640 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6641 }
6642};
6643
6644struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6645 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6646 : AAValueSimplifyImpl(IRP, A) {}
6647
6648 /// See AbstractAttribute::initialize(...).
6649 void initialize(Attributor &A) override {
6650 SimplifiedAssociatedValue = nullptr;
6651 indicateOptimisticFixpoint();
6652 }
6653 /// See AbstractAttribute::initialize(...).
6654 ChangeStatus updateImpl(Attributor &A) override {
6655 llvm_unreachable(
6656 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6657 }
6658 /// See AbstractAttribute::trackStatistics()
6659 void trackStatistics() const override {
6660 STATS_DECLTRACK_FN_ATTR(value_simplify)
6661 }
6662};
6663
6664struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6665 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6666 : AAValueSimplifyFunction(IRP, A) {}
6667 /// See AbstractAttribute::trackStatistics()
6668 void trackStatistics() const override {
6669 STATS_DECLTRACK_CS_ATTR(value_simplify)
6670 }
6671};
6672
6673struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6674 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6675 : AAValueSimplifyImpl(IRP, A) {}
6676
6677 void initialize(Attributor &A) override {
6678 AAValueSimplifyImpl::initialize(A);
6679 Function *Fn = getAssociatedFunction();
6680 assert(Fn && "Did expect an associted function");
6681 for (Argument &Arg : Fn->args()) {
6682 if (Arg.hasReturnedAttr()) {
6683 auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6684 ArgNo: Arg.getArgNo());
6685 if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6686 checkAndUpdate(A, QueryingAA: *this, IRP))
6687 indicateOptimisticFixpoint();
6688 else
6689 indicatePessimisticFixpoint();
6690 return;
6691 }
6692 }
6693 }
6694
6695 /// See AbstractAttribute::updateImpl(...).
6696 ChangeStatus updateImpl(Attributor &A) override {
6697 return indicatePessimisticFixpoint();
6698 }
6699
6700 void trackStatistics() const override {
6701 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6702 }
6703};
6704
6705struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6706 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6707 : AAValueSimplifyFloating(IRP, A) {}
6708
6709 /// See AbstractAttribute::manifest(...).
6710 ChangeStatus manifest(Attributor &A) override {
6711 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6712 // TODO: We should avoid simplification duplication to begin with.
6713 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6714 IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6715 if (FloatAA && FloatAA->getState().isValidState())
6716 return Changed;
6717
6718 if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6719 Use &U = cast<CallBase>(Val: &getAnchorValue())
6720 ->getArgOperandUse(i: getCallSiteArgNo());
6721 if (A.changeUseAfterManifest(U, NV&: *NewV))
6722 Changed = ChangeStatus::CHANGED;
6723 }
6724
6725 return Changed | AAValueSimplify::manifest(A);
6726 }
6727
6728 void trackStatistics() const override {
6729 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6730 }
6731};
6732} // namespace
6733
6734/// ----------------------- Heap-To-Stack Conversion ---------------------------
6735namespace {
6736struct AAHeapToStackFunction final : public AAHeapToStack {
6737
6738 static bool isGlobalizedLocal(const CallBase &CB) {
6739 Attribute A = CB.getFnAttr(Kind: "alloc-family");
6740 return A.isValid() && A.getValueAsString() == "__kmpc_alloc_shared";
6741 }
6742
6743 struct AllocationInfo {
6744 /// The call that allocates the memory.
6745 CallBase *const CB;
6746
6747 /// Whether this allocation is an OpenMP globalized local variable.
6748 bool IsGlobalizedLocal = false;
6749
6750 /// The status wrt. a rewrite.
6751 enum {
6752 STACK_DUE_TO_USE,
6753 STACK_DUE_TO_FREE,
6754 INVALID,
6755 } Status = STACK_DUE_TO_USE;
6756
6757 /// Flag to indicate if we encountered a use that might free this allocation
6758 /// but which is not in the deallocation infos.
6759 bool HasPotentiallyFreeingUnknownUses = false;
6760
6761 /// Flag to indicate that we should place the new alloca in the function
6762 /// entry block rather than where the call site (CB) is.
6763 bool MoveAllocaIntoEntry = true;
6764
6765 /// The set of free calls that use this allocation.
6766 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6767 };
6768
6769 struct DeallocationInfo {
6770 /// The call that deallocates the memory.
6771 CallBase *const CB;
6772 /// The value freed by the call.
6773 Value *FreedOp;
6774
6775 /// Flag to indicate if we don't know all objects this deallocation might
6776 /// free.
6777 bool MightFreeUnknownObjects = false;
6778
6779 /// The set of allocation calls that are potentially freed.
6780 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6781 };
6782
6783 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6784 : AAHeapToStack(IRP, A) {}
6785
6786 ~AAHeapToStackFunction() override {
6787 // Ensure we call the destructor so we release any memory allocated in the
6788 // sets.
6789 for (auto &It : AllocationInfos)
6790 It.second->~AllocationInfo();
6791 for (auto &It : DeallocationInfos)
6792 It.second->~DeallocationInfo();
6793 }
6794
6795 void initialize(Attributor &A) override {
6796 AAHeapToStack::initialize(A);
6797
6798 const Function *F = getAnchorScope();
6799 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6800
6801 auto AllocationIdentifierCB = [&](Instruction &I) {
6802 CallBase *CB = dyn_cast<CallBase>(Val: &I);
6803 if (!CB)
6804 return true;
6805 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6806 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6807 return true;
6808 }
6809 // To do heap to stack, we need to know that the allocation itself is
6810 // removable once uses are rewritten, and that we can initialize the
6811 // alloca to the same pattern as the original allocation result.
6812 if (isRemovableAlloc(V: CB, TLI)) {
6813 auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6814 if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6815 AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6816 AllocationInfos[CB] = AI;
6817 AI->IsGlobalizedLocal = isGlobalizedLocal(CB: *CB);
6818 }
6819 }
6820 return true;
6821 };
6822
6823 bool UsedAssumedInformation = false;
6824 bool Success = A.checkForAllCallLikeInstructions(
6825 Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6826 /* CheckBBLivenessOnly */ false,
6827 /* CheckPotentiallyDead */ true);
6828 (void)Success;
6829 assert(Success && "Did not expect the call base visit callback to fail!");
6830
6831 Attributor::SimplifictionCallbackTy SCB =
6832 [](const IRPosition &, const AbstractAttribute *,
6833 bool &) -> std::optional<Value *> { return nullptr; };
6834 for (const auto &It : AllocationInfos)
6835 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6836 CB: SCB);
6837 for (const auto &It : DeallocationInfos)
6838 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6839 CB: SCB);
6840 }
6841
6842 const std::string getAsStr(Attributor *A) const override {
6843 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6844 for (const auto &It : AllocationInfos) {
6845 if (It.second->Status == AllocationInfo::INVALID)
6846 ++NumInvalidMallocs;
6847 else
6848 ++NumH2SMallocs;
6849 }
6850 return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6851 std::to_string(val: NumInvalidMallocs);
6852 }
6853
6854 /// See AbstractAttribute::trackStatistics().
6855 void trackStatistics() const override {
6856 STATS_DECL(
6857 MallocCalls, Function,
6858 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6859 for (const auto &It : AllocationInfos)
6860 if (It.second->Status != AllocationInfo::INVALID)
6861 ++BUILD_STAT_NAME(MallocCalls, Function);
6862 }
6863
6864 bool isAssumedHeapToStack(const CallBase &CB) const override {
6865 if (isValidState())
6866 if (AllocationInfo *AI =
6867 AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6868 return AI->Status != AllocationInfo::INVALID;
6869 return false;
6870 }
6871
6872 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6873 if (!isValidState())
6874 return false;
6875
6876 for (const auto &It : AllocationInfos) {
6877 AllocationInfo &AI = *It.second;
6878 if (AI.Status == AllocationInfo::INVALID)
6879 continue;
6880
6881 if (AI.PotentialFreeCalls.count(key: &CB))
6882 return true;
6883 }
6884
6885 return false;
6886 }
6887
6888 ChangeStatus manifest(Attributor &A) override {
6889 assert(getState().isValidState() &&
6890 "Attempted to manifest an invalid state!");
6891
6892 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6893 Function *F = getAnchorScope();
6894 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6895
6896 for (auto &It : AllocationInfos) {
6897 AllocationInfo &AI = *It.second;
6898 if (AI.Status == AllocationInfo::INVALID)
6899 continue;
6900
6901 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6902 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6903 A.deleteAfterManifest(I&: *FreeCall);
6904 HasChanged = ChangeStatus::CHANGED;
6905 }
6906
6907 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6908 << "\n");
6909
6910 auto Remark = [&](OptimizationRemark OR) {
6911 if (AI.IsGlobalizedLocal)
6912 return OR << "Moving globalized variable to the stack.";
6913 return OR << "Moving memory allocation from the heap to the stack.";
6914 };
6915 if (AI.IsGlobalizedLocal)
6916 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6917 else
6918 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6919
6920 const DataLayout &DL = A.getInfoCache().getDL();
6921 Value *Size;
6922 std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6923 if (SizeAPI) {
6924 Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6925 } else {
6926 LLVMContext &Ctx = AI.CB->getContext();
6927 ObjectSizeOpts Opts;
6928 ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
6929 SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6930 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6931 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6932 Size = SizeOffsetPair.Size;
6933 }
6934
6935 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6936 ? F->getEntryBlock().begin()
6937 : AI.CB->getIterator();
6938
6939 Align Alignment(1);
6940 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6941 Alignment = std::max(a: Alignment, b: *RetAlign);
6942 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6943 std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6944 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6945 "Expected an alignment during manifest!");
6946 Alignment =
6947 std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6948 }
6949
6950 // TODO: Hoist the alloca towards the function entry.
6951 unsigned AS = DL.getAllocaAddrSpace();
6952 Instruction *Alloca =
6953 new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6954 AI.CB->getName() + ".h2s", IP);
6955
6956 if (Alloca->getType() != AI.CB->getType())
6957 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6958 S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6959
6960 auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6961 auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6962 assert(InitVal &&
6963 "Must be able to materialize initial memory state of allocation");
6964
6965 A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6966
6967 if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6968 auto *NBB = II->getNormalDest();
6969 UncondBrInst::Create(Target: NBB, InsertBefore: AI.CB->getParent());
6970 A.deleteAfterManifest(I&: *AI.CB);
6971 } else {
6972 A.deleteAfterManifest(I&: *AI.CB);
6973 }
6974
6975 // Initialize the alloca with the same value as used by the allocation
6976 // function. We can skip undef as the initial value of an alloc is
6977 // undef, and the memset would simply end up being DSEd.
6978 if (!isa<UndefValue>(Val: InitVal)) {
6979 IRBuilder<> Builder(Alloca->getNextNode());
6980 // TODO: Use alignment above if align!=1
6981 Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6982 }
6983 HasChanged = ChangeStatus::CHANGED;
6984 }
6985
6986 return HasChanged;
6987 }
6988
6989 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6990 Value &V) {
6991 bool UsedAssumedInformation = false;
6992 std::optional<Constant *> SimpleV =
6993 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6994 if (!SimpleV)
6995 return APInt(64, 0);
6996 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
6997 return CI->getValue();
6998 return std::nullopt;
6999 }
7000
7001 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
7002 AllocationInfo &AI) {
7003 auto Mapper = [&](const Value *V) -> const Value * {
7004 bool UsedAssumedInformation = false;
7005 if (std::optional<Constant *> SimpleV =
7006 A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
7007 if (*SimpleV)
7008 return *SimpleV;
7009 return V;
7010 };
7011
7012 const Function *F = getAnchorScope();
7013 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
7014 return getAllocSize(CB: AI.CB, TLI, Mapper);
7015 }
7016
7017 /// Collection of all malloc-like calls in a function with associated
7018 /// information.
7019 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
7020
7021 /// Collection of all free-like calls in a function with associated
7022 /// information.
7023 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
7024
7025 ChangeStatus updateImpl(Attributor &A) override;
7026};
7027
7028ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
7029 ChangeStatus Changed = ChangeStatus::UNCHANGED;
7030 const Function *F = getAnchorScope();
7031 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
7032
7033 const auto *LivenessAA =
7034 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
7035
7036 MustBeExecutedContextExplorer *Explorer =
7037 A.getInfoCache().getMustBeExecutedContextExplorer();
7038
7039 bool StackIsAccessibleByOtherThreads =
7040 A.getInfoCache().stackIsAccessibleByOtherThreads();
7041
7042 LoopInfo *LI =
7043 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
7044 std::optional<bool> MayContainIrreducibleControl;
7045 auto IsInLoop = [&](BasicBlock &BB) {
7046 if (&F->getEntryBlock() == &BB)
7047 return false;
7048 if (!MayContainIrreducibleControl.has_value())
7049 MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
7050 if (*MayContainIrreducibleControl)
7051 return true;
7052 if (!LI)
7053 return true;
7054 return LI->getLoopFor(BB: &BB) != nullptr;
7055 };
7056
7057 // Flag to ensure we update our deallocation information at most once per
7058 // updateImpl call and only if we use the free check reasoning.
7059 bool HasUpdatedFrees = false;
7060
7061 auto UpdateFrees = [&]() {
7062 HasUpdatedFrees = true;
7063
7064 for (auto &It : DeallocationInfos) {
7065 DeallocationInfo &DI = *It.second;
7066 // For now we cannot use deallocations that have unknown inputs, skip
7067 // them.
7068 if (DI.MightFreeUnknownObjects)
7069 continue;
7070
7071 // No need to analyze dead calls, ignore them instead.
7072 bool UsedAssumedInformation = false;
7073 if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
7074 /* CheckBBLivenessOnly */ true))
7075 continue;
7076
7077 // Use the non-optimistic version to get the freed object.
7078 Value *Obj = getUnderlyingObject(V: DI.FreedOp);
7079 if (!Obj) {
7080 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
7081 DI.MightFreeUnknownObjects = true;
7082 continue;
7083 }
7084
7085 // Free of null and undef can be ignored as no-ops (or UB in the latter
7086 // case).
7087 if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
7088 continue;
7089
7090 CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
7091 if (!ObjCB) {
7092 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
7093 << "\n");
7094 DI.MightFreeUnknownObjects = true;
7095 continue;
7096 }
7097
7098 AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
7099 if (!AI) {
7100 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
7101 << "\n");
7102 DI.MightFreeUnknownObjects = true;
7103 continue;
7104 }
7105
7106 DI.PotentialAllocationCalls.insert(X: ObjCB);
7107 }
7108 };
7109
7110 auto FreeCheck = [&](AllocationInfo &AI) {
7111 // If the stack is not accessible by other threads, the "must-free" logic
7112 // doesn't apply as the pointer could be shared and needs to be places in
7113 // "shareable" memory.
7114 if (!StackIsAccessibleByOtherThreads) {
7115 bool IsKnownNoSycn;
7116 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
7117 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn)) {
7118 LLVM_DEBUG(
7119 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
7120 "other threads and function is not nosync:\n");
7121 return false;
7122 }
7123 }
7124 if (!HasUpdatedFrees)
7125 UpdateFrees();
7126
7127 // TODO: Allow multi exit functions that have different free calls.
7128 if (AI.PotentialFreeCalls.size() != 1) {
7129 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
7130 << AI.PotentialFreeCalls.size() << "\n");
7131 return false;
7132 }
7133 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
7134 DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
7135 if (!DI) {
7136 LLVM_DEBUG(
7137 dbgs() << "[H2S] unique free call was not known as deallocation call "
7138 << *UniqueFree << "\n");
7139 return false;
7140 }
7141 if (DI->MightFreeUnknownObjects) {
7142 LLVM_DEBUG(
7143 dbgs() << "[H2S] unique free call might free unknown allocations\n");
7144 return false;
7145 }
7146 if (DI->PotentialAllocationCalls.empty())
7147 return true;
7148 if (DI->PotentialAllocationCalls.size() > 1) {
7149 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
7150 << DI->PotentialAllocationCalls.size()
7151 << " different allocations\n");
7152 return false;
7153 }
7154 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7155 LLVM_DEBUG(
7156 dbgs()
7157 << "[H2S] unique free call not known to free this allocation but "
7158 << **DI->PotentialAllocationCalls.begin() << "\n");
7159 return false;
7160 }
7161
7162 // __kmpc_alloc_shared and __kmpc_free_shared are by construction matched.
7163 if (!AI.IsGlobalizedLocal) {
7164 Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
7165 if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) {
7166 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7167 "with the allocation "
7168 << *UniqueFree << "\n");
7169 return false;
7170 }
7171 }
7172 return true;
7173 };
7174
7175 auto UsesCheck = [&](AllocationInfo &AI) {
7176 bool ValidUsesOnly = true;
7177
7178 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7179 Instruction *UserI = cast<Instruction>(Val: U.getUser());
7180 if (isa<LoadInst>(Val: UserI))
7181 return true;
7182 if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
7183 if (SI->getValueOperand() == U.get()) {
7184 LLVM_DEBUG(dbgs()
7185 << "[H2S] escaping store to memory: " << *UserI << "\n");
7186 ValidUsesOnly = false;
7187 } else {
7188 // A store into the malloc'ed memory is fine.
7189 }
7190 return true;
7191 }
7192 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7193 if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7194 return true;
7195 if (DeallocationInfos.count(Key: CB)) {
7196 AI.PotentialFreeCalls.insert(X: CB);
7197 return true;
7198 }
7199
7200 unsigned ArgNo = CB->getArgOperandNo(U: &U);
7201 auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7202
7203 bool IsKnownNoCapture;
7204 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7205 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
7206
7207 // If a call site argument use is nofree, we are fine.
7208 bool IsKnownNoFree;
7209 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7210 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoFree);
7211
7212 if (!IsAssumedNoCapture ||
7213 (!AI.IsGlobalizedLocal && !IsAssumedNoFree)) {
7214 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7215
7216 // Emit a missed remark if this is missed OpenMP globalization.
7217 auto Remark = [&](OptimizationRemarkMissed ORM) {
7218 return ORM
7219 << "Could not move globalized variable to the stack. "
7220 "Variable is potentially captured in call. Mark "
7221 "parameter as `__attribute__((noescape))` to override.";
7222 };
7223
7224 if (ValidUsesOnly && AI.IsGlobalizedLocal)
7225 A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7226
7227 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7228 ValidUsesOnly = false;
7229 }
7230 return true;
7231 }
7232
7233 if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7234 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7235 Follow = true;
7236 return true;
7237 }
7238 // Unknown user for which we can not track uses further (in a way that
7239 // makes sense).
7240 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7241 ValidUsesOnly = false;
7242 return true;
7243 };
7244 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7245 LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7246 EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7247 auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7248 return !SI || StackIsAccessibleByOtherThreads ||
7249 AA::isAssumedThreadLocalObject(
7250 A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7251 }))
7252 return false;
7253 return ValidUsesOnly;
7254 };
7255
7256 // The actual update starts here. We look at all allocations and depending on
7257 // their status perform the appropriate check(s).
7258 for (auto &It : AllocationInfos) {
7259 AllocationInfo &AI = *It.second;
7260 if (AI.Status == AllocationInfo::INVALID)
7261 continue;
7262
7263 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7264 std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7265 if (!APAlign) {
7266 // Can't generate an alloca which respects the required alignment
7267 // on the allocation.
7268 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7269 << "\n");
7270 AI.Status = AllocationInfo::INVALID;
7271 Changed = ChangeStatus::CHANGED;
7272 continue;
7273 }
7274 if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7275 !APAlign->isPowerOf2()) {
7276 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7277 << "\n");
7278 AI.Status = AllocationInfo::INVALID;
7279 Changed = ChangeStatus::CHANGED;
7280 continue;
7281 }
7282 }
7283
7284 std::optional<APInt> Size = getSize(A, AA: *this, AI);
7285 if (!AI.IsGlobalizedLocal && MaxHeapToStackSize != -1) {
7286 if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) {
7287 LLVM_DEBUG({
7288 if (!Size)
7289 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7290 else
7291 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7292 << MaxHeapToStackSize << "\n";
7293 });
7294
7295 AI.Status = AllocationInfo::INVALID;
7296 Changed = ChangeStatus::CHANGED;
7297 continue;
7298 }
7299 }
7300
7301 switch (AI.Status) {
7302 case AllocationInfo::STACK_DUE_TO_USE:
7303 if (UsesCheck(AI))
7304 break;
7305 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7306 [[fallthrough]];
7307 case AllocationInfo::STACK_DUE_TO_FREE:
7308 if (FreeCheck(AI))
7309 break;
7310 AI.Status = AllocationInfo::INVALID;
7311 Changed = ChangeStatus::CHANGED;
7312 break;
7313 case AllocationInfo::INVALID:
7314 llvm_unreachable("Invalid allocations should never reach this point!");
7315 };
7316
7317 // Check if we still think we can move it into the entry block. If the
7318 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7319 // ignore the potential complications associated with loops.
7320 bool IsGlobalizedLocal = AI.IsGlobalizedLocal;
7321 if (AI.MoveAllocaIntoEntry &&
7322 (!Size.has_value() ||
7323 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7324 AI.MoveAllocaIntoEntry = false;
7325 }
7326
7327 return Changed;
7328}
7329} // namespace
7330
7331/// ----------------------- Privatizable Pointers ------------------------------
7332namespace {
7333struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7334 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7335 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7336
7337 ChangeStatus indicatePessimisticFixpoint() override {
7338 AAPrivatizablePtr::indicatePessimisticFixpoint();
7339 PrivatizableType = nullptr;
7340 return ChangeStatus::CHANGED;
7341 }
7342
7343 /// Identify the type we can chose for a private copy of the underlying
7344 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7345 /// none.
7346 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7347
7348 /// Return a privatizable type that encloses both T0 and T1.
7349 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7350 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7351 std::optional<Type *> T1) {
7352 if (!T0)
7353 return T1;
7354 if (!T1)
7355 return T0;
7356 if (T0 == T1)
7357 return T0;
7358 return nullptr;
7359 }
7360
7361 std::optional<Type *> getPrivatizableType() const override {
7362 return PrivatizableType;
7363 }
7364
7365 const std::string getAsStr(Attributor *A) const override {
7366 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7367 }
7368
7369protected:
7370 std::optional<Type *> PrivatizableType;
7371};
7372
7373// TODO: Do this for call site arguments (probably also other values) as well.
7374
7375struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7376 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7377 : AAPrivatizablePtrImpl(IRP, A) {}
7378
7379 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7380 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7381 // If this is a byval argument and we know all the call sites (so we can
7382 // rewrite them), there is no need to check them explicitly.
7383 bool UsedAssumedInformation = false;
7384 SmallVector<Attribute, 1> Attrs;
7385 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::ByVal}, Attrs,
7386 /* IgnoreSubsumingPositions */ true);
7387 if (!Attrs.empty() &&
7388 A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7389 RequireAllCallSites: true, UsedAssumedInformation))
7390 return Attrs[0].getValueAsType();
7391
7392 std::optional<Type *> Ty;
7393 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7394
7395 // Make sure the associated call site argument has the same type at all call
7396 // sites and it is an allocation we know is safe to privatize, for now that
7397 // means we only allow alloca instructions.
7398 // TODO: We can additionally analyze the accesses in the callee to create
7399 // the type from that information instead. That is a little more
7400 // involved and will be done in a follow up patch.
7401 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7402 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7403 // Check if a coresponding argument was found or if it is one not
7404 // associated (which can happen for callback calls).
7405 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7406 return false;
7407
7408 // Check that all call sites agree on a type.
7409 auto *PrivCSArgAA =
7410 A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7411 if (!PrivCSArgAA)
7412 return false;
7413 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7414
7415 LLVM_DEBUG({
7416 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7417 if (CSTy && *CSTy)
7418 (*CSTy)->print(dbgs());
7419 else if (CSTy)
7420 dbgs() << "<nullptr>";
7421 else
7422 dbgs() << "<none>";
7423 });
7424
7425 Ty = combineTypes(T0: Ty, T1: CSTy);
7426
7427 LLVM_DEBUG({
7428 dbgs() << " : New Type: ";
7429 if (Ty && *Ty)
7430 (*Ty)->print(dbgs());
7431 else if (Ty)
7432 dbgs() << "<nullptr>";
7433 else
7434 dbgs() << "<none>";
7435 dbgs() << "\n";
7436 });
7437
7438 return !Ty || *Ty;
7439 };
7440
7441 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7442 UsedAssumedInformation))
7443 return nullptr;
7444 return Ty;
7445 }
7446
7447 /// See AbstractAttribute::updateImpl(...).
7448 ChangeStatus updateImpl(Attributor &A) override {
7449 PrivatizableType = identifyPrivatizableType(A);
7450 if (!PrivatizableType)
7451 return ChangeStatus::UNCHANGED;
7452 if (!*PrivatizableType)
7453 return indicatePessimisticFixpoint();
7454
7455 // The dependence is optional so we don't give up once we give up on the
7456 // alignment.
7457 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7458 DepClass: DepClassTy::OPTIONAL);
7459
7460 // Avoid arguments with padding for now.
7461 if (!A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal) &&
7462 !isDenselyPacked(Ty: *PrivatizableType, DL: A.getInfoCache().getDL())) {
7463 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7464 return indicatePessimisticFixpoint();
7465 }
7466
7467 // Collect the types that will replace the privatizable type in the function
7468 // signature.
7469 SmallVector<Type *, 16> ReplacementTypes;
7470 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7471
7472 // Verify callee and caller agree on how the promoted argument would be
7473 // passed.
7474 Function &Fn = *getIRPosition().getAnchorScope();
7475 const auto *TTI =
7476 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7477 if (!TTI) {
7478 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7479 << Fn.getName() << "\n");
7480 return indicatePessimisticFixpoint();
7481 }
7482
7483 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7484 CallBase *CB = ACS.getInstruction();
7485 return TTI->areTypesABICompatible(
7486 Caller: CB->getCaller(),
7487 Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7488 Types: ReplacementTypes);
7489 };
7490 bool UsedAssumedInformation = false;
7491 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7492 UsedAssumedInformation)) {
7493 LLVM_DEBUG(
7494 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7495 << Fn.getName() << "\n");
7496 return indicatePessimisticFixpoint();
7497 }
7498
7499 // Register a rewrite of the argument.
7500 Argument *Arg = getAssociatedArgument();
7501 if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7502 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7503 return indicatePessimisticFixpoint();
7504 }
7505
7506 unsigned ArgNo = Arg->getArgNo();
7507
7508 // Helper to check if for the given call site the associated argument is
7509 // passed to a callback where the privatization would be different.
7510 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7511 SmallVector<const Use *, 4> CallbackUses;
7512 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7513 for (const Use *U : CallbackUses) {
7514 AbstractCallSite CBACS(U);
7515 assert(CBACS && CBACS.isCallbackCall());
7516 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7517 int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7518
7519 LLVM_DEBUG({
7520 dbgs()
7521 << "[AAPrivatizablePtr] Argument " << *Arg
7522 << "check if can be privatized in the context of its parent ("
7523 << Arg->getParent()->getName()
7524 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7525 "callback ("
7526 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7527 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7528 << CBACS.getCallArgOperand(CBArg) << " vs "
7529 << CB.getArgOperand(ArgNo) << "\n"
7530 << "[AAPrivatizablePtr] " << CBArg << " : "
7531 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7532 });
7533
7534 if (CBArgNo != int(ArgNo))
7535 continue;
7536 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7537 QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7538 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7539 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7540 if (!CBArgPrivTy)
7541 continue;
7542 if (*CBArgPrivTy == PrivatizableType)
7543 continue;
7544 }
7545
7546 LLVM_DEBUG({
7547 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7548 << " cannot be privatized in the context of its parent ("
7549 << Arg->getParent()->getName()
7550 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7551 "callback ("
7552 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7553 << ").\n[AAPrivatizablePtr] for which the argument "
7554 "privatization is not compatible.\n";
7555 });
7556 return false;
7557 }
7558 }
7559 return true;
7560 };
7561
7562 // Helper to check if for the given call site the associated argument is
7563 // passed to a direct call where the privatization would be different.
7564 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7565 CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7566 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7567 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7568 "Expected a direct call operand for callback call operand");
7569
7570 Function *DCCallee =
7571 dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7572 LLVM_DEBUG({
7573 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7574 << " check if be privatized in the context of its parent ("
7575 << Arg->getParent()->getName()
7576 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7577 "direct call of ("
7578 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7579 });
7580
7581 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7582 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7583 QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7584 DepClass: DepClassTy::REQUIRED);
7585 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7586 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7587 if (!DCArgPrivTy)
7588 return true;
7589 if (*DCArgPrivTy == PrivatizableType)
7590 return true;
7591 }
7592 }
7593
7594 LLVM_DEBUG({
7595 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7596 << " cannot be privatized in the context of its parent ("
7597 << Arg->getParent()->getName()
7598 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7599 "direct call of ("
7600 << ACS.getInstruction()->getCalledOperand()->getName()
7601 << ").\n[AAPrivatizablePtr] for which the argument "
7602 "privatization is not compatible.\n";
7603 });
7604 return false;
7605 };
7606
7607 // Helper to check if the associated argument is used at the given abstract
7608 // call site in a way that is incompatible with the privatization assumed
7609 // here.
7610 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7611 if (ACS.isDirectCall())
7612 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7613 if (ACS.isCallbackCall())
7614 return IsCompatiblePrivArgOfDirectCS(ACS);
7615 return false;
7616 };
7617
7618 if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7619 UsedAssumedInformation))
7620 return indicatePessimisticFixpoint();
7621
7622 return ChangeStatus::UNCHANGED;
7623 }
7624
7625 /// Given a type to private \p PrivType, collect the constituates (which are
7626 /// used) in \p ReplacementTypes.
7627 static void
7628 identifyReplacementTypes(Type *PrivType,
7629 SmallVectorImpl<Type *> &ReplacementTypes) {
7630 // TODO: For now we expand the privatization type to the fullest which can
7631 // lead to dead arguments that need to be removed later.
7632 assert(PrivType && "Expected privatizable type!");
7633
7634 // Traverse the type, extract constituate types on the outermost level.
7635 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7636 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7637 ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7638 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7639 ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7640 Elt: PrivArrayType->getElementType());
7641 } else {
7642 ReplacementTypes.push_back(Elt: PrivType);
7643 }
7644 }
7645
7646 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7647 /// The values needed are taken from the arguments of \p F starting at
7648 /// position \p ArgNo.
7649 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7650 unsigned ArgNo, BasicBlock::iterator IP) {
7651 assert(PrivType && "Expected privatizable type!");
7652
7653 IRBuilder<NoFolder> IRB(IP->getParent(), IP);
7654 const DataLayout &DL = F.getDataLayout();
7655
7656 // Traverse the type, build GEPs and stores.
7657 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7658 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7659 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7660 Value *Ptr =
7661 constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7662 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7663 }
7664 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7665 Type *PointeeTy = PrivArrayType->getElementType();
7666 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7667 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7668 Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7669 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7670 }
7671 } else {
7672 new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7673 }
7674 }
7675
7676 /// Extract values from \p Base according to the type \p PrivType at the
7677 /// call position \p ACS. The values are appended to \p ReplacementValues.
7678 void createReplacementValues(Align Alignment, Type *PrivType,
7679 AbstractCallSite ACS, Value *Base,
7680 SmallVectorImpl<Value *> &ReplacementValues) {
7681 assert(Base && "Expected base value!");
7682 assert(PrivType && "Expected privatizable type!");
7683 Instruction *IP = ACS.getInstruction();
7684
7685 IRBuilder<NoFolder> IRB(IP);
7686 const DataLayout &DL = IP->getDataLayout();
7687
7688 // Traverse the type, build GEPs and loads.
7689 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7690 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7691 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7692 Type *PointeeTy = PrivStructType->getElementType(N: u);
7693 Value *Ptr =
7694 constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7695 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7696 L->setAlignment(Alignment);
7697 ReplacementValues.push_back(Elt: L);
7698 }
7699 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7700 Type *PointeeTy = PrivArrayType->getElementType();
7701 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7702 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7703 Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7704 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7705 L->setAlignment(Alignment);
7706 ReplacementValues.push_back(Elt: L);
7707 }
7708 } else {
7709 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7710 L->setAlignment(Alignment);
7711 ReplacementValues.push_back(Elt: L);
7712 }
7713 }
7714
7715 /// See AbstractAttribute::manifest(...)
7716 ChangeStatus manifest(Attributor &A) override {
7717 if (!PrivatizableType)
7718 return ChangeStatus::UNCHANGED;
7719 assert(*PrivatizableType && "Expected privatizable type!");
7720
7721 // Collect all tail calls in the function as we cannot allow new allocas to
7722 // escape into tail recursion.
7723 // TODO: Be smarter about new allocas escaping into tail calls.
7724 SmallVector<CallInst *, 16> TailCalls;
7725 bool UsedAssumedInformation = false;
7726 if (!A.checkForAllInstructions(
7727 Pred: [&](Instruction &I) {
7728 CallInst &CI = cast<CallInst>(Val&: I);
7729 if (CI.isTailCall())
7730 TailCalls.push_back(Elt: &CI);
7731 return true;
7732 },
7733 QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation))
7734 return ChangeStatus::UNCHANGED;
7735
7736 Argument *Arg = getAssociatedArgument();
7737 // Query AAAlign attribute for alignment of associated argument to
7738 // determine the best alignment of loads.
7739 const auto *AlignAA =
7740 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7741
7742 // Callback to repair the associated function. A new alloca is placed at the
7743 // beginning and initialized with the values passed through arguments. The
7744 // new alloca replaces the use of the old pointer argument.
7745 Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7746 [=](const Attributor::ArgumentReplacementInfo &ARI,
7747 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7748 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7749 BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7750 const DataLayout &DL = IP->getDataLayout();
7751 unsigned AS = DL.getAllocaAddrSpace();
7752 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7753 Arg->getName() + ".priv", IP);
7754 createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7755 ArgNo: ArgIt->getArgNo(), IP);
7756
7757 if (AI->getType() != Arg->getType())
7758 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7759 S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7760 Arg->replaceAllUsesWith(V: AI);
7761
7762 for (CallInst *CI : TailCalls)
7763 CI->setTailCall(false);
7764 };
7765
7766 // Callback to repair a call site of the associated function. The elements
7767 // of the privatizable type are loaded prior to the call and passed to the
7768 // new function version.
7769 Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7770 [=](const Attributor::ArgumentReplacementInfo &ARI,
7771 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7772 // When no alignment is specified for the load instruction,
7773 // natural alignment is assumed.
7774 createReplacementValues(
7775 Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7776 PrivType: *PrivatizableType, ACS,
7777 Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7778 ReplacementValues&: NewArgOperands);
7779 };
7780
7781 // Collect the types that will replace the privatizable type in the function
7782 // signature.
7783 SmallVector<Type *, 16> ReplacementTypes;
7784 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7785
7786 // Register a rewrite of the argument.
7787 if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7788 CalleeRepairCB: std::move(FnRepairCB),
7789 ACSRepairCB: std::move(ACSRepairCB)))
7790 return ChangeStatus::CHANGED;
7791 return ChangeStatus::UNCHANGED;
7792 }
7793
7794 /// See AbstractAttribute::trackStatistics()
7795 void trackStatistics() const override {
7796 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7797 }
7798};
7799
7800struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7801 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7802 : AAPrivatizablePtrImpl(IRP, A) {}
7803
7804 /// See AbstractAttribute::initialize(...).
7805 void initialize(Attributor &A) override {
7806 // TODO: We can privatize more than arguments.
7807 indicatePessimisticFixpoint();
7808 }
7809
7810 ChangeStatus updateImpl(Attributor &A) override {
7811 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7812 "updateImpl will not be called");
7813 }
7814
7815 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7816 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7817 Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7818 if (!Obj) {
7819 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7820 return nullptr;
7821 }
7822
7823 if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7824 if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7825 if (CI->isOne())
7826 return AI->getAllocatedType();
7827 if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7828 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7829 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7830 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7831 return PrivArgAA->getPrivatizableType();
7832 }
7833
7834 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7835 "alloca nor privatizable argument: "
7836 << *Obj << "!\n");
7837 return nullptr;
7838 }
7839
7840 /// See AbstractAttribute::trackStatistics()
7841 void trackStatistics() const override {
7842 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7843 }
7844};
7845
7846struct AAPrivatizablePtrCallSiteArgument final
7847 : public AAPrivatizablePtrFloating {
7848 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7849 : AAPrivatizablePtrFloating(IRP, A) {}
7850
7851 /// See AbstractAttribute::initialize(...).
7852 void initialize(Attributor &A) override {
7853 if (A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal))
7854 indicateOptimisticFixpoint();
7855 }
7856
7857 /// See AbstractAttribute::updateImpl(...).
7858 ChangeStatus updateImpl(Attributor &A) override {
7859 PrivatizableType = identifyPrivatizableType(A);
7860 if (!PrivatizableType)
7861 return ChangeStatus::UNCHANGED;
7862 if (!*PrivatizableType)
7863 return indicatePessimisticFixpoint();
7864
7865 const IRPosition &IRP = getIRPosition();
7866 bool IsKnownNoCapture;
7867 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7868 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture);
7869 if (!IsAssumedNoCapture) {
7870 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7871 return indicatePessimisticFixpoint();
7872 }
7873
7874 bool IsKnownNoAlias;
7875 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7876 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
7877 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7878 return indicatePessimisticFixpoint();
7879 }
7880
7881 bool IsKnown;
7882 if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7883 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7884 return indicatePessimisticFixpoint();
7885 }
7886
7887 return ChangeStatus::UNCHANGED;
7888 }
7889
7890 /// See AbstractAttribute::trackStatistics()
7891 void trackStatistics() const override {
7892 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7893 }
7894};
7895
7896struct AAPrivatizablePtrCallSiteReturned final
7897 : public AAPrivatizablePtrFloating {
7898 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7899 : AAPrivatizablePtrFloating(IRP, A) {}
7900
7901 /// See AbstractAttribute::initialize(...).
7902 void initialize(Attributor &A) override {
7903 // TODO: We can privatize more than arguments.
7904 indicatePessimisticFixpoint();
7905 }
7906
7907 /// See AbstractAttribute::trackStatistics()
7908 void trackStatistics() const override {
7909 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7910 }
7911};
7912
7913struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7914 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7915 : AAPrivatizablePtrFloating(IRP, A) {}
7916
7917 /// See AbstractAttribute::initialize(...).
7918 void initialize(Attributor &A) override {
7919 // TODO: We can privatize more than arguments.
7920 indicatePessimisticFixpoint();
7921 }
7922
7923 /// See AbstractAttribute::trackStatistics()
7924 void trackStatistics() const override {
7925 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7926 }
7927};
7928} // namespace
7929
7930/// -------------------- Memory Behavior Attributes ----------------------------
7931/// Includes read-none, read-only, and write-only.
7932/// ----------------------------------------------------------------------------
7933namespace {
7934struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7935 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7936 : AAMemoryBehavior(IRP, A) {}
7937
7938 /// See AbstractAttribute::initialize(...).
7939 void initialize(Attributor &A) override {
7940 intersectAssumedBits(BitsEncoding: BEST_STATE);
7941 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7942 AAMemoryBehavior::initialize(A);
7943 }
7944
7945 /// Return the memory behavior information encoded in the IR for \p IRP.
7946 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7947 BitIntegerState &State,
7948 bool IgnoreSubsumingPositions = false) {
7949 SmallVector<Attribute, 2> Attrs;
7950 A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7951 for (const Attribute &Attr : Attrs) {
7952 switch (Attr.getKindAsEnum()) {
7953 case Attribute::ReadNone:
7954 State.addKnownBits(Bits: NO_ACCESSES);
7955 break;
7956 case Attribute::ReadOnly:
7957 State.addKnownBits(Bits: NO_WRITES);
7958 break;
7959 case Attribute::WriteOnly:
7960 State.addKnownBits(Bits: NO_READS);
7961 break;
7962 default:
7963 llvm_unreachable("Unexpected attribute!");
7964 }
7965 }
7966
7967 if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) {
7968 if (!I->mayReadFromMemory())
7969 State.addKnownBits(Bits: NO_READS);
7970 if (!I->mayWriteToMemory())
7971 State.addKnownBits(Bits: NO_WRITES);
7972 }
7973 }
7974
7975 /// See AbstractAttribute::getDeducedAttributes(...).
7976 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7977 SmallVectorImpl<Attribute> &Attrs) const override {
7978 assert(Attrs.size() == 0);
7979 if (isAssumedReadNone())
7980 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadNone));
7981 else if (isAssumedReadOnly())
7982 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadOnly));
7983 else if (isAssumedWriteOnly())
7984 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::WriteOnly));
7985 assert(Attrs.size() <= 1);
7986 }
7987
7988 /// See AbstractAttribute::manifest(...).
7989 ChangeStatus manifest(Attributor &A) override {
7990 const IRPosition &IRP = getIRPosition();
7991
7992 if (A.hasAttr(IRP, AKs: Attribute::ReadNone,
7993 /* IgnoreSubsumingPositions */ true))
7994 return ChangeStatus::UNCHANGED;
7995
7996 // Check if we would improve the existing attributes first.
7997 SmallVector<Attribute, 4> DeducedAttrs;
7998 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
7999 if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
8000 return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
8001 /* IgnoreSubsumingPositions */ true);
8002 }))
8003 return ChangeStatus::UNCHANGED;
8004
8005 // Clear existing attributes.
8006 A.removeAttrs(IRP, AttrKinds);
8007 // Clear conflicting writable attribute.
8008 if (isAssumedReadOnly())
8009 A.removeAttrs(IRP, AttrKinds: Attribute::Writable);
8010
8011 // Use the generic manifest method.
8012 return IRAttribute::manifest(A);
8013 }
8014
8015 /// See AbstractState::getAsStr().
8016 const std::string getAsStr(Attributor *A) const override {
8017 if (isAssumedReadNone())
8018 return "readnone";
8019 if (isAssumedReadOnly())
8020 return "readonly";
8021 if (isAssumedWriteOnly())
8022 return "writeonly";
8023 return "may-read/write";
8024 }
8025
8026 /// The set of IR attributes AAMemoryBehavior deals with.
8027 static const Attribute::AttrKind AttrKinds[3];
8028};
8029
8030const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
8031 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
8032
8033/// Memory behavior attribute for a floating value.
8034struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
8035 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
8036 : AAMemoryBehaviorImpl(IRP, A) {}
8037
8038 /// See AbstractAttribute::updateImpl(...).
8039 ChangeStatus updateImpl(Attributor &A) override;
8040
8041 /// See AbstractAttribute::trackStatistics()
8042 void trackStatistics() const override {
8043 if (isAssumedReadNone())
8044 STATS_DECLTRACK_FLOATING_ATTR(readnone)
8045 else if (isAssumedReadOnly())
8046 STATS_DECLTRACK_FLOATING_ATTR(readonly)
8047 else if (isAssumedWriteOnly())
8048 STATS_DECLTRACK_FLOATING_ATTR(writeonly)
8049 }
8050
8051private:
8052 /// Return true if users of \p UserI might access the underlying
8053 /// variable/location described by \p U and should therefore be analyzed.
8054 bool followUsersOfUseIn(Attributor &A, const Use &U,
8055 const Instruction *UserI);
8056
8057 /// Update the state according to the effect of use \p U in \p UserI.
8058 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
8059};
8060
8061/// Memory behavior attribute for function argument.
8062struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
8063 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
8064 : AAMemoryBehaviorFloating(IRP, A) {}
8065
8066 /// See AbstractAttribute::initialize(...).
8067 void initialize(Attributor &A) override {
8068 intersectAssumedBits(BitsEncoding: BEST_STATE);
8069 const IRPosition &IRP = getIRPosition();
8070 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
8071 // can query it when we use has/getAttr. That would allow us to reuse the
8072 // initialize of the base class here.
8073 bool HasByVal = A.hasAttr(IRP, AKs: {Attribute::ByVal},
8074 /* IgnoreSubsumingPositions */ true);
8075 getKnownStateFromValue(A, IRP, State&: getState(),
8076 /* IgnoreSubsumingPositions */ HasByVal);
8077 }
8078
8079 ChangeStatus manifest(Attributor &A) override {
8080 // TODO: Pointer arguments are not supported on vectors of pointers yet.
8081 if (!getAssociatedValue().getType()->isPointerTy())
8082 return ChangeStatus::UNCHANGED;
8083
8084 // TODO: From readattrs.ll: "inalloca parameters are always
8085 // considered written"
8086 if (A.hasAttr(IRP: getIRPosition(),
8087 AKs: {Attribute::InAlloca, Attribute::Preallocated})) {
8088 removeKnownBits(BitsEncoding: NO_WRITES);
8089 removeAssumedBits(BitsEncoding: NO_WRITES);
8090 }
8091 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8092 return AAMemoryBehaviorFloating::manifest(A);
8093 }
8094
8095 /// See AbstractAttribute::trackStatistics()
8096 void trackStatistics() const override {
8097 if (isAssumedReadNone())
8098 STATS_DECLTRACK_ARG_ATTR(readnone)
8099 else if (isAssumedReadOnly())
8100 STATS_DECLTRACK_ARG_ATTR(readonly)
8101 else if (isAssumedWriteOnly())
8102 STATS_DECLTRACK_ARG_ATTR(writeonly)
8103 }
8104};
8105
8106struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
8107 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
8108 : AAMemoryBehaviorArgument(IRP, A) {}
8109
8110 /// See AbstractAttribute::initialize(...).
8111 void initialize(Attributor &A) override {
8112 // If we don't have an associated attribute this is either a variadic call
8113 // or an indirect call, either way, nothing to do here.
8114 Argument *Arg = getAssociatedArgument();
8115 if (!Arg) {
8116 indicatePessimisticFixpoint();
8117 return;
8118 }
8119 if (Arg->hasByValAttr()) {
8120 addKnownBits(Bits: NO_WRITES);
8121 removeKnownBits(BitsEncoding: NO_READS);
8122 removeAssumedBits(BitsEncoding: NO_READS);
8123 }
8124 AAMemoryBehaviorArgument::initialize(A);
8125 if (getAssociatedFunction()->isDeclaration())
8126 indicatePessimisticFixpoint();
8127 }
8128
8129 /// See AbstractAttribute::updateImpl(...).
8130 ChangeStatus updateImpl(Attributor &A) override {
8131 // TODO: Once we have call site specific value information we can provide
8132 // call site specific liveness liveness information and then it makes
8133 // sense to specialize attributes for call sites arguments instead of
8134 // redirecting requests to the callee argument.
8135 Argument *Arg = getAssociatedArgument();
8136 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
8137 auto *ArgAA =
8138 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
8139 if (!ArgAA)
8140 return indicatePessimisticFixpoint();
8141 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
8142 }
8143
8144 /// See AbstractAttribute::trackStatistics()
8145 void trackStatistics() const override {
8146 if (isAssumedReadNone())
8147 STATS_DECLTRACK_CSARG_ATTR(readnone)
8148 else if (isAssumedReadOnly())
8149 STATS_DECLTRACK_CSARG_ATTR(readonly)
8150 else if (isAssumedWriteOnly())
8151 STATS_DECLTRACK_CSARG_ATTR(writeonly)
8152 }
8153};
8154
8155/// Memory behavior attribute for a call site return position.
8156struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
8157 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
8158 : AAMemoryBehaviorFloating(IRP, A) {}
8159
8160 /// See AbstractAttribute::initialize(...).
8161 void initialize(Attributor &A) override {
8162 AAMemoryBehaviorImpl::initialize(A);
8163 }
8164 /// See AbstractAttribute::manifest(...).
8165 ChangeStatus manifest(Attributor &A) override {
8166 // We do not annotate returned values.
8167 return ChangeStatus::UNCHANGED;
8168 }
8169
8170 /// See AbstractAttribute::trackStatistics()
8171 void trackStatistics() const override {}
8172};
8173
8174/// An AA to represent the memory behavior function attributes.
8175struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
8176 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
8177 : AAMemoryBehaviorImpl(IRP, A) {}
8178
8179 /// See AbstractAttribute::updateImpl(Attributor &A).
8180 ChangeStatus updateImpl(Attributor &A) override;
8181
8182 /// See AbstractAttribute::manifest(...).
8183 ChangeStatus manifest(Attributor &A) override {
8184 // TODO: It would be better to merge this with AAMemoryLocation, so that
8185 // we could determine read/write per location. This would also have the
8186 // benefit of only one place trying to manifest the memory attribute.
8187 Function &F = cast<Function>(Val&: getAnchorValue());
8188 MemoryEffects ME = MemoryEffects::unknown();
8189 if (isAssumedReadNone())
8190 ME = MemoryEffects::none();
8191 else if (isAssumedReadOnly())
8192 ME = MemoryEffects::readOnly();
8193 else if (isAssumedWriteOnly())
8194 ME = MemoryEffects::writeOnly();
8195
8196 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8197 // Clear conflicting writable attribute.
8198 if (ME.onlyReadsMemory())
8199 for (Argument &Arg : F.args())
8200 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8201 return A.manifestAttrs(IRP: getIRPosition(),
8202 DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8203 }
8204
8205 /// See AbstractAttribute::trackStatistics()
8206 void trackStatistics() const override {
8207 if (isAssumedReadNone())
8208 STATS_DECLTRACK_FN_ATTR(readnone)
8209 else if (isAssumedReadOnly())
8210 STATS_DECLTRACK_FN_ATTR(readonly)
8211 else if (isAssumedWriteOnly())
8212 STATS_DECLTRACK_FN_ATTR(writeonly)
8213 }
8214};
8215
8216/// AAMemoryBehavior attribute for call sites.
8217struct AAMemoryBehaviorCallSite final
8218 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8219 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8220 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8221
8222 /// See AbstractAttribute::manifest(...).
8223 ChangeStatus manifest(Attributor &A) override {
8224 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8225 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8226 MemoryEffects ME = MemoryEffects::unknown();
8227 if (isAssumedReadNone())
8228 ME = MemoryEffects::none();
8229 else if (isAssumedReadOnly())
8230 ME = MemoryEffects::readOnly();
8231 else if (isAssumedWriteOnly())
8232 ME = MemoryEffects::writeOnly();
8233
8234 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8235 // Clear conflicting writable attribute.
8236 if (ME.onlyReadsMemory())
8237 for (Use &U : CB.args())
8238 A.removeAttrs(IRP: IRPosition::callsite_argument(CB, ArgNo: U.getOperandNo()),
8239 AttrKinds: Attribute::Writable);
8240 return A.manifestAttrs(
8241 IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8242 }
8243
8244 /// See AbstractAttribute::trackStatistics()
8245 void trackStatistics() const override {
8246 if (isAssumedReadNone())
8247 STATS_DECLTRACK_CS_ATTR(readnone)
8248 else if (isAssumedReadOnly())
8249 STATS_DECLTRACK_CS_ATTR(readonly)
8250 else if (isAssumedWriteOnly())
8251 STATS_DECLTRACK_CS_ATTR(writeonly)
8252 }
8253};
8254
8255ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8256
8257 // The current assumed state used to determine a change.
8258 auto AssumedState = getAssumed();
8259
8260 auto CheckRWInst = [&](Instruction &I) {
8261 // If the instruction has an own memory behavior state, use it to restrict
8262 // the local state. No further analysis is required as the other memory
8263 // state is as optimistic as it gets.
8264 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8265 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8266 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8267 if (MemBehaviorAA) {
8268 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8269 return !isAtFixpoint();
8270 }
8271 }
8272
8273 // Remove access kind modifiers if necessary.
8274 if (I.mayReadFromMemory())
8275 removeAssumedBits(BitsEncoding: NO_READS);
8276 if (I.mayWriteToMemory())
8277 removeAssumedBits(BitsEncoding: NO_WRITES);
8278 return !isAtFixpoint();
8279 };
8280
8281 bool UsedAssumedInformation = false;
8282 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8283 UsedAssumedInformation))
8284 return indicatePessimisticFixpoint();
8285
8286 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8287 : ChangeStatus::UNCHANGED;
8288}
8289
8290ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8291
8292 const IRPosition &IRP = getIRPosition();
8293 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8294 AAMemoryBehavior::StateType &S = getState();
8295
8296 // First, check the function scope. We take the known information and we avoid
8297 // work if the assumed information implies the current assumed information for
8298 // this attribute. This is a valid for all but byval arguments.
8299 Argument *Arg = IRP.getAssociatedArgument();
8300 AAMemoryBehavior::base_t FnMemAssumedState =
8301 AAMemoryBehavior::StateType::getWorstState();
8302 if (!Arg || !Arg->hasByValAttr()) {
8303 const auto *FnMemAA =
8304 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8305 if (FnMemAA) {
8306 FnMemAssumedState = FnMemAA->getAssumed();
8307 S.addKnownBits(Bits: FnMemAA->getKnown());
8308 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8309 return ChangeStatus::UNCHANGED;
8310 }
8311 }
8312
8313 // The current assumed state used to determine a change.
8314 auto AssumedState = S.getAssumed();
8315
8316 // Make sure the value is not captured (except through "return"), if
8317 // it is, any information derived would be irrelevant anyway as we cannot
8318 // check the potential aliases introduced by the capture. However, no need
8319 // to fall back to anythign less optimistic than the function state.
8320 bool IsKnownNoCapture;
8321 const AANoCapture *ArgNoCaptureAA = nullptr;
8322 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8323 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
8324 AAPtr: &ArgNoCaptureAA);
8325
8326 if (!IsAssumedNoCapture &&
8327 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8328 S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8329 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8330 : ChangeStatus::UNCHANGED;
8331 }
8332
8333 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8334 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8335 Instruction *UserI = cast<Instruction>(Val: U.getUser());
8336 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8337 << " \n");
8338
8339 // Droppable users, e.g., llvm::assume does not actually perform any action.
8340 if (UserI->isDroppable())
8341 return true;
8342
8343 // Check if the users of UserI should also be visited.
8344 Follow = followUsersOfUseIn(A, U, UserI);
8345
8346 // If UserI might touch memory we analyze the use in detail.
8347 if (UserI->mayReadOrWriteMemory())
8348 analyzeUseIn(A, U, UserI);
8349
8350 return !isAtFixpoint();
8351 };
8352
8353 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8354 return indicatePessimisticFixpoint();
8355
8356 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8357 : ChangeStatus::UNCHANGED;
8358}
8359
8360bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8361 const Instruction *UserI) {
8362 // The loaded value is unrelated to the pointer argument, no need to
8363 // follow the users of the load.
8364 if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8365 return false;
8366
8367 // By default we follow all uses assuming UserI might leak information on U,
8368 // we have special handling for call sites operands though.
8369 const auto *CB = dyn_cast<CallBase>(Val: UserI);
8370 if (!CB || !CB->isArgOperand(U: &U))
8371 return true;
8372
8373 // If the use is a call argument known not to be captured, the users of
8374 // the call do not need to be visited because they have to be unrelated to
8375 // the input. Note that this check is not trivial even though we disallow
8376 // general capturing of the underlying argument. The reason is that the
8377 // call might the argument "through return", which we allow and for which we
8378 // need to check call users.
8379 if (U.get()->getType()->isPointerTy()) {
8380 unsigned ArgNo = CB->getArgOperandNo(U: &U);
8381 bool IsKnownNoCapture;
8382 return !AA::hasAssumedIRAttr<Attribute::Captures>(
8383 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
8384 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
8385 }
8386
8387 return true;
8388}
8389
8390void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8391 const Instruction *UserI) {
8392 assert(UserI->mayReadOrWriteMemory());
8393
8394 switch (UserI->getOpcode()) {
8395 default:
8396 // TODO: Handle all atomics and other side-effect operations we know of.
8397 break;
8398 case Instruction::Load:
8399 // Loads cause the NO_READS property to disappear.
8400 removeAssumedBits(BitsEncoding: NO_READS);
8401 return;
8402
8403 case Instruction::Store:
8404 // Stores cause the NO_WRITES property to disappear if the use is the
8405 // pointer operand. Note that while capturing was taken care of somewhere
8406 // else we need to deal with stores of the value that is not looked through.
8407 if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8408 removeAssumedBits(BitsEncoding: NO_WRITES);
8409 else
8410 indicatePessimisticFixpoint();
8411 return;
8412
8413 case Instruction::Call:
8414 case Instruction::CallBr:
8415 case Instruction::Invoke: {
8416 // For call sites we look at the argument memory behavior attribute (this
8417 // could be recursive!) in order to restrict our own state.
8418 const auto *CB = cast<CallBase>(Val: UserI);
8419
8420 // Give up on operand bundles.
8421 if (CB->isBundleOperand(U: &U)) {
8422 indicatePessimisticFixpoint();
8423 return;
8424 }
8425
8426 // Calling a function does read the function pointer, maybe write it if the
8427 // function is self-modifying.
8428 if (CB->isCallee(U: &U)) {
8429 removeAssumedBits(BitsEncoding: NO_READS);
8430 break;
8431 }
8432
8433 // Adjust the possible access behavior based on the information on the
8434 // argument.
8435 IRPosition Pos;
8436 if (U.get()->getType()->isPointerTy())
8437 Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8438 else
8439 Pos = IRPosition::callsite_function(CB: *CB);
8440 const auto *MemBehaviorAA =
8441 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8442 if (!MemBehaviorAA)
8443 break;
8444 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8445 // and at least "known".
8446 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8447 return;
8448 }
8449 };
8450
8451 // Generally, look at the "may-properties" and adjust the assumed state if we
8452 // did not trigger special handling before.
8453 if (UserI->mayReadFromMemory())
8454 removeAssumedBits(BitsEncoding: NO_READS);
8455 if (UserI->mayWriteToMemory())
8456 removeAssumedBits(BitsEncoding: NO_WRITES);
8457}
8458} // namespace
8459
8460/// -------------------- Memory Locations Attributes ---------------------------
8461/// Includes read-none, argmemonly, inaccessiblememonly,
8462/// inaccessiblememorargmemonly
8463/// ----------------------------------------------------------------------------
8464
8465std::string AAMemoryLocation::getMemoryLocationsAsStr(
8466 AAMemoryLocation::MemoryLocationsKind MLK) {
8467 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8468 return "all memory";
8469 if (MLK == AAMemoryLocation::NO_LOCATIONS)
8470 return "no memory";
8471 std::string S = "memory:";
8472 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8473 S += "stack,";
8474 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8475 S += "constant,";
8476 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8477 S += "internal global,";
8478 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8479 S += "external global,";
8480 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8481 S += "argument,";
8482 if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8483 S += "inaccessible,";
8484 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8485 S += "malloced,";
8486 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8487 S += "unknown,";
8488 S.pop_back();
8489 return S;
8490}
8491
8492namespace {
8493struct AAMemoryLocationImpl : public AAMemoryLocation {
8494
8495 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8496 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8497 AccessKind2Accesses.fill(u: nullptr);
8498 }
8499
8500 ~AAMemoryLocationImpl() override {
8501 // The AccessSets are allocated via a BumpPtrAllocator, we call
8502 // the destructor manually.
8503 for (AccessSet *AS : AccessKind2Accesses)
8504 if (AS)
8505 AS->~AccessSet();
8506 }
8507
8508 /// See AbstractAttribute::initialize(...).
8509 void initialize(Attributor &A) override {
8510 intersectAssumedBits(BitsEncoding: BEST_STATE);
8511 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8512 AAMemoryLocation::initialize(A);
8513 }
8514
8515 /// Return the memory behavior information encoded in the IR for \p IRP.
8516 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8517 BitIntegerState &State,
8518 bool IgnoreSubsumingPositions = false) {
8519 // For internal functions we ignore `argmemonly` and
8520 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8521 // constant propagation. It is unclear if this is the best way but it is
8522 // unlikely this will cause real performance problems. If we are deriving
8523 // attributes for the anchor function we even remove the attribute in
8524 // addition to ignoring it.
8525 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8526 // MemoryEffects::Other as a possible location.
8527 bool UseArgMemOnly = true;
8528 Function *AnchorFn = IRP.getAnchorScope();
8529 if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8530 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8531
8532 SmallVector<Attribute, 2> Attrs;
8533 A.getAttrs(IRP, AKs: {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8534 for (const Attribute &Attr : Attrs) {
8535 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8536 MemoryEffects ME = Attr.getMemoryEffects();
8537 if (ME.doesNotAccessMemory()) {
8538 State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8539 continue;
8540 }
8541 if (ME.onlyAccessesInaccessibleMem()) {
8542 State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8543 continue;
8544 }
8545 if (ME.onlyAccessesArgPointees()) {
8546 if (UseArgMemOnly)
8547 State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8548 else {
8549 // Remove location information, only keep read/write info.
8550 ME = MemoryEffects(ME.getModRef());
8551 A.manifestAttrs(IRP,
8552 DeducedAttrs: Attribute::getWithMemoryEffects(
8553 Context&: IRP.getAnchorValue().getContext(), ME),
8554 /*ForceReplace*/ true);
8555 }
8556 continue;
8557 }
8558 if (ME.onlyAccessesInaccessibleOrArgMem()) {
8559 if (UseArgMemOnly)
8560 State.addKnownBits(Bits: inverseLocation(
8561 Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8562 else {
8563 // Remove location information, only keep read/write info.
8564 ME = MemoryEffects(ME.getModRef());
8565 A.manifestAttrs(IRP,
8566 DeducedAttrs: Attribute::getWithMemoryEffects(
8567 Context&: IRP.getAnchorValue().getContext(), ME),
8568 /*ForceReplace*/ true);
8569 }
8570 continue;
8571 }
8572 }
8573 }
8574
8575 /// See AbstractAttribute::getDeducedAttributes(...).
8576 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8577 SmallVectorImpl<Attribute> &Attrs) const override {
8578 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8579 assert(Attrs.size() == 0);
8580 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8581 if (isAssumedReadNone())
8582 Attrs.push_back(
8583 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8584 else if (isAssumedInaccessibleMemOnly())
8585 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8586 Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8587 else if (isAssumedArgMemOnly())
8588 Attrs.push_back(
8589 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8590 else if (isAssumedInaccessibleOrArgMemOnly())
8591 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8592 Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8593 }
8594 assert(Attrs.size() <= 1);
8595 }
8596
8597 /// See AbstractAttribute::manifest(...).
8598 ChangeStatus manifest(Attributor &A) override {
8599 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8600 // provide per-location modref information here.
8601 const IRPosition &IRP = getIRPosition();
8602
8603 SmallVector<Attribute, 1> DeducedAttrs;
8604 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8605 if (DeducedAttrs.size() != 1)
8606 return ChangeStatus::UNCHANGED;
8607 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8608
8609 return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8610 Context&: IRP.getAnchorValue().getContext(), ME));
8611 }
8612
8613 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8614 bool checkForAllAccessesToMemoryKind(
8615 function_ref<bool(const Instruction *, const Value *, AccessKind,
8616 MemoryLocationsKind)>
8617 Pred,
8618 MemoryLocationsKind RequestedMLK) const override {
8619 if (!isValidState())
8620 return false;
8621
8622 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8623 if (AssumedMLK == NO_LOCATIONS)
8624 return true;
8625
8626 unsigned Idx = 0;
8627 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8628 CurMLK *= 2, ++Idx) {
8629 if (CurMLK & RequestedMLK)
8630 continue;
8631
8632 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8633 for (const AccessInfo &AI : *Accesses)
8634 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8635 return false;
8636 }
8637
8638 return true;
8639 }
8640
8641 ChangeStatus indicatePessimisticFixpoint() override {
8642 // If we give up and indicate a pessimistic fixpoint this instruction will
8643 // become an access for all potential access kinds:
8644 // TODO: Add pointers for argmemonly and globals to improve the results of
8645 // checkForAllAccessesToMemoryKind.
8646 bool Changed = false;
8647 MemoryLocationsKind KnownMLK = getKnown();
8648 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8649 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8650 if (!(CurMLK & KnownMLK))
8651 updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8652 AK: getAccessKindFromInst(I));
8653 return AAMemoryLocation::indicatePessimisticFixpoint();
8654 }
8655
8656protected:
8657 /// Helper struct to tie together an instruction that has a read or write
8658 /// effect with the pointer it accesses (if any).
8659 struct AccessInfo {
8660
8661 /// The instruction that caused the access.
8662 const Instruction *I;
8663
8664 /// The base pointer that is accessed, or null if unknown.
8665 const Value *Ptr;
8666
8667 /// The kind of access (read/write/read+write).
8668 AccessKind Kind;
8669
8670 bool operator==(const AccessInfo &RHS) const {
8671 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8672 }
8673 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8674 if (LHS.I != RHS.I)
8675 return LHS.I < RHS.I;
8676 if (LHS.Ptr != RHS.Ptr)
8677 return LHS.Ptr < RHS.Ptr;
8678 if (LHS.Kind != RHS.Kind)
8679 return LHS.Kind < RHS.Kind;
8680 return false;
8681 }
8682 };
8683
8684 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8685 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8686 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8687 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8688 AccessKind2Accesses;
8689
8690 /// Categorize the pointer arguments of CB that might access memory in
8691 /// AccessedLoc and update the state and access map accordingly.
8692 void
8693 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8694 AAMemoryLocation::StateType &AccessedLocs,
8695 bool &Changed);
8696
8697 /// Return the kind(s) of location that may be accessed by \p V.
8698 AAMemoryLocation::MemoryLocationsKind
8699 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8700
8701 /// Return the access kind as determined by \p I.
8702 AccessKind getAccessKindFromInst(const Instruction *I) {
8703 AccessKind AK = READ_WRITE;
8704 if (I) {
8705 AK = I->mayReadFromMemory() ? READ : NONE;
8706 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8707 }
8708 return AK;
8709 }
8710
8711 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8712 /// an access of kind \p AK to a \p MLK memory location with the access
8713 /// pointer \p Ptr.
8714 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8715 MemoryLocationsKind MLK, const Instruction *I,
8716 const Value *Ptr, bool &Changed,
8717 AccessKind AK = READ_WRITE) {
8718
8719 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8720 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8721 if (!Accesses)
8722 Accesses = new (Allocator) AccessSet();
8723 Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8724 if (MLK == NO_UNKOWN_MEM)
8725 MLK = NO_LOCATIONS;
8726 State.removeAssumedBits(BitsEncoding: MLK);
8727 }
8728
8729 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8730 /// arguments, and update the state and access map accordingly.
8731 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8732 AAMemoryLocation::StateType &State, bool &Changed,
8733 unsigned AccessAS = 0);
8734
8735 /// Used to allocate access sets.
8736 BumpPtrAllocator &Allocator;
8737};
8738
8739void AAMemoryLocationImpl::categorizePtrValue(
8740 Attributor &A, const Instruction &I, const Value &Ptr,
8741 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8742 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8743 << Ptr << " ["
8744 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8745
8746 auto Pred = [&](Value &Obj) {
8747 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8748 // TODO: recognize the TBAA used for constant accesses.
8749 MemoryLocationsKind MLK = NO_LOCATIONS;
8750
8751 // Filter accesses to constant (GPU) memory if we have an AS at the access
8752 // site or the object is known to actually have the associated AS.
8753 if (AA::isGPU(M: A.getModule())) {
8754 if (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: AccessAS) ||
8755 (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: ObjectAS) &&
8756 isIdentifiedObject(V: &Obj)))
8757 return true;
8758 }
8759
8760 if (isa<UndefValue>(Val: &Obj))
8761 return true;
8762 if (isa<Argument>(Val: &Obj)) {
8763 // TODO: For now we do not treat byval arguments as local copies performed
8764 // on the call edge, though, we should. To make that happen we need to
8765 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8766 // would also allow us to mark functions only accessing byval arguments as
8767 // readnone again, arguably their accesses have no effect outside of the
8768 // function, like accesses to allocas.
8769 MLK = NO_ARGUMENT_MEM;
8770 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8771 // Reading constant memory is not treated as a read "effect" by the
8772 // function attr pass so we won't neither. Constants defined by TBAA are
8773 // similar. (We know we do not write it because it is constant.)
8774 if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8775 if (GVar->isConstant())
8776 return true;
8777
8778 if (GV->hasLocalLinkage())
8779 MLK = NO_GLOBAL_INTERNAL_MEM;
8780 else
8781 MLK = NO_GLOBAL_EXTERNAL_MEM;
8782 } else if (isa<ConstantPointerNull>(Val: &Obj) &&
8783 (!NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS) ||
8784 !NullPointerIsDefined(F: getAssociatedFunction(), AS: ObjectAS))) {
8785 return true;
8786 } else if (isa<AllocaInst>(Val: &Obj)) {
8787 MLK = NO_LOCAL_MEM;
8788 } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8789 bool IsKnownNoAlias;
8790 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8791 A, QueryingAA: this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::OPTIONAL,
8792 IsKnown&: IsKnownNoAlias))
8793 MLK = NO_MALLOCED_MEM;
8794 else
8795 MLK = NO_UNKOWN_MEM;
8796 } else {
8797 MLK = NO_UNKOWN_MEM;
8798 }
8799
8800 assert(MLK != NO_LOCATIONS && "No location specified!");
8801 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8802 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8803 updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8804 AK: getAccessKindFromInst(I: &I));
8805
8806 return true;
8807 };
8808
8809 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8810 QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8811 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8812 LLVM_DEBUG(
8813 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8814 updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8815 AK: getAccessKindFromInst(I: &I));
8816 return;
8817 }
8818
8819 LLVM_DEBUG(
8820 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8821 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8822}
8823
8824void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8825 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8826 bool &Changed) {
8827 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8828
8829 // Skip non-pointer arguments.
8830 const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8831 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8832 continue;
8833
8834 // Skip readnone arguments.
8835 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8836 const auto *ArgOpMemLocationAA =
8837 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8838
8839 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8840 continue;
8841
8842 // Categorize potentially accessed pointer arguments as if there was an
8843 // access instruction with them as pointer.
8844 categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8845 }
8846}
8847
8848AAMemoryLocation::MemoryLocationsKind
8849AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8850 bool &Changed) {
8851 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8852 << I << "\n");
8853
8854 AAMemoryLocation::StateType AccessedLocs;
8855 AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8856
8857 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8858
8859 // First check if we assume any memory is access is visible.
8860 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8861 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8862 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8863 << " [" << CBMemLocationAA << "]\n");
8864 if (!CBMemLocationAA) {
8865 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8866 Changed, AK: getAccessKindFromInst(I: &I));
8867 return NO_UNKOWN_MEM;
8868 }
8869
8870 if (CBMemLocationAA->isAssumedReadNone())
8871 return NO_LOCATIONS;
8872
8873 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8874 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8875 Changed, AK: getAccessKindFromInst(I: &I));
8876 return AccessedLocs.getAssumed();
8877 }
8878
8879 uint32_t CBAssumedNotAccessedLocs =
8880 CBMemLocationAA->getAssumedNotAccessedLocation();
8881
8882 // Set the argmemonly and global bit as we handle them separately below.
8883 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8884 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8885
8886 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8887 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8888 continue;
8889 updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8890 AK: getAccessKindFromInst(I: &I));
8891 }
8892
8893 // Now handle global memory if it might be accessed. This is slightly tricky
8894 // as NO_GLOBAL_MEM has multiple bits set.
8895 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8896 if (HasGlobalAccesses) {
8897 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8898 AccessKind Kind, MemoryLocationsKind MLK) {
8899 updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8900 AK: getAccessKindFromInst(I: &I));
8901 return true;
8902 };
8903 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8904 Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8905 return AccessedLocs.getWorstState();
8906 }
8907
8908 LLVM_DEBUG(
8909 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8910 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8911
8912 // Now handle argument memory if it might be accessed.
8913 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8914 if (HasArgAccesses)
8915 categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8916
8917 LLVM_DEBUG(
8918 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8919 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8920
8921 return AccessedLocs.getAssumed();
8922 }
8923
8924 if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8925 LLVM_DEBUG(
8926 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8927 << I << " [" << *Ptr << "]\n");
8928 categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8929 AccessAS: Ptr->getType()->getPointerAddressSpace());
8930 return AccessedLocs.getAssumed();
8931 }
8932
8933 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8934 << I << "\n");
8935 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8936 AK: getAccessKindFromInst(I: &I));
8937 return AccessedLocs.getAssumed();
8938}
8939
8940/// An AA to represent the memory behavior function attributes.
8941struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8942 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8943 : AAMemoryLocationImpl(IRP, A) {}
8944
8945 /// See AbstractAttribute::updateImpl(Attributor &A).
8946 ChangeStatus updateImpl(Attributor &A) override {
8947
8948 const auto *MemBehaviorAA =
8949 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8950 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8951 if (MemBehaviorAA->isKnownReadNone())
8952 return indicateOptimisticFixpoint();
8953 assert(isAssumedReadNone() &&
8954 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8955 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8956 return ChangeStatus::UNCHANGED;
8957 }
8958
8959 // The current assumed state used to determine a change.
8960 auto AssumedState = getAssumed();
8961 bool Changed = false;
8962
8963 auto CheckRWInst = [&](Instruction &I) {
8964 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8965 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8966 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8967 removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8968 // Stop once only the valid bit set in the *not assumed location*, thus
8969 // once we don't actually exclude any memory locations in the state.
8970 return getAssumedNotAccessedLocation() != VALID_STATE;
8971 };
8972
8973 bool UsedAssumedInformation = false;
8974 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8975 UsedAssumedInformation))
8976 return indicatePessimisticFixpoint();
8977
8978 Changed |= AssumedState != getAssumed();
8979 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8980 }
8981
8982 /// See AbstractAttribute::trackStatistics()
8983 void trackStatistics() const override {
8984 if (isAssumedReadNone())
8985 STATS_DECLTRACK_FN_ATTR(readnone)
8986 else if (isAssumedArgMemOnly())
8987 STATS_DECLTRACK_FN_ATTR(argmemonly)
8988 else if (isAssumedInaccessibleMemOnly())
8989 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8990 else if (isAssumedInaccessibleOrArgMemOnly())
8991 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8992 }
8993};
8994
8995/// AAMemoryLocation attribute for call sites.
8996struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8997 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8998 : AAMemoryLocationImpl(IRP, A) {}
8999
9000 /// See AbstractAttribute::updateImpl(...).
9001 ChangeStatus updateImpl(Attributor &A) override {
9002 // TODO: Once we have call site specific value information we can provide
9003 // call site specific liveness liveness information and then it makes
9004 // sense to specialize attributes for call sites arguments instead of
9005 // redirecting requests to the callee argument.
9006 Function *F = getAssociatedFunction();
9007 const IRPosition &FnPos = IRPosition::function(F: *F);
9008 auto *FnAA =
9009 A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
9010 if (!FnAA)
9011 return indicatePessimisticFixpoint();
9012 bool Changed = false;
9013 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
9014 AccessKind Kind, MemoryLocationsKind MLK) {
9015 updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
9016 AK: getAccessKindFromInst(I));
9017 return true;
9018 };
9019 if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
9020 return indicatePessimisticFixpoint();
9021 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
9022 }
9023
9024 /// See AbstractAttribute::trackStatistics()
9025 void trackStatistics() const override {
9026 if (isAssumedReadNone())
9027 STATS_DECLTRACK_CS_ATTR(readnone)
9028 }
9029};
9030} // namespace
9031
9032/// ------------------ denormal-fp-math Attribute -------------------------
9033
9034namespace {
9035struct AADenormalFPMathImpl : public AADenormalFPMath {
9036 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
9037 : AADenormalFPMath(IRP, A) {}
9038
9039 const std::string getAsStr(Attributor *A) const override {
9040 std::string Str("AADenormalFPMath[");
9041 raw_string_ostream OS(Str);
9042
9043 DenormalState Known = getKnown();
9044 if (Known.Mode.isValid())
9045 OS << "denormal-fp-math=" << Known.Mode;
9046 else
9047 OS << "invalid";
9048
9049 if (Known.ModeF32.isValid())
9050 OS << " denormal-fp-math-f32=" << Known.ModeF32;
9051 OS << ']';
9052 return Str;
9053 }
9054};
9055
9056struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
9057 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
9058 : AADenormalFPMathImpl(IRP, A) {}
9059
9060 void initialize(Attributor &A) override {
9061 const Function *F = getAnchorScope();
9062 DenormalFPEnv DenormEnv = F->getDenormalFPEnv();
9063
9064 Known = DenormalState{.Mode: DenormEnv.DefaultMode, .ModeF32: DenormEnv.F32Mode};
9065 if (isModeFixed())
9066 indicateFixpoint();
9067 }
9068
9069 ChangeStatus updateImpl(Attributor &A) override {
9070 ChangeStatus Change = ChangeStatus::UNCHANGED;
9071
9072 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
9073 Function *Caller = CS.getInstruction()->getFunction();
9074 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
9075 << "->" << getAssociatedFunction()->getName() << '\n');
9076
9077 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
9078 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
9079 if (!CallerInfo)
9080 return false;
9081
9082 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
9083 R: CallerInfo->getState());
9084 return true;
9085 };
9086
9087 bool AllCallSitesKnown = true;
9088 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
9089 return indicatePessimisticFixpoint();
9090
9091 if (Change == ChangeStatus::CHANGED && isModeFixed())
9092 indicateFixpoint();
9093 return Change;
9094 }
9095
9096 ChangeStatus manifest(Attributor &A) override {
9097 LLVMContext &Ctx = getAssociatedFunction()->getContext();
9098
9099 SmallVector<Attribute, 2> AttrToAdd;
9100 SmallVector<Attribute::AttrKind, 2> AttrToRemove;
9101
9102 // TODO: Change to use DenormalFPEnv everywhere.
9103 DenormalFPEnv KnownEnv(Known.Mode, Known.ModeF32);
9104
9105 if (KnownEnv == DenormalFPEnv::getDefault()) {
9106 AttrToRemove.push_back(Elt: Attribute::DenormalFPEnv);
9107 } else {
9108 AttrToAdd.push_back(Elt: Attribute::get(
9109 Context&: Ctx, Kind: Attribute::DenormalFPEnv,
9110 Val: DenormalFPEnv(Known.Mode, Known.ModeF32).toIntValue()));
9111 }
9112
9113 auto &IRP = getIRPosition();
9114
9115 // TODO: There should be a combined add and remove API.
9116 return A.removeAttrs(IRP, AttrKinds: AttrToRemove) |
9117 A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
9118 }
9119
9120 void trackStatistics() const override {
9121 STATS_DECLTRACK_FN_ATTR(denormal_fpenv)
9122 }
9123};
9124} // namespace
9125
9126/// ------------------ Value Constant Range Attribute -------------------------
9127
9128namespace {
9129struct AAValueConstantRangeImpl : AAValueConstantRange {
9130 using StateType = IntegerRangeState;
9131 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
9132 : AAValueConstantRange(IRP, A) {}
9133
9134 /// See AbstractAttribute::initialize(..).
9135 void initialize(Attributor &A) override {
9136 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
9137 indicatePessimisticFixpoint();
9138 return;
9139 }
9140
9141 // Intersect a range given by SCEV.
9142 intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
9143
9144 // Intersect a range given by LVI.
9145 intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
9146 }
9147
9148 /// See AbstractAttribute::getAsStr().
9149 const std::string getAsStr(Attributor *A) const override {
9150 std::string Str;
9151 llvm::raw_string_ostream OS(Str);
9152 OS << "range(" << getBitWidth() << ")<";
9153 getKnown().print(OS);
9154 OS << " / ";
9155 getAssumed().print(OS);
9156 OS << ">";
9157 return Str;
9158 }
9159
9160 /// Helper function to get a SCEV expr for the associated value at program
9161 /// point \p I.
9162 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
9163 if (!getAnchorScope())
9164 return nullptr;
9165
9166 ScalarEvolution *SE =
9167 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9168 F: *getAnchorScope());
9169
9170 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9171 F: *getAnchorScope());
9172
9173 if (!SE || !LI)
9174 return nullptr;
9175
9176 const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
9177 if (!I)
9178 return S;
9179
9180 return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
9181 }
9182
9183 /// Helper function to get a range from SCEV for the associated value at
9184 /// program point \p I.
9185 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9186 const Instruction *I = nullptr) const {
9187 if (!getAnchorScope())
9188 return getWorstState(BitWidth: getBitWidth());
9189
9190 ScalarEvolution *SE =
9191 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9192 F: *getAnchorScope());
9193
9194 const SCEV *S = getSCEV(A, I);
9195 if (!SE || !S)
9196 return getWorstState(BitWidth: getBitWidth());
9197
9198 return SE->getUnsignedRange(S);
9199 }
9200
9201 /// Helper function to get a range from LVI for the associated value at
9202 /// program point \p I.
9203 ConstantRange
9204 getConstantRangeFromLVI(Attributor &A,
9205 const Instruction *CtxI = nullptr) const {
9206 if (!getAnchorScope())
9207 return getWorstState(BitWidth: getBitWidth());
9208
9209 LazyValueInfo *LVI =
9210 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9211 F: *getAnchorScope());
9212
9213 if (!LVI || !CtxI)
9214 return getWorstState(BitWidth: getBitWidth());
9215 return LVI->getConstantRange(V: &getAssociatedValue(),
9216 CxtI: const_cast<Instruction *>(CtxI),
9217 /*UndefAllowed*/ false);
9218 }
9219
9220 /// Return true if \p CtxI is valid for querying outside analyses.
9221 /// This basically makes sure we do not ask intra-procedural analysis
9222 /// about a context in the wrong function or a context that violates
9223 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9224 /// if the original context of this AA is OK or should be considered invalid.
9225 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9226 const Instruction *CtxI,
9227 bool AllowAACtxI) const {
9228 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9229 return false;
9230
9231 // Our context might be in a different function, neither intra-procedural
9232 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9233 if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9234 return false;
9235
9236 // If the context is not dominated by the value there are paths to the
9237 // context that do not define the value. This cannot be handled by
9238 // LazyValueInfo so we need to bail.
9239 if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9240 InformationCache &InfoCache = A.getInfoCache();
9241 const DominatorTree *DT =
9242 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9243 F: *I->getFunction());
9244 return DT && DT->dominates(Def: I, User: CtxI);
9245 }
9246
9247 return true;
9248 }
9249
9250 /// See AAValueConstantRange::getKnownConstantRange(..).
9251 ConstantRange
9252 getKnownConstantRange(Attributor &A,
9253 const Instruction *CtxI = nullptr) const override {
9254 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9255 /* AllowAACtxI */ false))
9256 return getKnown();
9257
9258 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9259 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9260 return getKnown().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9261 }
9262
9263 /// See AAValueConstantRange::getAssumedConstantRange(..).
9264 ConstantRange
9265 getAssumedConstantRange(Attributor &A,
9266 const Instruction *CtxI = nullptr) const override {
9267 // TODO: Make SCEV use Attributor assumption.
9268 // We may be able to bound a variable range via assumptions in
9269 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9270 // evolve to x^2 + x, then we can say that y is in [2, 12].
9271 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9272 /* AllowAACtxI */ false))
9273 return getAssumed();
9274
9275 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9276 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9277 return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9278 }
9279
9280 /// Helper function to create MDNode for range metadata.
9281 static MDNode *
9282 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9283 const ConstantRange &AssumedConstantRange) {
9284 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9285 Ty, V: AssumedConstantRange.getLower())),
9286 ConstantAsMetadata::get(C: ConstantInt::get(
9287 Ty, V: AssumedConstantRange.getUpper()))};
9288 return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9289 }
9290
9291 /// Return true if \p Assumed is included in ranges from instruction \p I.
9292 static bool isBetterRange(const ConstantRange &Assumed,
9293 const Instruction &I) {
9294 if (Assumed.isFullSet())
9295 return false;
9296
9297 std::optional<ConstantRange> Known;
9298
9299 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
9300 Known = CB->getRange();
9301 } else if (MDNode *KnownRanges = I.getMetadata(KindID: LLVMContext::MD_range)) {
9302 // If multiple ranges are annotated in IR, we give up to annotate assumed
9303 // range for now.
9304
9305 // TODO: If there exists a known range which containts assumed range, we
9306 // can say assumed range is better.
9307 if (KnownRanges->getNumOperands() > 2)
9308 return false;
9309
9310 ConstantInt *Lower =
9311 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9312 ConstantInt *Upper =
9313 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9314
9315 Known.emplace(args: Lower->getValue(), args: Upper->getValue());
9316 }
9317 return !Known || (*Known != Assumed && Known->contains(CR: Assumed));
9318 }
9319
9320 /// Helper function to set range metadata.
9321 static bool
9322 setRangeMetadataIfisBetterRange(Instruction *I,
9323 const ConstantRange &AssumedConstantRange) {
9324 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9325 I->setMetadata(KindID: LLVMContext::MD_range,
9326 Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9327 AssumedConstantRange));
9328 return true;
9329 }
9330 return false;
9331 }
9332 /// Helper function to set range return attribute.
9333 static bool
9334 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9335 Instruction *I,
9336 const ConstantRange &AssumedConstantRange) {
9337 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9338 A.manifestAttrs(IRP,
9339 DeducedAttrs: Attribute::get(Context&: I->getContext(), Kind: Attribute::Range,
9340 CR: AssumedConstantRange),
9341 /*ForceReplace*/ true);
9342 return true;
9343 }
9344 return false;
9345 }
9346
9347 /// See AbstractAttribute::manifest()
9348 ChangeStatus manifest(Attributor &A) override {
9349 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9350 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9351 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9352
9353 auto &V = getAssociatedValue();
9354 if (!AssumedConstantRange.isEmptySet() &&
9355 !AssumedConstantRange.isSingleElement()) {
9356 if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9357 assert(I == getCtxI() && "Should not annotate an instruction which is "
9358 "not the context instruction");
9359 if (isa<LoadInst>(Val: I))
9360 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9361 Changed = ChangeStatus::CHANGED;
9362 if (isa<CallInst>(Val: I))
9363 if (setRangeRetAttrIfisBetterRange(A, IRP: getIRPosition(), I,
9364 AssumedConstantRange))
9365 Changed = ChangeStatus::CHANGED;
9366 }
9367 }
9368
9369 return Changed;
9370 }
9371};
9372
9373struct AAValueConstantRangeArgument final
9374 : AAArgumentFromCallSiteArguments<
9375 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9376 true /* BridgeCallBaseContext */> {
9377 using Base = AAArgumentFromCallSiteArguments<
9378 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9379 true /* BridgeCallBaseContext */>;
9380 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9381 : Base(IRP, A) {}
9382
9383 /// See AbstractAttribute::trackStatistics()
9384 void trackStatistics() const override {
9385 STATS_DECLTRACK_ARG_ATTR(value_range)
9386 }
9387};
9388
9389struct AAValueConstantRangeReturned
9390 : AAReturnedFromReturnedValues<AAValueConstantRange,
9391 AAValueConstantRangeImpl,
9392 AAValueConstantRangeImpl::StateType,
9393 /* PropagateCallBaseContext */ true> {
9394 using Base =
9395 AAReturnedFromReturnedValues<AAValueConstantRange,
9396 AAValueConstantRangeImpl,
9397 AAValueConstantRangeImpl::StateType,
9398 /* PropagateCallBaseContext */ true>;
9399 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9400 : Base(IRP, A) {}
9401
9402 /// See AbstractAttribute::initialize(...).
9403 void initialize(Attributor &A) override {
9404 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9405 indicatePessimisticFixpoint();
9406 }
9407
9408 /// See AbstractAttribute::trackStatistics()
9409 void trackStatistics() const override {
9410 STATS_DECLTRACK_FNRET_ATTR(value_range)
9411 }
9412};
9413
9414struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9415 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9416 : AAValueConstantRangeImpl(IRP, A) {}
9417
9418 /// See AbstractAttribute::initialize(...).
9419 void initialize(Attributor &A) override {
9420 AAValueConstantRangeImpl::initialize(A);
9421 if (isAtFixpoint())
9422 return;
9423
9424 Value &V = getAssociatedValue();
9425
9426 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9427 unionAssumed(R: ConstantRange(C->getValue()));
9428 indicateOptimisticFixpoint();
9429 return;
9430 }
9431
9432 if (isa<UndefValue>(Val: &V)) {
9433 // Collapse the undef state to 0.
9434 unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9435 indicateOptimisticFixpoint();
9436 return;
9437 }
9438
9439 if (isa<CallBase>(Val: &V))
9440 return;
9441
9442 if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9443 return;
9444
9445 // If it is a load instruction with range metadata, use it.
9446 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9447 if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9448 intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9449 return;
9450 }
9451
9452 // We can work with PHI and select instruction as we traverse their operands
9453 // during update.
9454 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9455 return;
9456
9457 // Otherwise we give up.
9458 indicatePessimisticFixpoint();
9459
9460 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9461 << getAssociatedValue() << "\n");
9462 }
9463
9464 bool calculateBinaryOperator(
9465 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9466 const Instruction *CtxI,
9467 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9468 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9469 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9470
9471 // Simplify the operands first.
9472 bool UsedAssumedInformation = false;
9473 const auto &SimplifiedLHS = A.getAssumedSimplified(
9474 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9475 UsedAssumedInformation, S: AA::Interprocedural);
9476 if (!SimplifiedLHS.has_value())
9477 return true;
9478 if (!*SimplifiedLHS)
9479 return false;
9480 LHS = *SimplifiedLHS;
9481
9482 const auto &SimplifiedRHS = A.getAssumedSimplified(
9483 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9484 UsedAssumedInformation, S: AA::Interprocedural);
9485 if (!SimplifiedRHS.has_value())
9486 return true;
9487 if (!*SimplifiedRHS)
9488 return false;
9489 RHS = *SimplifiedRHS;
9490
9491 // TODO: Allow non integers as well.
9492 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9493 return false;
9494
9495 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9496 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9497 DepClass: DepClassTy::REQUIRED);
9498 if (!LHSAA)
9499 return false;
9500 QuerriedAAs.push_back(Elt: LHSAA);
9501 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9502
9503 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9504 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9505 DepClass: DepClassTy::REQUIRED);
9506 if (!RHSAA)
9507 return false;
9508 QuerriedAAs.push_back(Elt: RHSAA);
9509 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9510
9511 auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9512
9513 T.unionAssumed(R: AssumedRange);
9514
9515 // TODO: Track a known state too.
9516
9517 return T.isValidState();
9518 }
9519
9520 bool calculateCastInst(
9521 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9522 const Instruction *CtxI,
9523 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9524 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9525 // TODO: Allow non integers as well.
9526 Value *OpV = CastI->getOperand(i_nocapture: 0);
9527
9528 // Simplify the operand first.
9529 bool UsedAssumedInformation = false;
9530 const auto &SimplifiedOpV = A.getAssumedSimplified(
9531 IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9532 UsedAssumedInformation, S: AA::Interprocedural);
9533 if (!SimplifiedOpV.has_value())
9534 return true;
9535 if (!*SimplifiedOpV)
9536 return false;
9537 OpV = *SimplifiedOpV;
9538
9539 if (!OpV->getType()->isIntegerTy())
9540 return false;
9541
9542 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9543 QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9544 DepClass: DepClassTy::REQUIRED);
9545 if (!OpAA)
9546 return false;
9547 QuerriedAAs.push_back(Elt: OpAA);
9548 T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9549 BitWidth: getState().getBitWidth()));
9550 return T.isValidState();
9551 }
9552
9553 bool
9554 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9555 const Instruction *CtxI,
9556 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9557 Value *LHS = CmpI->getOperand(i_nocapture: 0);
9558 Value *RHS = CmpI->getOperand(i_nocapture: 1);
9559
9560 // Simplify the operands first.
9561 bool UsedAssumedInformation = false;
9562 const auto &SimplifiedLHS = A.getAssumedSimplified(
9563 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9564 UsedAssumedInformation, S: AA::Interprocedural);
9565 if (!SimplifiedLHS.has_value())
9566 return true;
9567 if (!*SimplifiedLHS)
9568 return false;
9569 LHS = *SimplifiedLHS;
9570
9571 const auto &SimplifiedRHS = A.getAssumedSimplified(
9572 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9573 UsedAssumedInformation, S: AA::Interprocedural);
9574 if (!SimplifiedRHS.has_value())
9575 return true;
9576 if (!*SimplifiedRHS)
9577 return false;
9578 RHS = *SimplifiedRHS;
9579
9580 // TODO: Allow non integers as well.
9581 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9582 return false;
9583
9584 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9585 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9586 DepClass: DepClassTy::REQUIRED);
9587 if (!LHSAA)
9588 return false;
9589 QuerriedAAs.push_back(Elt: LHSAA);
9590 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9591 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9592 DepClass: DepClassTy::REQUIRED);
9593 if (!RHSAA)
9594 return false;
9595 QuerriedAAs.push_back(Elt: RHSAA);
9596 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9597 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9598
9599 // If one of them is empty set, we can't decide.
9600 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9601 return true;
9602
9603 bool MustTrue = false, MustFalse = false;
9604
9605 auto AllowedRegion =
9606 ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9607
9608 if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9609 MustFalse = true;
9610
9611 if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9612 MustTrue = true;
9613
9614 assert((!MustTrue || !MustFalse) &&
9615 "Either MustTrue or MustFalse should be false!");
9616
9617 if (MustTrue)
9618 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9619 else if (MustFalse)
9620 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9621 else
9622 T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9623
9624 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9625 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9626 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9627 << *RHSAA);
9628
9629 // TODO: Track a known state too.
9630 return T.isValidState();
9631 }
9632
9633 /// See AbstractAttribute::updateImpl(...).
9634 ChangeStatus updateImpl(Attributor &A) override {
9635
9636 IntegerRangeState T(getBitWidth());
9637 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9638 Instruction *I = dyn_cast<Instruction>(Val: &V);
9639 if (!I || isa<CallBase>(Val: I)) {
9640
9641 // Simplify the operand first.
9642 bool UsedAssumedInformation = false;
9643 const auto &SimplifiedOpV = A.getAssumedSimplified(
9644 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9645 UsedAssumedInformation, S: AA::Interprocedural);
9646 if (!SimplifiedOpV.has_value())
9647 return true;
9648 if (!*SimplifiedOpV)
9649 return false;
9650 Value *VPtr = *SimplifiedOpV;
9651
9652 // If the value is not instruction, we query AA to Attributor.
9653 const auto *AA = A.getAAFor<AAValueConstantRange>(
9654 QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9655 DepClass: DepClassTy::REQUIRED);
9656
9657 // Clamp operator is not used to utilize a program point CtxI.
9658 if (AA)
9659 T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9660 else
9661 return false;
9662
9663 return T.isValidState();
9664 }
9665
9666 SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9667 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9668 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9669 return false;
9670 } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9671 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9672 return false;
9673 } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9674 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9675 return false;
9676 } else {
9677 // Give up with other instructions.
9678 // TODO: Add other instructions
9679
9680 T.indicatePessimisticFixpoint();
9681 return false;
9682 }
9683
9684 // Catch circular reasoning in a pessimistic way for now.
9685 // TODO: Check how the range evolves and if we stripped anything, see also
9686 // AADereferenceable or AAAlign for similar situations.
9687 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9688 if (QueriedAA != this)
9689 continue;
9690 // If we are in a stady state we do not need to worry.
9691 if (T.getAssumed() == getState().getAssumed())
9692 continue;
9693 T.indicatePessimisticFixpoint();
9694 }
9695
9696 return T.isValidState();
9697 };
9698
9699 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9700 return indicatePessimisticFixpoint();
9701
9702 // Ensure that long def-use chains can't cause circular reasoning either by
9703 // introducing a cutoff below.
9704 if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9705 return ChangeStatus::UNCHANGED;
9706 if (++NumChanges > MaxNumChanges) {
9707 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9708 << " but only " << MaxNumChanges
9709 << " are allowed to avoid cyclic reasoning.");
9710 return indicatePessimisticFixpoint();
9711 }
9712 return ChangeStatus::CHANGED;
9713 }
9714
9715 /// See AbstractAttribute::trackStatistics()
9716 void trackStatistics() const override {
9717 STATS_DECLTRACK_FLOATING_ATTR(value_range)
9718 }
9719
9720 /// Tracker to bail after too many widening steps of the constant range.
9721 int NumChanges = 0;
9722
9723 /// Upper bound for the number of allowed changes (=widening steps) for the
9724 /// constant range before we give up.
9725 static constexpr int MaxNumChanges = 5;
9726};
9727
9728struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9729 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9730 : AAValueConstantRangeImpl(IRP, A) {}
9731
9732 /// See AbstractAttribute::initialize(...).
9733 ChangeStatus updateImpl(Attributor &A) override {
9734 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9735 "not be called");
9736 }
9737
9738 /// See AbstractAttribute::trackStatistics()
9739 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9740};
9741
9742struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9743 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9744 : AAValueConstantRangeFunction(IRP, A) {}
9745
9746 /// See AbstractAttribute::trackStatistics()
9747 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9748};
9749
9750struct AAValueConstantRangeCallSiteReturned
9751 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9752 AAValueConstantRangeImpl::StateType,
9753 /* IntroduceCallBaseContext */ true> {
9754 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9755 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9756 AAValueConstantRangeImpl::StateType,
9757 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9758
9759 /// See AbstractAttribute::initialize(...).
9760 void initialize(Attributor &A) override {
9761 // If it is a call instruction with range attribute, use the range.
9762 if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) {
9763 if (std::optional<ConstantRange> Range = CI->getRange())
9764 intersectKnown(R: *Range);
9765 }
9766
9767 AAValueConstantRangeImpl::initialize(A);
9768 }
9769
9770 /// See AbstractAttribute::trackStatistics()
9771 void trackStatistics() const override {
9772 STATS_DECLTRACK_CSRET_ATTR(value_range)
9773 }
9774};
9775struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9776 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9777 : AAValueConstantRangeFloating(IRP, A) {}
9778
9779 /// See AbstractAttribute::manifest()
9780 ChangeStatus manifest(Attributor &A) override {
9781 return ChangeStatus::UNCHANGED;
9782 }
9783
9784 /// See AbstractAttribute::trackStatistics()
9785 void trackStatistics() const override {
9786 STATS_DECLTRACK_CSARG_ATTR(value_range)
9787 }
9788};
9789} // namespace
9790
9791/// ------------------ Potential Values Attribute -------------------------
9792
9793namespace {
9794struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9795 using StateType = PotentialConstantIntValuesState;
9796
9797 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9798 : AAPotentialConstantValues(IRP, A) {}
9799
9800 /// See AbstractAttribute::initialize(..).
9801 void initialize(Attributor &A) override {
9802 if (A.hasSimplificationCallback(IRP: getIRPosition()))
9803 indicatePessimisticFixpoint();
9804 else
9805 AAPotentialConstantValues::initialize(A);
9806 }
9807
9808 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9809 bool &ContainsUndef, bool ForSelf) {
9810 SmallVector<AA::ValueAndContext> Values;
9811 bool UsedAssumedInformation = false;
9812 if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9813 UsedAssumedInformation)) {
9814 // Avoid recursion when the caller is computing constant values for this
9815 // IRP itself.
9816 if (ForSelf)
9817 return false;
9818 if (!IRP.getAssociatedType()->isIntegerTy())
9819 return false;
9820 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9821 QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9822 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9823 return false;
9824 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9825 S = PotentialValuesAA->getState().getAssumedSet();
9826 return true;
9827 }
9828
9829 // Copy all the constant values, except UndefValue. ContainsUndef is true
9830 // iff Values contains only UndefValue instances. If there are other known
9831 // constants, then UndefValue is dropped.
9832 ContainsUndef = false;
9833 for (auto &It : Values) {
9834 if (isa<UndefValue>(Val: It.getValue())) {
9835 ContainsUndef = true;
9836 continue;
9837 }
9838 auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9839 if (!CI)
9840 return false;
9841 S.insert(X: CI->getValue());
9842 }
9843 ContainsUndef &= S.empty();
9844
9845 return true;
9846 }
9847
9848 /// See AbstractAttribute::getAsStr().
9849 const std::string getAsStr(Attributor *A) const override {
9850 std::string Str;
9851 llvm::raw_string_ostream OS(Str);
9852 OS << getState();
9853 return Str;
9854 }
9855
9856 /// See AbstractAttribute::updateImpl(...).
9857 ChangeStatus updateImpl(Attributor &A) override {
9858 return indicatePessimisticFixpoint();
9859 }
9860};
9861
9862struct AAPotentialConstantValuesArgument final
9863 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9864 AAPotentialConstantValuesImpl,
9865 PotentialConstantIntValuesState> {
9866 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9867 AAPotentialConstantValuesImpl,
9868 PotentialConstantIntValuesState>;
9869 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9870 : Base(IRP, A) {}
9871
9872 /// See AbstractAttribute::trackStatistics()
9873 void trackStatistics() const override {
9874 STATS_DECLTRACK_ARG_ATTR(potential_values)
9875 }
9876};
9877
9878struct AAPotentialConstantValuesReturned
9879 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9880 AAPotentialConstantValuesImpl> {
9881 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9882 AAPotentialConstantValuesImpl>;
9883 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9884 : Base(IRP, A) {}
9885
9886 void initialize(Attributor &A) override {
9887 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9888 indicatePessimisticFixpoint();
9889 Base::initialize(A);
9890 }
9891
9892 /// See AbstractAttribute::trackStatistics()
9893 void trackStatistics() const override {
9894 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9895 }
9896};
9897
9898struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9899 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9900 : AAPotentialConstantValuesImpl(IRP, A) {}
9901
9902 /// See AbstractAttribute::initialize(..).
9903 void initialize(Attributor &A) override {
9904 AAPotentialConstantValuesImpl::initialize(A);
9905 if (isAtFixpoint())
9906 return;
9907
9908 Value &V = getAssociatedValue();
9909
9910 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9911 unionAssumed(C: C->getValue());
9912 indicateOptimisticFixpoint();
9913 return;
9914 }
9915
9916 if (isa<UndefValue>(Val: &V)) {
9917 unionAssumedWithUndef();
9918 indicateOptimisticFixpoint();
9919 return;
9920 }
9921
9922 if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9923 return;
9924
9925 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9926 return;
9927
9928 indicatePessimisticFixpoint();
9929
9930 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9931 << getAssociatedValue() << "\n");
9932 }
9933
9934 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9935 const APInt &RHS) {
9936 return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9937 }
9938
9939 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9940 uint32_t ResultBitWidth) {
9941 Instruction::CastOps CastOp = CI->getOpcode();
9942 switch (CastOp) {
9943 default:
9944 llvm_unreachable("unsupported or not integer cast");
9945 case Instruction::Trunc:
9946 return Src.trunc(width: ResultBitWidth);
9947 case Instruction::SExt:
9948 return Src.sext(width: ResultBitWidth);
9949 case Instruction::ZExt:
9950 return Src.zext(width: ResultBitWidth);
9951 case Instruction::BitCast:
9952 return Src;
9953 }
9954 }
9955
9956 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9957 const APInt &LHS, const APInt &RHS,
9958 bool &SkipOperation, bool &Unsupported) {
9959 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9960 // Unsupported is set to true when the binary operator is not supported.
9961 // SkipOperation is set to true when UB occur with the given operand pair
9962 // (LHS, RHS).
9963 // TODO: we should look at nsw and nuw keywords to handle operations
9964 // that create poison or undef value.
9965 switch (BinOpcode) {
9966 default:
9967 Unsupported = true;
9968 return LHS;
9969 case Instruction::Add:
9970 return LHS + RHS;
9971 case Instruction::Sub:
9972 return LHS - RHS;
9973 case Instruction::Mul:
9974 return LHS * RHS;
9975 case Instruction::UDiv:
9976 if (RHS.isZero()) {
9977 SkipOperation = true;
9978 return LHS;
9979 }
9980 return LHS.udiv(RHS);
9981 case Instruction::SDiv:
9982 if (RHS.isZero()) {
9983 SkipOperation = true;
9984 return LHS;
9985 }
9986 return LHS.sdiv(RHS);
9987 case Instruction::URem:
9988 if (RHS.isZero()) {
9989 SkipOperation = true;
9990 return LHS;
9991 }
9992 return LHS.urem(RHS);
9993 case Instruction::SRem:
9994 if (RHS.isZero()) {
9995 SkipOperation = true;
9996 return LHS;
9997 }
9998 return LHS.srem(RHS);
9999 case Instruction::Shl:
10000 return LHS.shl(ShiftAmt: RHS);
10001 case Instruction::LShr:
10002 return LHS.lshr(ShiftAmt: RHS);
10003 case Instruction::AShr:
10004 return LHS.ashr(ShiftAmt: RHS);
10005 case Instruction::And:
10006 return LHS & RHS;
10007 case Instruction::Or:
10008 return LHS | RHS;
10009 case Instruction::Xor:
10010 return LHS ^ RHS;
10011 }
10012 }
10013
10014 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
10015 const APInt &LHS, const APInt &RHS) {
10016 bool SkipOperation = false;
10017 bool Unsupported = false;
10018 APInt Result =
10019 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
10020 if (Unsupported)
10021 return false;
10022 // If SkipOperation is true, we can ignore this operand pair (L, R).
10023 if (!SkipOperation)
10024 unionAssumed(C: Result);
10025 return isValidState();
10026 }
10027
10028 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
10029 auto AssumedBefore = getAssumed();
10030 Value *LHS = ICI->getOperand(i_nocapture: 0);
10031 Value *RHS = ICI->getOperand(i_nocapture: 1);
10032
10033 bool LHSContainsUndef = false, RHSContainsUndef = false;
10034 SetTy LHSAAPVS, RHSAAPVS;
10035 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10036 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10037 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10038 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10039 return indicatePessimisticFixpoint();
10040
10041 // TODO: make use of undef flag to limit potential values aggressively.
10042 bool MaybeTrue = false, MaybeFalse = false;
10043 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
10044 if (LHSContainsUndef && RHSContainsUndef) {
10045 // The result of any comparison between undefs can be soundly replaced
10046 // with undef.
10047 unionAssumedWithUndef();
10048 } else if (LHSContainsUndef) {
10049 for (const APInt &R : RHSAAPVS) {
10050 bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
10051 MaybeTrue |= CmpResult;
10052 MaybeFalse |= !CmpResult;
10053 if (MaybeTrue & MaybeFalse)
10054 return indicatePessimisticFixpoint();
10055 }
10056 } else if (RHSContainsUndef) {
10057 for (const APInt &L : LHSAAPVS) {
10058 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
10059 MaybeTrue |= CmpResult;
10060 MaybeFalse |= !CmpResult;
10061 if (MaybeTrue & MaybeFalse)
10062 return indicatePessimisticFixpoint();
10063 }
10064 } else {
10065 for (const APInt &L : LHSAAPVS) {
10066 for (const APInt &R : RHSAAPVS) {
10067 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
10068 MaybeTrue |= CmpResult;
10069 MaybeFalse |= !CmpResult;
10070 if (MaybeTrue & MaybeFalse)
10071 return indicatePessimisticFixpoint();
10072 }
10073 }
10074 }
10075 if (MaybeTrue)
10076 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
10077 if (MaybeFalse)
10078 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
10079 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10080 : ChangeStatus::CHANGED;
10081 }
10082
10083 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
10084 auto AssumedBefore = getAssumed();
10085 Value *LHS = SI->getTrueValue();
10086 Value *RHS = SI->getFalseValue();
10087
10088 bool UsedAssumedInformation = false;
10089 std::optional<Constant *> C = A.getAssumedConstant(
10090 V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
10091
10092 // Check if we only need one operand.
10093 bool OnlyLeft = false, OnlyRight = false;
10094 if (C && *C && (*C)->isOneValue())
10095 OnlyLeft = true;
10096 else if (C && *C && (*C)->isNullValue())
10097 OnlyRight = true;
10098
10099 bool LHSContainsUndef = false, RHSContainsUndef = false;
10100 SetTy LHSAAPVS, RHSAAPVS;
10101 if (!OnlyRight &&
10102 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10103 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
10104 return indicatePessimisticFixpoint();
10105
10106 if (!OnlyLeft &&
10107 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10108 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10109 return indicatePessimisticFixpoint();
10110
10111 if (OnlyLeft || OnlyRight) {
10112 // select (true/false), lhs, rhs
10113 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
10114 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
10115
10116 if (Undef)
10117 unionAssumedWithUndef();
10118 else {
10119 for (const auto &It : *OpAA)
10120 unionAssumed(C: It);
10121 }
10122
10123 } else if (LHSContainsUndef && RHSContainsUndef) {
10124 // select i1 *, undef , undef => undef
10125 unionAssumedWithUndef();
10126 } else {
10127 for (const auto &It : LHSAAPVS)
10128 unionAssumed(C: It);
10129 for (const auto &It : RHSAAPVS)
10130 unionAssumed(C: It);
10131 }
10132 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10133 : ChangeStatus::CHANGED;
10134 }
10135
10136 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
10137 auto AssumedBefore = getAssumed();
10138 if (!CI->isIntegerCast())
10139 return indicatePessimisticFixpoint();
10140 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
10141 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
10142 Value *Src = CI->getOperand(i_nocapture: 0);
10143
10144 bool SrcContainsUndef = false;
10145 SetTy SrcPVS;
10146 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
10147 ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
10148 return indicatePessimisticFixpoint();
10149
10150 if (SrcContainsUndef)
10151 unionAssumedWithUndef();
10152 else {
10153 for (const APInt &S : SrcPVS) {
10154 APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
10155 unionAssumed(C: T);
10156 }
10157 }
10158 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10159 : ChangeStatus::CHANGED;
10160 }
10161
10162 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
10163 auto AssumedBefore = getAssumed();
10164 Value *LHS = BinOp->getOperand(i_nocapture: 0);
10165 Value *RHS = BinOp->getOperand(i_nocapture: 1);
10166
10167 bool LHSContainsUndef = false, RHSContainsUndef = false;
10168 SetTy LHSAAPVS, RHSAAPVS;
10169 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10170 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10171 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10172 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10173 return indicatePessimisticFixpoint();
10174
10175 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
10176
10177 // TODO: make use of undef flag to limit potential values aggressively.
10178 if (LHSContainsUndef && RHSContainsUndef) {
10179 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
10180 return indicatePessimisticFixpoint();
10181 } else if (LHSContainsUndef) {
10182 for (const APInt &R : RHSAAPVS) {
10183 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
10184 return indicatePessimisticFixpoint();
10185 }
10186 } else if (RHSContainsUndef) {
10187 for (const APInt &L : LHSAAPVS) {
10188 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
10189 return indicatePessimisticFixpoint();
10190 }
10191 } else {
10192 for (const APInt &L : LHSAAPVS) {
10193 for (const APInt &R : RHSAAPVS) {
10194 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
10195 return indicatePessimisticFixpoint();
10196 }
10197 }
10198 }
10199 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10200 : ChangeStatus::CHANGED;
10201 }
10202
10203 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10204 auto AssumedBefore = getAssumed();
10205 SetTy Incoming;
10206 bool ContainsUndef;
10207 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10208 ContainsUndef, /* ForSelf */ true))
10209 return indicatePessimisticFixpoint();
10210 if (ContainsUndef) {
10211 unionAssumedWithUndef();
10212 } else {
10213 for (const auto &It : Incoming)
10214 unionAssumed(C: It);
10215 }
10216 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10217 : ChangeStatus::CHANGED;
10218 }
10219
10220 /// See AbstractAttribute::updateImpl(...).
10221 ChangeStatus updateImpl(Attributor &A) override {
10222 Value &V = getAssociatedValue();
10223 Instruction *I = dyn_cast<Instruction>(Val: &V);
10224
10225 if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10226 return updateWithICmpInst(A, ICI);
10227
10228 if (auto *SI = dyn_cast<SelectInst>(Val: I))
10229 return updateWithSelectInst(A, SI);
10230
10231 if (auto *CI = dyn_cast<CastInst>(Val: I))
10232 return updateWithCastInst(A, CI);
10233
10234 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10235 return updateWithBinaryOperator(A, BinOp);
10236
10237 if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10238 return updateWithInstruction(A, Inst: I);
10239
10240 return indicatePessimisticFixpoint();
10241 }
10242
10243 /// See AbstractAttribute::trackStatistics()
10244 void trackStatistics() const override {
10245 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10246 }
10247};
10248
10249struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10250 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10251 : AAPotentialConstantValuesImpl(IRP, A) {}
10252
10253 /// See AbstractAttribute::initialize(...).
10254 ChangeStatus updateImpl(Attributor &A) override {
10255 llvm_unreachable(
10256 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10257 "not be called");
10258 }
10259
10260 /// See AbstractAttribute::trackStatistics()
10261 void trackStatistics() const override {
10262 STATS_DECLTRACK_FN_ATTR(potential_values)
10263 }
10264};
10265
10266struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10267 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10268 : AAPotentialConstantValuesFunction(IRP, A) {}
10269
10270 /// See AbstractAttribute::trackStatistics()
10271 void trackStatistics() const override {
10272 STATS_DECLTRACK_CS_ATTR(potential_values)
10273 }
10274};
10275
10276struct AAPotentialConstantValuesCallSiteReturned
10277 : AACalleeToCallSite<AAPotentialConstantValues,
10278 AAPotentialConstantValuesImpl> {
10279 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10280 Attributor &A)
10281 : AACalleeToCallSite<AAPotentialConstantValues,
10282 AAPotentialConstantValuesImpl>(IRP, A) {}
10283
10284 /// See AbstractAttribute::trackStatistics()
10285 void trackStatistics() const override {
10286 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10287 }
10288};
10289
10290struct AAPotentialConstantValuesCallSiteArgument
10291 : AAPotentialConstantValuesFloating {
10292 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10293 Attributor &A)
10294 : AAPotentialConstantValuesFloating(IRP, A) {}
10295
10296 /// See AbstractAttribute::initialize(..).
10297 void initialize(Attributor &A) override {
10298 AAPotentialConstantValuesImpl::initialize(A);
10299 if (isAtFixpoint())
10300 return;
10301
10302 Value &V = getAssociatedValue();
10303
10304 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10305 unionAssumed(C: C->getValue());
10306 indicateOptimisticFixpoint();
10307 return;
10308 }
10309
10310 if (isa<UndefValue>(Val: &V)) {
10311 unionAssumedWithUndef();
10312 indicateOptimisticFixpoint();
10313 return;
10314 }
10315 }
10316
10317 /// See AbstractAttribute::updateImpl(...).
10318 ChangeStatus updateImpl(Attributor &A) override {
10319 Value &V = getAssociatedValue();
10320 auto AssumedBefore = getAssumed();
10321 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10322 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10323 if (!AA)
10324 return indicatePessimisticFixpoint();
10325 const auto &S = AA->getAssumed();
10326 unionAssumed(PVS: S);
10327 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10328 : ChangeStatus::CHANGED;
10329 }
10330
10331 /// See AbstractAttribute::trackStatistics()
10332 void trackStatistics() const override {
10333 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10334 }
10335};
10336} // namespace
10337
10338/// ------------------------ NoUndef Attribute ---------------------------------
10339bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10340 Attribute::AttrKind ImpliedAttributeKind,
10341 bool IgnoreSubsumingPositions) {
10342 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10343 "Unexpected attribute kind");
10344 if (A.hasAttr(IRP, AKs: {Attribute::NoUndef}, IgnoreSubsumingPositions,
10345 ImpliedAttributeKind: Attribute::NoUndef))
10346 return true;
10347
10348 Value &Val = IRP.getAssociatedValue();
10349 if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10350 isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10351 LLVMContext &Ctx = Val.getContext();
10352 A.manifestAttrs(IRP, DeducedAttrs: Attribute::get(Context&: Ctx, Kind: Attribute::NoUndef));
10353 return true;
10354 }
10355
10356 return false;
10357}
10358
10359namespace {
10360struct AANoUndefImpl : AANoUndef {
10361 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10362
10363 /// See AbstractAttribute::initialize(...).
10364 void initialize(Attributor &A) override {
10365 Value &V = getAssociatedValue();
10366 if (isa<UndefValue>(Val: V))
10367 indicatePessimisticFixpoint();
10368 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10369 }
10370
10371 /// See followUsesInMBEC
10372 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10373 AANoUndef::StateType &State) {
10374 const Value *UseV = U->get();
10375 const DominatorTree *DT = nullptr;
10376 AssumptionCache *AC = nullptr;
10377 InformationCache &InfoCache = A.getInfoCache();
10378 if (Function *F = getAnchorScope()) {
10379 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10380 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10381 }
10382 State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10383 bool TrackUse = false;
10384 // Track use for instructions which must produce undef or poison bits when
10385 // at least one operand contains such bits.
10386 if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10387 TrackUse = true;
10388 return TrackUse;
10389 }
10390
10391 /// See AbstractAttribute::getAsStr().
10392 const std::string getAsStr(Attributor *A) const override {
10393 return getAssumed() ? "noundef" : "may-undef-or-poison";
10394 }
10395
10396 ChangeStatus manifest(Attributor &A) override {
10397 // We don't manifest noundef attribute for dead positions because the
10398 // associated values with dead positions would be replaced with undef
10399 // values.
10400 bool UsedAssumedInformation = false;
10401 if (A.isAssumedDead(IRP: getIRPosition(), QueryingAA: nullptr, FnLivenessAA: nullptr,
10402 UsedAssumedInformation))
10403 return ChangeStatus::UNCHANGED;
10404 // A position whose simplified value does not have any value is
10405 // considered to be dead. We don't manifest noundef in such positions for
10406 // the same reason above.
10407 if (!A.getAssumedSimplified(IRP: getIRPosition(), AA: *this, UsedAssumedInformation,
10408 S: AA::Interprocedural)
10409 .has_value())
10410 return ChangeStatus::UNCHANGED;
10411 return AANoUndef::manifest(A);
10412 }
10413};
10414
10415struct AANoUndefFloating : public AANoUndefImpl {
10416 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10417 : AANoUndefImpl(IRP, A) {}
10418
10419 /// See AbstractAttribute::initialize(...).
10420 void initialize(Attributor &A) override {
10421 AANoUndefImpl::initialize(A);
10422 if (!getState().isAtFixpoint() && getAnchorScope() &&
10423 !getAnchorScope()->isDeclaration())
10424 if (Instruction *CtxI = getCtxI())
10425 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10426 }
10427
10428 /// See AbstractAttribute::updateImpl(...).
10429 ChangeStatus updateImpl(Attributor &A) override {
10430 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10431 bool IsKnownNoUndef;
10432 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10433 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoUndef);
10434 };
10435
10436 bool Stripped;
10437 bool UsedAssumedInformation = false;
10438 Value *AssociatedValue = &getAssociatedValue();
10439 SmallVector<AA::ValueAndContext> Values;
10440 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10441 S: AA::AnyScope, UsedAssumedInformation))
10442 Stripped = false;
10443 else
10444 Stripped =
10445 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10446
10447 if (!Stripped) {
10448 // If we haven't stripped anything we might still be able to use a
10449 // different AA, but only if the IRP changes. Effectively when we
10450 // interpret this not as a call site value but as a floating/argument
10451 // value.
10452 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10453 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10454 return indicatePessimisticFixpoint();
10455 return ChangeStatus::UNCHANGED;
10456 }
10457
10458 for (const auto &VAC : Values)
10459 if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10460 return indicatePessimisticFixpoint();
10461
10462 return ChangeStatus::UNCHANGED;
10463 }
10464
10465 /// See AbstractAttribute::trackStatistics()
10466 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10467};
10468
10469struct AANoUndefReturned final
10470 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10471 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10472 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10473
10474 /// See AbstractAttribute::trackStatistics()
10475 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10476};
10477
10478struct AANoUndefArgument final
10479 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10480 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10481 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10482
10483 /// See AbstractAttribute::trackStatistics()
10484 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10485};
10486
10487struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10488 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10489 : AANoUndefFloating(IRP, A) {}
10490
10491 /// See AbstractAttribute::trackStatistics()
10492 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10493};
10494
10495struct AANoUndefCallSiteReturned final
10496 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10497 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10498 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10499
10500 /// See AbstractAttribute::trackStatistics()
10501 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10502};
10503
10504/// ------------------------ NoFPClass Attribute -------------------------------
10505
10506struct AANoFPClassImpl : AANoFPClass {
10507 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10508
10509 void initialize(Attributor &A) override {
10510 const IRPosition &IRP = getIRPosition();
10511
10512 Value &V = IRP.getAssociatedValue();
10513 if (isa<UndefValue>(Val: V)) {
10514 indicateOptimisticFixpoint();
10515 return;
10516 }
10517
10518 SmallVector<Attribute> Attrs;
10519 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::NoFPClass}, Attrs, IgnoreSubsumingPositions: false);
10520 for (const auto &Attr : Attrs) {
10521 addKnownBits(Bits: Attr.getNoFPClass());
10522 }
10523
10524 Instruction *CtxI = getCtxI();
10525
10526 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10527 const DataLayout &DL = A.getDataLayout();
10528 InformationCache &InfoCache = A.getInfoCache();
10529
10530 const DominatorTree *DT = nullptr;
10531 AssumptionCache *AC = nullptr;
10532 const TargetLibraryInfo *TLI = nullptr;
10533 Function *F = getAnchorScope();
10534 if (F) {
10535 TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10536 if (!F->isDeclaration()) {
10537 DT =
10538 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10539 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10540 }
10541 }
10542
10543 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10544
10545 KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, InterestedClasses: fcAllFlags, SQ: Q);
10546 addKnownBits(Bits: ~KnownFPClass.KnownFPClasses);
10547 }
10548
10549 if (CtxI)
10550 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10551 }
10552
10553 /// See followUsesInMBEC
10554 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10555 AANoFPClass::StateType &State) {
10556 // TODO: Determine what instructions can be looked through.
10557 auto *CB = dyn_cast<CallBase>(Val: I);
10558 if (!CB)
10559 return false;
10560
10561 if (!CB->isArgOperand(U))
10562 return false;
10563
10564 unsigned ArgNo = CB->getArgOperandNo(U);
10565 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
10566 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP, DepClass: DepClassTy::NONE))
10567 State.addKnownBits(Bits: NoFPAA->getState().getKnown());
10568 return false;
10569 }
10570
10571 const std::string getAsStr(Attributor *A) const override {
10572 std::string Result = "nofpclass";
10573 raw_string_ostream OS(Result);
10574 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10575 return Result;
10576 }
10577
10578 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10579 SmallVectorImpl<Attribute> &Attrs) const override {
10580 Attrs.emplace_back(Args: Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10581 }
10582};
10583
10584struct AANoFPClassFloating : public AANoFPClassImpl {
10585 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10586 : AANoFPClassImpl(IRP, A) {}
10587
10588 /// See AbstractAttribute::updateImpl(...).
10589 ChangeStatus updateImpl(Attributor &A) override {
10590 SmallVector<AA::ValueAndContext> Values;
10591 bool UsedAssumedInformation = false;
10592 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10593 S: AA::AnyScope, UsedAssumedInformation)) {
10594 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10595 }
10596
10597 StateType T;
10598 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10599 const auto *AA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP: IRPosition::value(V),
10600 DepClass: DepClassTy::REQUIRED);
10601 if (!AA || this == AA) {
10602 T.indicatePessimisticFixpoint();
10603 } else {
10604 const AANoFPClass::StateType &S =
10605 static_cast<const AANoFPClass::StateType &>(AA->getState());
10606 T ^= S;
10607 }
10608 return T.isValidState();
10609 };
10610
10611 for (const auto &VAC : Values)
10612 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10613 return indicatePessimisticFixpoint();
10614
10615 return clampStateAndIndicateChange(S&: getState(), R: T);
10616 }
10617
10618 /// See AbstractAttribute::trackStatistics()
10619 void trackStatistics() const override {
10620 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10621 }
10622};
10623
10624struct AANoFPClassReturned final
10625 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10626 AANoFPClassImpl::StateType, false,
10627 Attribute::None, false> {
10628 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10629 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10630 AANoFPClassImpl::StateType, false,
10631 Attribute::None, false>(IRP, A) {}
10632
10633 /// See AbstractAttribute::trackStatistics()
10634 void trackStatistics() const override {
10635 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10636 }
10637};
10638
10639struct AANoFPClassArgument final
10640 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10641 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10642 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10643
10644 /// See AbstractAttribute::trackStatistics()
10645 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10646};
10647
10648struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10649 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10650 : AANoFPClassFloating(IRP, A) {}
10651
10652 /// See AbstractAttribute::trackStatistics()
10653 void trackStatistics() const override {
10654 STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10655 }
10656};
10657
10658struct AANoFPClassCallSiteReturned final
10659 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10660 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10661 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10662
10663 /// See AbstractAttribute::trackStatistics()
10664 void trackStatistics() const override {
10665 STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10666 }
10667};
10668
10669struct AACallEdgesImpl : public AACallEdges {
10670 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10671
10672 const SetVector<Function *> &getOptimisticEdges() const override {
10673 return CalledFunctions;
10674 }
10675
10676 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10677
10678 bool hasNonAsmUnknownCallee() const override {
10679 return HasUnknownCalleeNonAsm;
10680 }
10681
10682 const std::string getAsStr(Attributor *A) const override {
10683 return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10684 std::to_string(val: CalledFunctions.size()) + "]";
10685 }
10686
10687 void trackStatistics() const override {}
10688
10689protected:
10690 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10691 if (CalledFunctions.insert(X: Fn)) {
10692 Change = ChangeStatus::CHANGED;
10693 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10694 << "\n");
10695 }
10696 }
10697
10698 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10699 if (!HasUnknownCallee)
10700 Change = ChangeStatus::CHANGED;
10701 if (NonAsm && !HasUnknownCalleeNonAsm)
10702 Change = ChangeStatus::CHANGED;
10703 HasUnknownCalleeNonAsm |= NonAsm;
10704 HasUnknownCallee = true;
10705 }
10706
10707private:
10708 /// Optimistic set of functions that might be called by this position.
10709 SetVector<Function *> CalledFunctions;
10710
10711 /// Is there any call with a unknown callee.
10712 bool HasUnknownCallee = false;
10713
10714 /// Is there any call with a unknown callee, excluding any inline asm.
10715 bool HasUnknownCalleeNonAsm = false;
10716};
10717
10718struct AACallEdgesCallSite : public AACallEdgesImpl {
10719 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10720 : AACallEdgesImpl(IRP, A) {}
10721 /// See AbstractAttribute::updateImpl(...).
10722 ChangeStatus updateImpl(Attributor &A) override {
10723 ChangeStatus Change = ChangeStatus::UNCHANGED;
10724
10725 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10726 if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10727 addCalledFunction(Fn, Change);
10728 } else {
10729 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10730 setHasUnknownCallee(NonAsm: true, Change);
10731 }
10732
10733 // Explore all values.
10734 return true;
10735 };
10736
10737 SmallVector<AA::ValueAndContext> Values;
10738 // Process any value that we might call.
10739 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10740 if (isa<Constant>(Val: V)) {
10741 VisitValue(*V, CtxI);
10742 return;
10743 }
10744
10745 bool UsedAssumedInformation = false;
10746 Values.clear();
10747 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10748 S: AA::AnyScope, UsedAssumedInformation)) {
10749 Values.push_back(Elt: {*V, CtxI});
10750 }
10751 for (auto &VAC : Values)
10752 VisitValue(*VAC.getValue(), VAC.getCtxI());
10753 };
10754
10755 CallBase *CB = cast<CallBase>(Val: getCtxI());
10756
10757 if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10758 if (IA->hasSideEffects() &&
10759 !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10760 !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10761 setHasUnknownCallee(NonAsm: false, Change);
10762 }
10763 return Change;
10764 }
10765
10766 if (CB->isIndirectCall())
10767 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10768 QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10769 if (IndirectCallAA->foreachCallee(
10770 CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10771 return Change;
10772
10773 // The most simple case.
10774 ProcessCalledOperand(CB->getCalledOperand(), CB);
10775
10776 // Process callback functions.
10777 SmallVector<const Use *, 4u> CallbackUses;
10778 AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10779 for (const Use *U : CallbackUses)
10780 ProcessCalledOperand(U->get(), CB);
10781
10782 return Change;
10783 }
10784};
10785
10786struct AACallEdgesFunction : public AACallEdgesImpl {
10787 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10788 : AACallEdgesImpl(IRP, A) {}
10789
10790 /// See AbstractAttribute::updateImpl(...).
10791 ChangeStatus updateImpl(Attributor &A) override {
10792 ChangeStatus Change = ChangeStatus::UNCHANGED;
10793
10794 auto ProcessCallInst = [&](Instruction &Inst) {
10795 CallBase &CB = cast<CallBase>(Val&: Inst);
10796
10797 auto *CBEdges = A.getAAFor<AACallEdges>(
10798 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10799 if (!CBEdges)
10800 return false;
10801 if (CBEdges->hasNonAsmUnknownCallee())
10802 setHasUnknownCallee(NonAsm: true, Change);
10803 if (CBEdges->hasUnknownCallee())
10804 setHasUnknownCallee(NonAsm: false, Change);
10805
10806 for (Function *F : CBEdges->getOptimisticEdges())
10807 addCalledFunction(Fn: F, Change);
10808
10809 return true;
10810 };
10811
10812 // Visit all callable instructions.
10813 bool UsedAssumedInformation = false;
10814 if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10815 UsedAssumedInformation,
10816 /* CheckBBLivenessOnly */ true)) {
10817 // If we haven't looked at all call like instructions, assume that there
10818 // are unknown callees.
10819 setHasUnknownCallee(NonAsm: true, Change);
10820 }
10821
10822 return Change;
10823 }
10824};
10825
10826/// -------------------AAInterFnReachability Attribute--------------------------
10827
10828struct AAInterFnReachabilityFunction
10829 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10830 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10831 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10832 : Base(IRP, A) {}
10833
10834 bool instructionCanReach(
10835 Attributor &A, const Instruction &From, const Function &To,
10836 const AA::InstExclusionSetTy *ExclusionSet) const override {
10837 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10838 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10839
10840 RQITy StackRQI(A, From, To, ExclusionSet, false);
10841 RQITy::Reachable Result;
10842 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10843 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10844 /*IsTemporaryRQI=*/true);
10845 return Result == RQITy::Reachable::Yes;
10846 }
10847
10848 bool isReachableImpl(Attributor &A, RQITy &RQI,
10849 bool IsTemporaryRQI) override {
10850 const Instruction *EntryI =
10851 &RQI.From->getFunction()->getEntryBlock().front();
10852 if (EntryI != RQI.From &&
10853 !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10854 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10855 IsTemporaryRQI);
10856
10857 auto CheckReachableCallBase = [&](CallBase *CB) {
10858 auto *CBEdges = A.getAAFor<AACallEdges>(
10859 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10860 if (!CBEdges || !CBEdges->getState().isValidState())
10861 return false;
10862 // TODO Check To backwards in this case.
10863 if (CBEdges->hasUnknownCallee())
10864 return false;
10865
10866 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10867 if (Fn == RQI.To)
10868 return false;
10869
10870 if (Fn->isDeclaration()) {
10871 if (Fn->hasFnAttribute(Kind: Attribute::NoCallback))
10872 continue;
10873 // TODO Check To backwards in this case.
10874 return false;
10875 }
10876
10877 if (Fn == getAnchorScope()) {
10878 if (EntryI == RQI.From)
10879 continue;
10880 return false;
10881 }
10882
10883 const AAInterFnReachability *InterFnReachability =
10884 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10885 DepClass: DepClassTy::OPTIONAL);
10886
10887 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10888 if (!InterFnReachability ||
10889 InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10890 ExclusionSet: RQI.ExclusionSet))
10891 return false;
10892 }
10893 return true;
10894 };
10895
10896 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10897 QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10898 DepClass: DepClassTy::OPTIONAL);
10899
10900 // Determine call like instructions that we can reach from the inst.
10901 auto CheckCallBase = [&](Instruction &CBInst) {
10902 // There are usually less nodes in the call graph, check inter function
10903 // reachability first.
10904 if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10905 return true;
10906 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10907 A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10908 };
10909
10910 bool UsedExclusionSet = /* conservative */ true;
10911 bool UsedAssumedInformation = false;
10912 if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10913 UsedAssumedInformation,
10914 /* CheckBBLivenessOnly */ true))
10915 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10916 IsTemporaryRQI);
10917
10918 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10919 IsTemporaryRQI);
10920 }
10921
10922 void trackStatistics() const override {}
10923};
10924} // namespace
10925
10926template <typename AAType>
10927static std::optional<Constant *>
10928askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10929 const IRPosition &IRP, Type &Ty) {
10930 if (!Ty.isIntegerTy())
10931 return nullptr;
10932
10933 // This will also pass the call base context.
10934 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10935 if (!AA)
10936 return nullptr;
10937
10938 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10939
10940 if (!COpt.has_value()) {
10941 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10942 return std::nullopt;
10943 }
10944 if (auto *C = *COpt) {
10945 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10946 return C;
10947 }
10948 return nullptr;
10949}
10950
10951Value *AAPotentialValues::getSingleValue(
10952 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10953 SmallVectorImpl<AA::ValueAndContext> &Values) {
10954 Type &Ty = *IRP.getAssociatedType();
10955 std::optional<Value *> V;
10956 for (auto &It : Values) {
10957 V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10958 if (V.has_value() && !*V)
10959 break;
10960 }
10961 if (!V.has_value())
10962 return UndefValue::get(T: &Ty);
10963 return *V;
10964}
10965
10966namespace {
10967struct AAPotentialValuesImpl : AAPotentialValues {
10968 using StateType = PotentialLLVMValuesState;
10969
10970 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10971 : AAPotentialValues(IRP, A) {}
10972
10973 /// See AbstractAttribute::initialize(..).
10974 void initialize(Attributor &A) override {
10975 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10976 indicatePessimisticFixpoint();
10977 return;
10978 }
10979 Value *Stripped = getAssociatedValue().stripPointerCasts();
10980 if (isa<Constant>(Val: Stripped) && !isa<ConstantExpr>(Val: Stripped)) {
10981 addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10982 AnchorScope: getAnchorScope());
10983 indicateOptimisticFixpoint();
10984 return;
10985 }
10986 AAPotentialValues::initialize(A);
10987 }
10988
10989 /// See AbstractAttribute::getAsStr().
10990 const std::string getAsStr(Attributor *A) const override {
10991 std::string Str;
10992 llvm::raw_string_ostream OS(Str);
10993 OS << getState();
10994 return Str;
10995 }
10996
10997 template <typename AAType>
10998 static std::optional<Value *> askOtherAA(Attributor &A,
10999 const AbstractAttribute &AA,
11000 const IRPosition &IRP, Type &Ty) {
11001 if (isa<Constant>(Val: IRP.getAssociatedValue()))
11002 return &IRP.getAssociatedValue();
11003 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
11004 if (!C)
11005 return std::nullopt;
11006 if (*C)
11007 if (auto *CC = AA::getWithType(V&: **C, Ty))
11008 return CC;
11009 return nullptr;
11010 }
11011
11012 virtual void addValue(Attributor &A, StateType &State, Value &V,
11013 const Instruction *CtxI, AA::ValueScope S,
11014 Function *AnchorScope) const {
11015
11016 IRPosition ValIRP = IRPosition::value(V);
11017 if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
11018 for (const auto &U : CB->args()) {
11019 if (U.get() != &V)
11020 continue;
11021 ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
11022 break;
11023 }
11024 }
11025
11026 Value *VPtr = &V;
11027 if (ValIRP.getAssociatedType()->isIntegerTy()) {
11028 Type &Ty = *getAssociatedType();
11029 std::optional<Value *> SimpleV =
11030 askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
11031 if (SimpleV.has_value() && !*SimpleV) {
11032 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
11033 QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
11034 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
11035 for (const auto &It : PotentialConstantsAA->getAssumedSet())
11036 State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
11037 if (PotentialConstantsAA->undefIsContained())
11038 State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
11039 return;
11040 }
11041 }
11042 if (!SimpleV.has_value())
11043 return;
11044
11045 if (*SimpleV)
11046 VPtr = *SimpleV;
11047 }
11048
11049 if (isa<ConstantInt>(Val: VPtr))
11050 CtxI = nullptr;
11051 if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
11052 S = AA::ValueScope(S | AA::Interprocedural);
11053
11054 State.unionAssumed(C: {{*VPtr, CtxI}, S});
11055 }
11056
11057 /// Helper struct to tie a value+context pair together with the scope for
11058 /// which this is the simplified version.
11059 struct ItemInfo {
11060 AA::ValueAndContext I;
11061 AA::ValueScope S;
11062
11063 bool operator==(const ItemInfo &II) const {
11064 return II.I == I && II.S == S;
11065 };
11066 bool operator<(const ItemInfo &II) const {
11067 return std::tie(args: I, args: S) < std::tie(args: II.I, args: II.S);
11068 };
11069 };
11070
11071 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
11072 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
11073 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
11074 if (!(CS & S))
11075 continue;
11076
11077 bool UsedAssumedInformation = false;
11078 SmallVector<AA::ValueAndContext> Values;
11079 if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
11080 UsedAssumedInformation))
11081 return false;
11082
11083 for (auto &It : Values)
11084 ValueScopeMap[It] += CS;
11085 }
11086 for (auto &It : ValueScopeMap)
11087 addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11088 S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
11089
11090 return true;
11091 }
11092
11093 void giveUpOnIntraprocedural(Attributor &A) {
11094 auto NewS = StateType::getBestState(PVS: getState());
11095 for (const auto &It : getAssumedSet()) {
11096 if (It.second == AA::Intraprocedural)
11097 continue;
11098 addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11099 S: AA::Interprocedural, AnchorScope: getAnchorScope());
11100 }
11101 assert(!undefIsContained() && "Undef should be an explicit value!");
11102 addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
11103 AnchorScope: getAnchorScope());
11104 getState() = NewS;
11105 }
11106
11107 /// See AbstractState::indicatePessimisticFixpoint(...).
11108 ChangeStatus indicatePessimisticFixpoint() override {
11109 getState() = StateType::getBestState(PVS: getState());
11110 getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11111 AAPotentialValues::indicateOptimisticFixpoint();
11112 return ChangeStatus::CHANGED;
11113 }
11114
11115 /// See AbstractAttribute::updateImpl(...).
11116 ChangeStatus updateImpl(Attributor &A) override {
11117 return indicatePessimisticFixpoint();
11118 }
11119
11120 /// See AbstractAttribute::manifest(...).
11121 ChangeStatus manifest(Attributor &A) override {
11122 SmallVector<AA::ValueAndContext> Values;
11123 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11124 Values.clear();
11125 if (!getAssumedSimplifiedValues(A, Values, S))
11126 continue;
11127 Value &OldV = getAssociatedValue();
11128 if (isa<UndefValue>(Val: OldV))
11129 continue;
11130 Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11131 if (!NewV || NewV == &OldV)
11132 continue;
11133 if (getCtxI() &&
11134 !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
11135 continue;
11136 if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
11137 return ChangeStatus::CHANGED;
11138 }
11139 return ChangeStatus::UNCHANGED;
11140 }
11141
11142 bool getAssumedSimplifiedValues(
11143 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11144 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11145 if (!isValidState())
11146 return false;
11147 bool UsedAssumedInformation = false;
11148 for (const auto &It : getAssumedSet())
11149 if (It.second & S) {
11150 if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
11151 isa<SelectInst>(Val: It.first.getValue()))) {
11152 if (A.getAssumedSimplifiedValues(
11153 IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
11154 AA: this, Values, S, UsedAssumedInformation))
11155 continue;
11156 }
11157 Values.push_back(Elt: It.first);
11158 }
11159 assert(!undefIsContained() && "Undef should be an explicit value!");
11160 return true;
11161 }
11162};
11163
11164struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11165 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11166 : AAPotentialValuesImpl(IRP, A) {}
11167
11168 /// See AbstractAttribute::updateImpl(...).
11169 ChangeStatus updateImpl(Attributor &A) override {
11170 auto AssumedBefore = getAssumed();
11171
11172 genericValueTraversal(A, InitialV: &getAssociatedValue());
11173
11174 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11175 : ChangeStatus::CHANGED;
11176 }
11177
11178 /// Helper struct to remember which AAIsDead instances we actually used.
11179 struct LivenessInfo {
11180 const AAIsDead *LivenessAA = nullptr;
11181 bool AnyDead = false;
11182 };
11183
11184 /// Check if \p Cmp is a comparison we can simplify.
11185 ///
11186 /// We handle multiple cases, one in which at least one operand is an
11187 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11188 /// operand. Return true if successful, in that case Worklist will be updated.
11189 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11190 CmpInst::Predicate Pred, ItemInfo II,
11191 SmallVectorImpl<ItemInfo> &Worklist) {
11192
11193 // Simplify the operands first.
11194 bool UsedAssumedInformation = false;
11195 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11196 auto GetSimplifiedValues = [&](Value &V,
11197 SmallVector<AA::ValueAndContext> &Values) {
11198 if (!A.getAssumedSimplifiedValues(
11199 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11200 S: AA::Intraprocedural, UsedAssumedInformation)) {
11201 Values.clear();
11202 Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11203 }
11204 return Values.empty();
11205 };
11206 if (GetSimplifiedValues(*LHS, LHSValues))
11207 return true;
11208 if (GetSimplifiedValues(*RHS, RHSValues))
11209 return true;
11210
11211 LLVMContext &Ctx = LHS->getContext();
11212
11213 InformationCache &InfoCache = A.getInfoCache();
11214 Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11215 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11216 const auto *DT =
11217 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11218 : nullptr;
11219 const auto *TLI =
11220 F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11221 auto *AC =
11222 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11223 : nullptr;
11224
11225 const DataLayout &DL = A.getDataLayout();
11226 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11227
11228 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11229 if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11230 addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11231 /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11232 return true;
11233 }
11234
11235 // Handle the trivial case first in which we don't even need to think
11236 // about null or non-null.
11237 if (&LHSV == &RHSV &&
11238 (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11239 Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11240 V: CmpInst::isTrueWhenEqual(predicate: Pred));
11241 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11242 AnchorScope: getAnchorScope());
11243 return true;
11244 }
11245
11246 auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11247 auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11248 if (TypedLHS && TypedRHS) {
11249 Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11250 if (NewV && NewV != &Cmp) {
11251 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11252 AnchorScope: getAnchorScope());
11253 return true;
11254 }
11255 }
11256
11257 // From now on we only handle equalities (==, !=).
11258 if (!CmpInst::isEquality(pred: Pred))
11259 return false;
11260
11261 bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11262 bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11263 if (!LHSIsNull && !RHSIsNull)
11264 return false;
11265
11266 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11267 // non-nullptr operand and if we assume it's non-null we can conclude the
11268 // result of the comparison.
11269 assert((LHSIsNull || RHSIsNull) &&
11270 "Expected nullptr versus non-nullptr comparison at this point");
11271
11272 // The index is the operand that we assume is not null.
11273 unsigned PtrIdx = LHSIsNull;
11274 bool IsKnownNonNull;
11275 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11276 A, QueryingAA: this, IRP: IRPosition::value(V: *(PtrIdx ? &RHSV : &LHSV)),
11277 DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNonNull);
11278 if (!IsAssumedNonNull)
11279 return false;
11280
11281 // The new value depends on the predicate, true for != and false for ==.
11282 Constant *NewV =
11283 ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11284 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11285 AnchorScope: getAnchorScope());
11286 return true;
11287 };
11288
11289 for (auto &LHSValue : LHSValues)
11290 for (auto &RHSValue : RHSValues)
11291 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11292 return false;
11293 return true;
11294 }
11295
11296 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11297 SmallVectorImpl<ItemInfo> &Worklist) {
11298 const Instruction *CtxI = II.I.getCtxI();
11299 bool UsedAssumedInformation = false;
11300
11301 std::optional<Constant *> C =
11302 A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11303 bool NoValueYet = !C.has_value();
11304 if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11305 return true;
11306 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11307 if (CI->isZero())
11308 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11309 else
11310 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11311 } else if (&SI == &getAssociatedValue()) {
11312 // We could not simplify the condition, assume both values.
11313 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11314 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11315 } else {
11316 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11317 IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11318 if (!SimpleV.has_value())
11319 return true;
11320 if (*SimpleV) {
11321 addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11322 return true;
11323 }
11324 return false;
11325 }
11326 return true;
11327 }
11328
11329 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11330 SmallVectorImpl<ItemInfo> &Worklist) {
11331 SmallSetVector<Value *, 4> PotentialCopies;
11332 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11333 bool UsedAssumedInformation = false;
11334 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11335 PotentialValueOrigins, QueryingAA: *this,
11336 UsedAssumedInformation,
11337 /* OnlyExact */ true)) {
11338 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11339 "loaded values for load instruction "
11340 << LI << "\n");
11341 return false;
11342 }
11343
11344 // Do not simplify loads that are only used in llvm.assume if we cannot also
11345 // remove all stores that may feed into the load. The reason is that the
11346 // assume is probably worth something as long as the stores are around.
11347 InformationCache &InfoCache = A.getInfoCache();
11348 if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11349 if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11350 if (!I || isa<AssumeInst>(Val: I))
11351 return true;
11352 if (auto *SI = dyn_cast<StoreInst>(Val: I))
11353 return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11354 /* LivenessAA */ FnLivenessAA: nullptr,
11355 UsedAssumedInformation,
11356 /* CheckBBLivenessOnly */ false);
11357 return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11358 UsedAssumedInformation,
11359 /* CheckBBLivenessOnly */ false);
11360 })) {
11361 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11362 "and we cannot delete all the stores: "
11363 << LI << "\n");
11364 return false;
11365 }
11366 }
11367
11368 // Values have to be dynamically unique or we loose the fact that a
11369 // single llvm::Value might represent two runtime values (e.g.,
11370 // stack locations in different recursive calls).
11371 const Instruction *CtxI = II.I.getCtxI();
11372 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11373 bool AllLocal = ScopeIsLocal;
11374 bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11375 AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11376 return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11377 });
11378 if (!DynamicallyUnique) {
11379 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11380 "values are dynamically unique: "
11381 << LI << "\n");
11382 return false;
11383 }
11384
11385 for (auto *PotentialCopy : PotentialCopies) {
11386 if (AllLocal) {
11387 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11388 } else {
11389 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11390 }
11391 }
11392 if (!AllLocal && ScopeIsLocal)
11393 addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11394 return true;
11395 }
11396
11397 bool handlePHINode(
11398 Attributor &A, PHINode &PHI, ItemInfo II,
11399 SmallVectorImpl<ItemInfo> &Worklist,
11400 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11401 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11402 LivenessInfo &LI = LivenessAAs[&F];
11403 if (!LI.LivenessAA)
11404 LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11405 DepClass: DepClassTy::NONE);
11406 return LI;
11407 };
11408
11409 if (&PHI == &getAssociatedValue()) {
11410 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11411 const auto *CI =
11412 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11413 F: *PHI.getFunction());
11414
11415 CycleRef C;
11416 bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11417 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11418 BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11419 if (LI.LivenessAA &&
11420 LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11421 LI.AnyDead = true;
11422 continue;
11423 }
11424 Value *V = PHI.getIncomingValue(i: u);
11425 if (V == &PHI)
11426 continue;
11427
11428 // If the incoming value is not the PHI but an instruction in the same
11429 // cycle we might have multiple versions of it flying around.
11430 if (CyclePHI && isa<Instruction>(Val: V) &&
11431 (!C || CI->contains(C, Block: cast<Instruction>(Val: V)->getParent())))
11432 return false;
11433
11434 Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11435 }
11436 return true;
11437 }
11438
11439 bool UsedAssumedInformation = false;
11440 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11441 IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11442 if (!SimpleV.has_value())
11443 return true;
11444 if (!(*SimpleV))
11445 return false;
11446 addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11447 return true;
11448 }
11449
11450 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11451 /// simplify any operand of the instruction \p I. Return true if successful,
11452 /// in that case Worklist will be updated.
11453 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11454 SmallVectorImpl<ItemInfo> &Worklist) {
11455 bool SomeSimplified = false;
11456 bool UsedAssumedInformation = false;
11457
11458 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11459 int Idx = 0;
11460 for (Value *Op : I.operands()) {
11461 const auto &SimplifiedOp = A.getAssumedSimplified(
11462 IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11463 UsedAssumedInformation, S: AA::Intraprocedural);
11464 // If we are not sure about any operand we are not sure about the entire
11465 // instruction, we'll wait.
11466 if (!SimplifiedOp.has_value())
11467 return true;
11468
11469 if (*SimplifiedOp)
11470 NewOps[Idx] = *SimplifiedOp;
11471 else
11472 NewOps[Idx] = Op;
11473
11474 SomeSimplified |= (NewOps[Idx] != Op);
11475 ++Idx;
11476 }
11477
11478 // We won't bother with the InstSimplify interface if we didn't simplify any
11479 // operand ourselves.
11480 if (!SomeSimplified)
11481 return false;
11482
11483 InformationCache &InfoCache = A.getInfoCache();
11484 Function *F = I.getFunction();
11485 const auto *DT =
11486 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11487 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11488 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11489
11490 const DataLayout &DL = I.getDataLayout();
11491 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11492 Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11493 if (!NewV || NewV == &I)
11494 return false;
11495
11496 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11497 << *NewV << "\n");
11498 Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11499 return true;
11500 }
11501
11502 bool simplifyInstruction(
11503 Attributor &A, Instruction &I, ItemInfo II,
11504 SmallVectorImpl<ItemInfo> &Worklist,
11505 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11506 if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11507 return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11508 Pred: CI->getPredicate(), II, Worklist);
11509
11510 switch (I.getOpcode()) {
11511 case Instruction::Select:
11512 return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11513 case Instruction::PHI:
11514 return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11515 case Instruction::Load:
11516 return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11517 default:
11518 return handleGenericInst(A, I, II, Worklist);
11519 };
11520 return false;
11521 }
11522
11523 void genericValueTraversal(Attributor &A, Value *InitialV) {
11524 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11525
11526 SmallSet<ItemInfo, 16> Visited;
11527 SmallVector<ItemInfo, 16> Worklist;
11528 Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11529
11530 int Iteration = 0;
11531 do {
11532 ItemInfo II = Worklist.pop_back_val();
11533 Value *V = II.I.getValue();
11534 assert(V);
11535 const Instruction *CtxI = II.I.getCtxI();
11536 AA::ValueScope S = II.S;
11537
11538 // Check if we should process the current value. To prevent endless
11539 // recursion keep a record of the values we followed!
11540 if (!Visited.insert(V: II).second)
11541 continue;
11542
11543 // Make sure we limit the compile time for complex expressions.
11544 if (Iteration++ >= MaxPotentialValuesIterations) {
11545 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11546 << Iteration << "!\n");
11547 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11548 continue;
11549 }
11550
11551 // Explicitly look through calls with a "returned" attribute if we do
11552 // not have a pointer as stripPointerCasts only works on them.
11553 Value *NewV = nullptr;
11554 if (V->getType()->isPointerTy()) {
11555 NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11556 } else {
11557 if (auto *CB = dyn_cast<CallBase>(Val: V))
11558 if (auto *Callee =
11559 dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11560 for (Argument &Arg : Callee->args())
11561 if (Arg.hasReturnedAttr()) {
11562 NewV = CB->getArgOperand(i: Arg.getArgNo());
11563 break;
11564 }
11565 }
11566 }
11567 if (NewV && NewV != V) {
11568 Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11569 continue;
11570 }
11571
11572 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11573 if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11574 continue;
11575 }
11576
11577 if (V != InitialV || isa<Argument>(Val: V))
11578 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11579 continue;
11580
11581 // If we haven't stripped anything we give up.
11582 if (V == InitialV && CtxI == getCtxI()) {
11583 indicatePessimisticFixpoint();
11584 return;
11585 }
11586
11587 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11588 } while (!Worklist.empty());
11589
11590 // If we actually used liveness information so we have to record a
11591 // dependence.
11592 for (auto &It : LivenessAAs)
11593 if (It.second.AnyDead)
11594 A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11595 }
11596
11597 /// See AbstractAttribute::trackStatistics()
11598 void trackStatistics() const override {
11599 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11600 }
11601};
11602
11603struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11604 using Base = AAPotentialValuesImpl;
11605 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11606 : Base(IRP, A) {}
11607
11608 /// See AbstractAttribute::initialize(..).
11609 void initialize(Attributor &A) override {
11610 auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11611 if (Arg.hasPointeeInMemoryValueAttr())
11612 indicatePessimisticFixpoint();
11613 }
11614
11615 /// See AbstractAttribute::updateImpl(...).
11616 ChangeStatus updateImpl(Attributor &A) override {
11617 auto AssumedBefore = getAssumed();
11618
11619 unsigned ArgNo = getCalleeArgNo();
11620
11621 bool UsedAssumedInformation = false;
11622 SmallVector<AA::ValueAndContext> Values;
11623 auto CallSitePred = [&](AbstractCallSite ACS) {
11624 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11625 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11626 return false;
11627
11628 if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11629 S: AA::Interprocedural,
11630 UsedAssumedInformation))
11631 return false;
11632
11633 return isValidState();
11634 };
11635
11636 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11637 /* RequireAllCallSites */ true,
11638 UsedAssumedInformation))
11639 return indicatePessimisticFixpoint();
11640
11641 Function *Fn = getAssociatedFunction();
11642 bool AnyNonLocal = false;
11643 for (auto &It : Values) {
11644 if (isa<Constant>(Val: It.getValue())) {
11645 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11646 AnchorScope: getAnchorScope());
11647 continue;
11648 }
11649 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11650 return indicatePessimisticFixpoint();
11651
11652 if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11653 if (Arg->getParent() == Fn) {
11654 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11655 AnchorScope: getAnchorScope());
11656 continue;
11657 }
11658 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11659 AnchorScope: getAnchorScope());
11660 AnyNonLocal = true;
11661 }
11662 assert(!undefIsContained() && "Undef should be an explicit value!");
11663 if (AnyNonLocal)
11664 giveUpOnIntraprocedural(A);
11665
11666 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11667 : ChangeStatus::CHANGED;
11668 }
11669
11670 /// See AbstractAttribute::trackStatistics()
11671 void trackStatistics() const override {
11672 STATS_DECLTRACK_ARG_ATTR(potential_values)
11673 }
11674};
11675
11676struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11677 using Base = AAPotentialValuesFloating;
11678 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11679 : Base(IRP, A) {}
11680
11681 /// See AbstractAttribute::initialize(..).
11682 void initialize(Attributor &A) override {
11683 Function *F = getAssociatedFunction();
11684 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11685 indicatePessimisticFixpoint();
11686 return;
11687 }
11688
11689 for (Argument &Arg : F->args())
11690 if (Arg.hasReturnedAttr()) {
11691 addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11692 ReturnedArg = &Arg;
11693 break;
11694 }
11695 if (!A.isFunctionIPOAmendable(F: *F) ||
11696 A.hasSimplificationCallback(IRP: getIRPosition())) {
11697 if (!ReturnedArg)
11698 indicatePessimisticFixpoint();
11699 else
11700 indicateOptimisticFixpoint();
11701 }
11702 }
11703
11704 /// See AbstractAttribute::updateImpl(...).
11705 ChangeStatus updateImpl(Attributor &A) override {
11706 auto AssumedBefore = getAssumed();
11707 bool UsedAssumedInformation = false;
11708
11709 SmallVector<AA::ValueAndContext> Values;
11710 Function *AnchorScope = getAnchorScope();
11711 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11712 bool AddValues) {
11713 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11714 Values.clear();
11715 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11716 UsedAssumedInformation,
11717 /* RecurseForSelectAndPHI */ true))
11718 return false;
11719 if (!AddValues)
11720 continue;
11721
11722 bool AllInterAreIntra = false;
11723 if (S == AA::Interprocedural)
11724 AllInterAreIntra =
11725 llvm::all_of(Range&: Values, P: [&](const AA::ValueAndContext &VAC) {
11726 return AA::isValidInScope(V: *VAC.getValue(), Scope: AnchorScope);
11727 });
11728
11729 for (const AA::ValueAndContext &VAC : Values) {
11730 addValue(A, State&: getState(), V&: *VAC.getValue(),
11731 CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11732 S: AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11733 }
11734 if (AllInterAreIntra)
11735 break;
11736 }
11737 return true;
11738 };
11739
11740 if (ReturnedArg) {
11741 HandleReturnedValue(*ReturnedArg, nullptr, true);
11742 } else {
11743 auto RetInstPred = [&](Instruction &RetI) {
11744 bool AddValues = true;
11745 if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11746 isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11747 addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11748 AnchorScope);
11749 AddValues = false;
11750 }
11751 return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11752 };
11753
11754 if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11755 UsedAssumedInformation,
11756 /* CheckBBLivenessOnly */ true))
11757 return indicatePessimisticFixpoint();
11758 }
11759
11760 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11761 : ChangeStatus::CHANGED;
11762 }
11763
11764 ChangeStatus manifest(Attributor &A) override {
11765 if (ReturnedArg)
11766 return ChangeStatus::UNCHANGED;
11767 SmallVector<AA::ValueAndContext> Values;
11768 if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11769 /* RecurseForSelectAndPHI */ true))
11770 return ChangeStatus::UNCHANGED;
11771 Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11772 if (!NewVal)
11773 return ChangeStatus::UNCHANGED;
11774
11775 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11776 if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11777 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11778 "Number of function with unique return");
11779 Changed |= A.manifestAttrs(
11780 IRP: IRPosition::argument(Arg: *Arg),
11781 DeducedAttrs: {Attribute::get(Context&: Arg->getContext(), Kind: Attribute::Returned)});
11782 STATS_DECLTRACK_ARG_ATTR(returned);
11783 }
11784
11785 auto RetInstPred = [&](Instruction &RetI) {
11786 Value *RetOp = RetI.getOperand(i: 0);
11787 if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11788 return true;
11789 if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11790 if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11791 Changed = ChangeStatus::CHANGED;
11792 return true;
11793 };
11794 bool UsedAssumedInformation = false;
11795 (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11796 UsedAssumedInformation,
11797 /* CheckBBLivenessOnly */ true);
11798 return Changed;
11799 }
11800
11801 ChangeStatus indicatePessimisticFixpoint() override {
11802 return AAPotentialValues::indicatePessimisticFixpoint();
11803 }
11804
11805 /// See AbstractAttribute::trackStatistics()
11806 void trackStatistics() const override{
11807 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11808
11809 /// The argumented with an existing `returned` attribute.
11810 Argument *ReturnedArg = nullptr;
11811};
11812
11813struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11814 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11815 : AAPotentialValuesImpl(IRP, A) {}
11816
11817 /// See AbstractAttribute::updateImpl(...).
11818 ChangeStatus updateImpl(Attributor &A) override {
11819 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11820 "not be called");
11821 }
11822
11823 /// See AbstractAttribute::trackStatistics()
11824 void trackStatistics() const override {
11825 STATS_DECLTRACK_FN_ATTR(potential_values)
11826 }
11827};
11828
11829struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11830 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11831 : AAPotentialValuesFunction(IRP, A) {}
11832
11833 /// See AbstractAttribute::trackStatistics()
11834 void trackStatistics() const override {
11835 STATS_DECLTRACK_CS_ATTR(potential_values)
11836 }
11837};
11838
11839struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11840 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11841 : AAPotentialValuesImpl(IRP, A) {}
11842
11843 /// See AbstractAttribute::updateImpl(...).
11844 ChangeStatus updateImpl(Attributor &A) override {
11845 auto AssumedBefore = getAssumed();
11846
11847 Function *Callee = getAssociatedFunction();
11848 if (!Callee)
11849 return indicatePessimisticFixpoint();
11850
11851 bool UsedAssumedInformation = false;
11852 auto *CB = cast<CallBase>(Val: getCtxI());
11853 if (CB->isMustTailCall() &&
11854 !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11855 UsedAssumedInformation))
11856 return indicatePessimisticFixpoint();
11857
11858 Function *Caller = CB->getCaller();
11859
11860 auto AddScope = [&](AA::ValueScope S) {
11861 SmallVector<AA::ValueAndContext> Values;
11862 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11863 Values, S, UsedAssumedInformation))
11864 return false;
11865
11866 for (auto &It : Values) {
11867 Value *V = It.getValue();
11868 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11869 V, CB&: *CB, AA: *this, UsedAssumedInformation);
11870 if (!CallerV.has_value()) {
11871 // Nothing to do as long as no value was determined.
11872 continue;
11873 }
11874 V = *CallerV ? *CallerV : V;
11875 if (*CallerV && AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) {
11876 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S))
11877 continue;
11878 }
11879 if (S == AA::Intraprocedural && !AA::isValidInScope(V: *V, Scope: Caller)) {
11880 giveUpOnIntraprocedural(A);
11881 return true;
11882 }
11883 addValue(A, State&: getState(), V&: *V, CtxI: CB, S, AnchorScope: getAnchorScope());
11884 }
11885 return true;
11886 };
11887 if (!AddScope(AA::Intraprocedural))
11888 return indicatePessimisticFixpoint();
11889 if (!AddScope(AA::Interprocedural))
11890 return indicatePessimisticFixpoint();
11891 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11892 : ChangeStatus::CHANGED;
11893 }
11894
11895 ChangeStatus indicatePessimisticFixpoint() override {
11896 return AAPotentialValues::indicatePessimisticFixpoint();
11897 }
11898
11899 /// See AbstractAttribute::trackStatistics()
11900 void trackStatistics() const override {
11901 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11902 }
11903};
11904
11905struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11906 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11907 : AAPotentialValuesFloating(IRP, A) {}
11908
11909 /// See AbstractAttribute::trackStatistics()
11910 void trackStatistics() const override {
11911 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11912 }
11913};
11914} // namespace
11915
11916/// ---------------------- Assumption Propagation ------------------------------
11917namespace {
11918struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11919 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11920 const DenseSet<StringRef> &Known)
11921 : AAAssumptionInfo(IRP, A, Known) {}
11922
11923 /// See AbstractAttribute::manifest(...).
11924 ChangeStatus manifest(Attributor &A) override {
11925 // Don't manifest a universal set if it somehow made it here.
11926 if (getKnown().isUniversal())
11927 return ChangeStatus::UNCHANGED;
11928
11929 const IRPosition &IRP = getIRPosition();
11930 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11931 getAssumed().getSet().end());
11932 llvm::sort(C&: Set);
11933 return A.manifestAttrs(IRP,
11934 DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(),
11935 Kind: AssumptionAttrKey,
11936 Val: llvm::join(R&: Set, Separator: ",")),
11937 /*ForceReplace=*/true);
11938 }
11939
11940 bool hasAssumption(const StringRef Assumption) const override {
11941 return isValidState() && setContains(Assumption);
11942 }
11943
11944 /// See AbstractAttribute::getAsStr()
11945 const std::string getAsStr(Attributor *A) const override {
11946 const SetContents &Known = getKnown();
11947 const SetContents &Assumed = getAssumed();
11948
11949 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11950 llvm::sort(C&: Set);
11951 const std::string KnownStr = llvm::join(R&: Set, Separator: ",");
11952
11953 std::string AssumedStr = "Universal";
11954 if (!Assumed.isUniversal()) {
11955 Set.assign(in_start: Assumed.getSet().begin(), in_end: Assumed.getSet().end());
11956 AssumedStr = llvm::join(R&: Set, Separator: ",");
11957 }
11958 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11959 }
11960};
11961
11962/// Propagates assumption information from parent functions to all of their
11963/// successors. An assumption can be propagated if the containing function
11964/// dominates the called function.
11965///
11966/// We start with a "known" set of assumptions already valid for the associated
11967/// function and an "assumed" set that initially contains all possible
11968/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11969/// contents as concrete values are known. The concrete values are seeded by the
11970/// first nodes that are either entries into the call graph, or contains no
11971/// assumptions. Each node is updated as the intersection of the assumed state
11972/// with all of its predecessors.
11973struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11974 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11975 : AAAssumptionInfoImpl(IRP, A,
11976 getAssumptions(F: *IRP.getAssociatedFunction())) {}
11977
11978 /// See AbstractAttribute::updateImpl(...).
11979 ChangeStatus updateImpl(Attributor &A) override {
11980 bool Changed = false;
11981
11982 auto CallSitePred = [&](AbstractCallSite ACS) {
11983 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11984 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11985 DepClass: DepClassTy::REQUIRED);
11986 if (!AssumptionAA)
11987 return false;
11988 // Get the set of assumptions shared by all of this function's callers.
11989 Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11990 return !getAssumed().empty() || !getKnown().empty();
11991 };
11992
11993 bool UsedAssumedInformation = false;
11994 // Get the intersection of all assumptions held by this node's predecessors.
11995 // If we don't know all the call sites then this is either an entry into the
11996 // call graph or an empty node. This node is known to only contain its own
11997 // assumptions and can be propagated to its successors.
11998 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11999 UsedAssumedInformation))
12000 return indicatePessimisticFixpoint();
12001
12002 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12003 }
12004
12005 void trackStatistics() const override {}
12006};
12007
12008/// Assumption Info defined for call sites.
12009struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
12010
12011 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
12012 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
12013
12014 /// See AbstractAttribute::initialize(...).
12015 void initialize(Attributor &A) override {
12016 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
12017 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
12018 }
12019
12020 /// See AbstractAttribute::updateImpl(...).
12021 ChangeStatus updateImpl(Attributor &A) override {
12022 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
12023 auto *AssumptionAA =
12024 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
12025 if (!AssumptionAA)
12026 return indicatePessimisticFixpoint();
12027 bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
12028 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12029 }
12030
12031 /// See AbstractAttribute::trackStatistics()
12032 void trackStatistics() const override {}
12033
12034private:
12035 /// Helper to initialized the known set as all the assumptions this call and
12036 /// the callee contain.
12037 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
12038 const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
12039 auto Assumptions = getAssumptions(CB);
12040 if (const Function *F = CB.getCaller())
12041 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12042 if (Function *F = IRP.getAssociatedFunction())
12043 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12044 return Assumptions;
12045 }
12046};
12047} // namespace
12048
12049AACallGraphNode *AACallEdgeIterator::operator*() const {
12050 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
12051 A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
12052}
12053
12054void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
12055
12056/// ------------------------ UnderlyingObjects ---------------------------------
12057
12058namespace {
12059struct AAUnderlyingObjectsImpl
12060 : StateWrapper<BooleanState, AAUnderlyingObjects> {
12061 using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
12062 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
12063
12064 /// See AbstractAttribute::getAsStr().
12065 const std::string getAsStr(Attributor *A) const override {
12066 if (!isValidState())
12067 return "<invalid>";
12068 std::string Str;
12069 llvm::raw_string_ostream OS(Str);
12070 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
12071 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
12072 << " objects.\n";
12073 if (!InterAssumedUnderlyingObjects.empty()) {
12074 OS << "inter objects:\n";
12075 for (auto *Obj : InterAssumedUnderlyingObjects)
12076 OS << *Obj << '\n';
12077 }
12078 if (!IntraAssumedUnderlyingObjects.empty()) {
12079 OS << "intra objects:\n";
12080 for (auto *Obj : IntraAssumedUnderlyingObjects)
12081 OS << *Obj << '\n';
12082 }
12083 return Str;
12084 }
12085
12086 /// See AbstractAttribute::trackStatistics()
12087 void trackStatistics() const override {}
12088
12089 /// See AbstractAttribute::updateImpl(...).
12090 ChangeStatus updateImpl(Attributor &A) override {
12091 auto &Ptr = getAssociatedValue();
12092
12093 bool UsedAssumedInformation = false;
12094 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
12095 AA::ValueScope Scope) {
12096 SmallPtrSet<Value *, 8> SeenObjects;
12097 SmallVector<AA::ValueAndContext> Values;
12098
12099 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
12100 S: Scope, UsedAssumedInformation))
12101 return UnderlyingObjects.insert(X: &Ptr);
12102
12103 bool Changed = false;
12104
12105 for (unsigned I = 0; I < Values.size(); ++I) {
12106 auto &VAC = Values[I];
12107 auto *Obj = VAC.getValue();
12108 Value *UO = getUnderlyingObject(V: Obj);
12109 if (!SeenObjects.insert(Ptr: UO ? UO : Obj).second)
12110 continue;
12111 if (UO && UO != Obj) {
12112 if (isa<AllocaInst>(Val: UO) || isa<GlobalValue>(Val: UO)) {
12113 Changed |= UnderlyingObjects.insert(X: UO);
12114 continue;
12115 }
12116
12117 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12118 QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
12119 auto Pred = [&](Value &V) {
12120 if (&V == UO)
12121 Changed |= UnderlyingObjects.insert(X: UO);
12122 else
12123 Values.emplace_back(Args&: V, Args: nullptr);
12124 return true;
12125 };
12126
12127 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12128 llvm_unreachable(
12129 "The forall call should not return false at this position");
12130 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12131 continue;
12132 }
12133
12134 if (isa<SelectInst>(Val: Obj)) {
12135 Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope,
12136 UsedAssumedInformation);
12137 continue;
12138 }
12139 if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
12140 // Explicitly look through PHIs as we do not care about dynamically
12141 // uniqueness.
12142 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12143 Changed |=
12144 handleIndirect(A, V&: *PHI->getIncomingValue(i: u), UnderlyingObjects,
12145 Scope, UsedAssumedInformation);
12146 }
12147 continue;
12148 }
12149
12150 Changed |= UnderlyingObjects.insert(X: Obj);
12151 }
12152
12153 return Changed;
12154 };
12155
12156 bool Changed = false;
12157 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12158 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12159 if (!UsedAssumedInformation)
12160 indicateOptimisticFixpoint();
12161 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12162 }
12163
12164 bool forallUnderlyingObjects(
12165 function_ref<bool(Value &)> Pred,
12166 AA::ValueScope Scope = AA::Interprocedural) const override {
12167 if (!isValidState())
12168 return Pred(getAssociatedValue());
12169
12170 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12171 ? IntraAssumedUnderlyingObjects
12172 : InterAssumedUnderlyingObjects;
12173 for (Value *Obj : AssumedUnderlyingObjects)
12174 if (!Pred(*Obj))
12175 return false;
12176
12177 return true;
12178 }
12179
12180private:
12181 /// Handle the case where the value is not the actual underlying value, such
12182 /// as a phi node or a select instruction.
12183 bool handleIndirect(Attributor &A, Value &V,
12184 SmallSetVector<Value *, 8> &UnderlyingObjects,
12185 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12186 bool Changed = false;
12187 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12188 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
12189 auto Pred = [&](Value &V) {
12190 Changed |= UnderlyingObjects.insert(X: &V);
12191 return true;
12192 };
12193 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12194 llvm_unreachable(
12195 "The forall call should not return false at this position");
12196 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12197 return Changed;
12198 }
12199
12200 /// All the underlying objects collected so far via intra procedural scope.
12201 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12202 /// All the underlying objects collected so far via inter procedural scope.
12203 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12204};
12205
12206struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12207 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12208 : AAUnderlyingObjectsImpl(IRP, A) {}
12209};
12210
12211struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12212 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12213 : AAUnderlyingObjectsImpl(IRP, A) {}
12214};
12215
12216struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12217 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12218 : AAUnderlyingObjectsImpl(IRP, A) {}
12219};
12220
12221struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12222 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12223 : AAUnderlyingObjectsImpl(IRP, A) {}
12224};
12225
12226struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12227 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12228 : AAUnderlyingObjectsImpl(IRP, A) {}
12229};
12230
12231struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12232 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12233 : AAUnderlyingObjectsImpl(IRP, A) {}
12234};
12235
12236struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12237 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12238 : AAUnderlyingObjectsImpl(IRP, A) {}
12239};
12240} // namespace
12241
12242/// ------------------------ Global Value Info -------------------------------
12243namespace {
12244struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12245 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12246 : AAGlobalValueInfo(IRP, A) {}
12247
12248 /// See AbstractAttribute::initialize(...).
12249 void initialize(Attributor &A) override {}
12250
12251 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12252 SmallVectorImpl<const Value *> &Worklist) {
12253 Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12254 if (!UInst) {
12255 Follow = true;
12256 return true;
12257 }
12258
12259 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12260 << *UInst << "\n");
12261
12262 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12263 int Idx = &Cmp->getOperandUse(i: 0) == &U;
12264 if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12265 return true;
12266 return U == &getAnchorValue();
12267 }
12268
12269 // Explicitly catch return instructions.
12270 if (isa<ReturnInst>(Val: UInst)) {
12271 auto CallSitePred = [&](AbstractCallSite ACS) {
12272 Worklist.push_back(Elt: ACS.getInstruction());
12273 return true;
12274 };
12275 bool UsedAssumedInformation = false;
12276 // TODO: We should traverse the uses or add a "non-call-site" CB.
12277 if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12278 /*RequireAllCallSites=*/true, QueryingAA: this,
12279 UsedAssumedInformation))
12280 return false;
12281 return true;
12282 }
12283
12284 // For now we only use special logic for call sites. However, the tracker
12285 // itself knows about a lot of other non-capturing cases already.
12286 auto *CB = dyn_cast<CallBase>(Val: UInst);
12287 if (!CB)
12288 return false;
12289 // Direct calls are OK uses.
12290 if (CB->isCallee(U: &U))
12291 return true;
12292 // Non-argument uses are scary.
12293 if (!CB->isArgOperand(U: &U))
12294 return false;
12295 // TODO: Iterate callees.
12296 auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12297 if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12298 return false;
12299
12300 unsigned ArgNo = CB->getArgOperandNo(U: &U);
12301 Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12302 return true;
12303 }
12304
12305 ChangeStatus updateImpl(Attributor &A) override {
12306 unsigned NumUsesBefore = Uses.size();
12307
12308 SmallPtrSet<const Value *, 8> Visited;
12309 SmallVector<const Value *> Worklist;
12310 Worklist.push_back(Elt: &getAnchorValue());
12311
12312 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12313 Uses.insert(Ptr: &U);
12314 // TODO(captures): Make this more precise.
12315 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12316 if (CI.isPassthrough()) {
12317 Follow = true;
12318 return true;
12319 }
12320 return checkUse(A, U, Follow, Worklist);
12321 };
12322 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12323 Uses.insert(Ptr: &OldU);
12324 return true;
12325 };
12326
12327 while (!Worklist.empty()) {
12328 const Value *V = Worklist.pop_back_val();
12329 if (!Visited.insert(Ptr: V).second)
12330 continue;
12331 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12332 /* CheckBBLivenessOnly */ true,
12333 LivenessDepClass: DepClassTy::OPTIONAL,
12334 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12335 return indicatePessimisticFixpoint();
12336 }
12337 }
12338
12339 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12340 : ChangeStatus::CHANGED;
12341 }
12342
12343 bool isPotentialUse(const Use &U) const override {
12344 return !isValidState() || Uses.contains(Ptr: &U);
12345 }
12346
12347 /// See AbstractAttribute::manifest(...).
12348 ChangeStatus manifest(Attributor &A) override {
12349 return ChangeStatus::UNCHANGED;
12350 }
12351
12352 /// See AbstractAttribute::getAsStr().
12353 const std::string getAsStr(Attributor *A) const override {
12354 return "[" + std::to_string(val: Uses.size()) + " uses]";
12355 }
12356
12357 void trackStatistics() const override {
12358 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12359 }
12360
12361private:
12362 /// Set of (transitive) uses of this GlobalValue.
12363 SmallPtrSet<const Use *, 8> Uses;
12364};
12365} // namespace
12366
12367/// ------------------------ Indirect Call Info -------------------------------
12368namespace {
12369struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12370 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12371 : AAIndirectCallInfo(IRP, A) {}
12372
12373 /// See AbstractAttribute::initialize(...).
12374 void initialize(Attributor &A) override {
12375 auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12376 if (!MD && !A.isClosedWorldModule())
12377 return;
12378
12379 if (MD) {
12380 for (const auto &Op : MD->operands())
12381 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12382 PotentialCallees.insert(X: Callee);
12383 } else if (A.isClosedWorldModule()) {
12384 ArrayRef<Function *> IndirectlyCallableFunctions =
12385 A.getInfoCache().getIndirectlyCallableFunctions(A);
12386 PotentialCallees.insert_range(R&: IndirectlyCallableFunctions);
12387 }
12388
12389 if (PotentialCallees.empty())
12390 indicateOptimisticFixpoint();
12391 }
12392
12393 ChangeStatus updateImpl(Attributor &A) override {
12394 CallBase *CB = cast<CallBase>(Val: getCtxI());
12395 const Use &CalleeUse = CB->getCalledOperandUse();
12396 Value *FP = CB->getCalledOperand();
12397
12398 SmallSetVector<Function *, 4> AssumedCalleesNow;
12399 bool AllCalleesKnownNow = AllCalleesKnown;
12400
12401 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12402 bool &UsedAssumedInformation) {
12403 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12404 QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12405 if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12406 return true;
12407 UsedAssumedInformation = !GIAA->isAtFixpoint();
12408 return false;
12409 };
12410
12411 auto AddPotentialCallees = [&]() {
12412 for (auto *PotentialCallee : PotentialCallees) {
12413 bool UsedAssumedInformation = false;
12414 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12415 AssumedCalleesNow.insert(X: PotentialCallee);
12416 }
12417 };
12418
12419 // Use simplification to find potential callees, if !callees was present,
12420 // fallback to that set if necessary.
12421 bool UsedAssumedInformation = false;
12422 SmallVector<AA::ValueAndContext> Values;
12423 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12424 S: AA::ValueScope::AnyScope,
12425 UsedAssumedInformation)) {
12426 if (PotentialCallees.empty())
12427 return indicatePessimisticFixpoint();
12428 AddPotentialCallees();
12429 }
12430
12431 // Try to find a reason for \p Fn not to be a potential callee. If none was
12432 // found, add it to the assumed callees set.
12433 auto CheckPotentialCallee = [&](Function &Fn) {
12434 if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12435 return false;
12436
12437 auto &CachedResult = FilterResults[&Fn];
12438 if (CachedResult.has_value())
12439 return CachedResult.value();
12440
12441 bool UsedAssumedInformation = false;
12442 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12443 if (!UsedAssumedInformation)
12444 CachedResult = false;
12445 return false;
12446 }
12447
12448 int NumFnArgs = Fn.arg_size();
12449 int NumCBArgs = CB->arg_size();
12450
12451 // Check if any excess argument (which we fill up with poison) is known to
12452 // be UB on undef.
12453 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12454 bool IsKnown = false;
12455 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12456 A, QueryingAA: this, IRP: IRPosition::argument(Arg: *Fn.getArg(i: I)),
12457 DepClass: DepClassTy::OPTIONAL, IsKnown)) {
12458 if (IsKnown)
12459 CachedResult = false;
12460 return false;
12461 }
12462 }
12463
12464 CachedResult = true;
12465 return true;
12466 };
12467
12468 // Check simplification result, prune known UB callees, also restrict it to
12469 // the !callees set, if present.
12470 for (auto &VAC : Values) {
12471 if (isa<UndefValue>(Val: VAC.getValue()))
12472 continue;
12473 if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12474 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12475 continue;
12476 // TODO: Check for known UB, e.g., poison + noundef.
12477 if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12478 if (CheckPotentialCallee(*VACFn))
12479 AssumedCalleesNow.insert(X: VACFn);
12480 continue;
12481 }
12482 if (!PotentialCallees.empty()) {
12483 AddPotentialCallees();
12484 break;
12485 }
12486 AllCalleesKnownNow = false;
12487 }
12488
12489 if (AssumedCalleesNow == AssumedCallees &&
12490 AllCalleesKnown == AllCalleesKnownNow)
12491 return ChangeStatus::UNCHANGED;
12492
12493 std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12494 AllCalleesKnown = AllCalleesKnownNow;
12495 return ChangeStatus::CHANGED;
12496 }
12497
12498 /// See AbstractAttribute::manifest(...).
12499 ChangeStatus manifest(Attributor &A) override {
12500 // If we can't specialize at all, give up now.
12501 if (!AllCalleesKnown && AssumedCallees.empty())
12502 return ChangeStatus::UNCHANGED;
12503
12504 CallBase *CB = cast<CallBase>(Val: getCtxI());
12505 bool UsedAssumedInformation = false;
12506 if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12507 UsedAssumedInformation))
12508 return ChangeStatus::UNCHANGED;
12509
12510 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12511 Value *FP = CB->getCalledOperand();
12512 if (FP->getType()->getPointerAddressSpace())
12513 FP = new AddrSpaceCastInst(FP, PointerType::get(C&: FP->getContext(), AddressSpace: 0),
12514 FP->getName() + ".as0", CB->getIterator());
12515
12516 bool CBIsVoid = CB->getType()->isVoidTy();
12517 BasicBlock::iterator IP = CB->getIterator();
12518 FunctionType *CSFT = CB->getFunctionType();
12519 SmallVector<Value *> CSArgs(CB->args());
12520
12521 // If we know all callees and there are none, the call site is (effectively)
12522 // dead (or UB).
12523 if (AssumedCallees.empty()) {
12524 assert(AllCalleesKnown &&
12525 "Expected all callees to be known if there are none.");
12526 A.changeToUnreachableAfterManifest(I: CB);
12527 return ChangeStatus::CHANGED;
12528 }
12529
12530 // Special handling for the single callee case.
12531 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12532 auto *NewCallee = AssumedCallees.front();
12533 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12534 promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12535 NumIndirectCallsPromoted++;
12536 return ChangeStatus::CHANGED;
12537 }
12538 Instruction *NewCall =
12539 CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12540 NameStr: CB->getName(), InsertBefore: CB->getIterator());
12541 if (!CBIsVoid)
12542 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12543 A.deleteAfterManifest(I&: *CB);
12544 return ChangeStatus::CHANGED;
12545 }
12546
12547 // For each potential value we create a conditional
12548 //
12549 // ```
12550 // if (ptr == value) value(args);
12551 // else ...
12552 // ```
12553 //
12554 bool SpecializedForAnyCallees = false;
12555 bool SpecializedForAllCallees = AllCalleesKnown;
12556 ICmpInst *LastCmp = nullptr;
12557 SmallVector<Function *, 8> SkippedAssumedCallees;
12558 SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12559 for (Function *NewCallee : AssumedCallees) {
12560 if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee,
12561 NumAssumedCallees: AssumedCallees.size())) {
12562 SkippedAssumedCallees.push_back(Elt: NewCallee);
12563 SpecializedForAllCallees = false;
12564 continue;
12565 }
12566 SpecializedForAnyCallees = true;
12567
12568 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12569 Instruction *ThenTI =
12570 SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12571 BasicBlock *CBBB = CB->getParent();
12572 A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12573 A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12574 auto *SplitTI = cast<CondBrInst>(Val: LastCmp->getNextNode());
12575 BasicBlock *ElseBB;
12576 if (&*IP == CB) {
12577 ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12578 Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12579 A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12580 IP = UncondBrInst::Create(Target: CBBB, InsertBefore: ElseBB)->getIterator();
12581 SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12582 } else {
12583 ElseBB = IP->getParent();
12584 ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12585 }
12586 CastInst *RetBC = nullptr;
12587 CallInst *NewCall = nullptr;
12588 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12589 auto *CBClone = cast<CallBase>(Val: CB->clone());
12590 CBClone->insertBefore(InsertPos: ThenTI->getIterator());
12591 NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12592 NumIndirectCallsPromoted++;
12593 } else {
12594 NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12595 NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12596 }
12597 NewCalls.push_back(Elt: {NewCall, RetBC});
12598 }
12599
12600 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12601 if (!AllCalleesKnown)
12602 return ChangeStatus::UNCHANGED;
12603 MDBuilder MDB(IndirectCB.getContext());
12604 MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12605 IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12606 return ChangeStatus::CHANGED;
12607 };
12608
12609 if (!SpecializedForAnyCallees)
12610 return AttachCalleeMetadata(*CB);
12611
12612 // Check if we need the fallback indirect call still.
12613 if (SpecializedForAllCallees) {
12614 LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12615 LastCmp->eraseFromParent();
12616 new UnreachableInst(IP->getContext(), IP);
12617 IP->eraseFromParent();
12618 } else {
12619 auto *CBClone = cast<CallInst>(Val: CB->clone());
12620 CBClone->setName(CB->getName());
12621 CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12622 NewCalls.push_back(Elt: {CBClone, nullptr});
12623 AttachCalleeMetadata(*CBClone);
12624 }
12625
12626 // Check if we need a PHI to merge the results.
12627 if (!CBIsVoid) {
12628 auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12629 NameStr: CB->getName() + ".phi",
12630 InsertBefore: CB->getParent()->getFirstInsertionPt());
12631 for (auto &It : NewCalls) {
12632 CallBase *NewCall = It.first;
12633 Instruction *CallRet = It.second ? It.second : It.first;
12634 if (CallRet->getType() == CB->getType())
12635 PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12636 else if (NewCall->getType()->isVoidTy())
12637 PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12638 BB: NewCall->getParent());
12639 else
12640 llvm_unreachable("Call return should match or be void!");
12641 }
12642 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12643 }
12644
12645 A.deleteAfterManifest(I&: *CB);
12646 Changed = ChangeStatus::CHANGED;
12647
12648 return Changed;
12649 }
12650
12651 /// See AbstractAttribute::getAsStr().
12652 const std::string getAsStr(Attributor *A) const override {
12653 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12654 " indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12655 " functions";
12656 }
12657
12658 void trackStatistics() const override {
12659 if (AllCalleesKnown) {
12660 STATS_DECLTRACK(
12661 Eliminated, CallSites,
12662 "Number of indirect call sites eliminated via specialization")
12663 } else {
12664 STATS_DECLTRACK(Specialized, CallSites,
12665 "Number of indirect call sites specialized")
12666 }
12667 }
12668
12669 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12670 return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12671 }
12672
12673private:
12674 /// Map to remember filter results.
12675 DenseMap<Function *, std::optional<bool>> FilterResults;
12676
12677 /// If the !callee metadata was present, this set will contain all potential
12678 /// callees (superset).
12679 SmallSetVector<Function *, 4> PotentialCallees;
12680
12681 /// This set contains all currently assumed calllees, which might grow over
12682 /// time.
12683 SmallSetVector<Function *, 4> AssumedCallees;
12684
12685 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12686 /// if there could be others.
12687 bool AllCalleesKnown = true;
12688};
12689} // namespace
12690
12691/// --------------------- Invariant Load Pointer -------------------------------
12692namespace {
12693
12694struct AAInvariantLoadPointerImpl
12695 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12696 AAInvariantLoadPointer> {
12697
12698 enum {
12699 // pointer does not alias within the bounds of the function
12700 IS_NOALIAS = 1 << 0,
12701 // pointer is not involved in any effectful instructions within the bounds
12702 // of the function
12703 IS_NOEFFECT = 1 << 1,
12704 // loads are invariant within the bounds of the function
12705 IS_LOCALLY_INVARIANT = 1 << 2,
12706 // memory lifetime is constrained within the bounds of the function
12707 IS_LOCALLY_CONSTRAINED = 1 << 3,
12708
12709 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12710 IS_LOCALLY_CONSTRAINED,
12711 };
12712 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12713
12714 using Base =
12715 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12716
12717 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12718 // pessimistic about IS_KNOWN_INVARIANT
12719 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12720 : Base(IRP) {}
12721
12722 bool isKnownInvariant() const final {
12723 return isKnownLocallyInvariant() && isKnown(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12724 }
12725
12726 bool isKnownLocallyInvariant() const final {
12727 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT))
12728 return true;
12729 return isKnown(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12730 }
12731
12732 bool isAssumedInvariant() const final {
12733 return isAssumedLocallyInvariant() && isAssumed(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12734 }
12735
12736 bool isAssumedLocallyInvariant() const final {
12737 if (isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12738 return true;
12739 return isAssumed(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12740 }
12741
12742 ChangeStatus updateImpl(Attributor &A) override {
12743 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12744
12745 Changed |= updateNoAlias(A);
12746 if (requiresNoAlias() && !isAssumed(BitsEncoding: IS_NOALIAS))
12747 return indicatePessimisticFixpoint();
12748
12749 Changed |= updateNoEffect(A);
12750
12751 Changed |= updateLocalInvariance(A);
12752
12753 return Changed;
12754 }
12755
12756 ChangeStatus manifest(Attributor &A) override {
12757 if (!isKnownInvariant())
12758 return ChangeStatus::UNCHANGED;
12759
12760 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12761 const Value *Ptr = &getAssociatedValue();
12762 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12763 if (U.get() != Ptr)
12764 return true;
12765 auto *I = dyn_cast<Instruction>(Val: U.getUser());
12766 if (!I)
12767 return true;
12768
12769 // Ensure that we are only changing uses from the corresponding callgraph
12770 // SSC in the case that the AA isn't run on the entire module
12771 if (!A.isRunOn(Fn: I->getFunction()))
12772 return true;
12773
12774 if (I->hasMetadata(KindID: LLVMContext::MD_invariant_load))
12775 return true;
12776
12777 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
12778 LI->setMetadata(KindID: LLVMContext::MD_invariant_load,
12779 Node: MDNode::get(Context&: LI->getContext(), MDs: {}));
12780 Changed = ChangeStatus::CHANGED;
12781 }
12782 return true;
12783 };
12784
12785 (void)A.checkForAllUses(Pred: TagInvariantLoads, QueryingAA: *this, V: *Ptr);
12786 return Changed;
12787 }
12788
12789 /// See AbstractAttribute::getAsStr().
12790 const std::string getAsStr(Attributor *) const override {
12791 if (isKnownInvariant())
12792 return "load-invariant pointer";
12793 return "non-invariant pointer";
12794 }
12795
12796 /// See AbstractAttribute::trackStatistics().
12797 void trackStatistics() const override {}
12798
12799private:
12800 /// Indicate that noalias is required for the pointer to be invariant.
12801 bool requiresNoAlias() const {
12802 switch (getPositionKind()) {
12803 default:
12804 // Conservatively default to require noalias.
12805 return true;
12806 case IRP_FLOAT:
12807 case IRP_RETURNED:
12808 case IRP_CALL_SITE:
12809 return false;
12810 case IRP_CALL_SITE_RETURNED: {
12811 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12812 return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12813 Call: &CB, /*MustPreserveOffset=*/false);
12814 }
12815 case IRP_ARGUMENT: {
12816 const Function *F = getAssociatedFunction();
12817 assert(F && "no associated function for argument");
12818 return !isCallableCC(CC: F->getCallingConv());
12819 }
12820 }
12821 }
12822
12823 bool isExternal() const {
12824 const Function *F = getAssociatedFunction();
12825 if (!F)
12826 return true;
12827 return isCallableCC(CC: F->getCallingConv()) &&
12828 getPositionKind() != IRP_CALL_SITE_RETURNED;
12829 }
12830
12831 ChangeStatus updateNoAlias(Attributor &A) {
12832 if (isKnown(BitsEncoding: IS_NOALIAS) || !isAssumed(BitsEncoding: IS_NOALIAS))
12833 return ChangeStatus::UNCHANGED;
12834
12835 // Try to use AANoAlias.
12836 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12837 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12838 if (ANoAlias->isKnownNoAlias()) {
12839 addKnownBits(Bits: IS_NOALIAS);
12840 return ChangeStatus::CHANGED;
12841 }
12842
12843 if (!ANoAlias->isAssumedNoAlias()) {
12844 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12845 return ChangeStatus::CHANGED;
12846 }
12847
12848 return ChangeStatus::UNCHANGED;
12849 }
12850
12851 // Try to infer noalias from argument attribute, since it is applicable for
12852 // the duration of the function.
12853 if (const Argument *Arg = getAssociatedArgument()) {
12854 if (Arg->hasNoAliasAttr()) {
12855 addKnownBits(Bits: IS_NOALIAS);
12856 return ChangeStatus::UNCHANGED;
12857 }
12858
12859 // Noalias information is not provided, and cannot be inferred,
12860 // so we conservatively assume the pointer aliases.
12861 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12862 return ChangeStatus::CHANGED;
12863 }
12864
12865 return ChangeStatus::UNCHANGED;
12866 }
12867
12868 ChangeStatus updateNoEffect(Attributor &A) {
12869 if (isKnown(BitsEncoding: IS_NOEFFECT) || !isAssumed(BitsEncoding: IS_NOEFFECT))
12870 return ChangeStatus::UNCHANGED;
12871
12872 if (!getAssociatedFunction())
12873 return indicatePessimisticFixpoint();
12874
12875 if (isa<AllocaInst>(Val: &getAssociatedValue()))
12876 return indicatePessimisticFixpoint();
12877
12878 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12879 const auto *LI = dyn_cast<LoadInst>(Val: U.getUser());
12880 return !LI || !LI->mayHaveSideEffects();
12881 };
12882 if (!A.checkForAllUses(Pred: HasNoEffectLoads, QueryingAA: *this, V: getAssociatedValue()))
12883 return indicatePessimisticFixpoint();
12884
12885 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12886 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12887 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12888 // attribute
12889 if (!AMemoryBehavior->isAssumedReadOnly())
12890 return indicatePessimisticFixpoint();
12891
12892 if (AMemoryBehavior->isKnownReadOnly()) {
12893 addKnownBits(Bits: IS_NOEFFECT);
12894 return ChangeStatus::UNCHANGED;
12895 }
12896
12897 return ChangeStatus::UNCHANGED;
12898 }
12899
12900 if (const Argument *Arg = getAssociatedArgument()) {
12901 if (Arg->onlyReadsMemory()) {
12902 addKnownBits(Bits: IS_NOEFFECT);
12903 return ChangeStatus::UNCHANGED;
12904 }
12905
12906 // Readonly information is not provided, and cannot be inferred from
12907 // AAMemoryBehavior.
12908 return indicatePessimisticFixpoint();
12909 }
12910
12911 return ChangeStatus::UNCHANGED;
12912 }
12913
12914 ChangeStatus updateLocalInvariance(Attributor &A) {
12915 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT) || !isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12916 return ChangeStatus::UNCHANGED;
12917
12918 // try to infer invariance from underlying objects
12919 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12920 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
12921 if (!AUO)
12922 return ChangeStatus::UNCHANGED;
12923
12924 bool UsedAssumedInformation = false;
12925 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12926 if (!V.getType()->isPointerTy())
12927 return true;
12928 const auto *IsInvariantLoadPointer =
12929 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRP: IRPosition::value(V), QueryingAA: this,
12930 DepClass: DepClassTy::REQUIRED);
12931 // Conservatively fail if invariance cannot be inferred.
12932 if (!IsInvariantLoadPointer)
12933 return false;
12934
12935 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12936 return true;
12937 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12938 return false;
12939
12940 UsedAssumedInformation = true;
12941 return true;
12942 };
12943 if (!AUO->forallUnderlyingObjects(Pred: IsLocallyInvariantLoadIfPointer))
12944 return indicatePessimisticFixpoint();
12945
12946 if (const auto *CB = dyn_cast<CallBase>(Val: &getAnchorValue())) {
12947 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12948 Call: CB, /*MustPreserveOffset=*/false)) {
12949 for (const Value *Arg : CB->args()) {
12950 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12951 return indicatePessimisticFixpoint();
12952 }
12953 }
12954 }
12955
12956 if (!UsedAssumedInformation) {
12957 // Pointer is known and not just assumed to be locally invariant.
12958 addKnownBits(Bits: IS_LOCALLY_INVARIANT);
12959 return ChangeStatus::CHANGED;
12960 }
12961
12962 return ChangeStatus::UNCHANGED;
12963 }
12964};
12965
12966struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12967 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12968 : AAInvariantLoadPointerImpl(IRP, A) {}
12969};
12970
12971struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12972 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12973 : AAInvariantLoadPointerImpl(IRP, A) {}
12974
12975 void initialize(Attributor &) override {
12976 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12977 }
12978};
12979
12980struct AAInvariantLoadPointerCallSiteReturned final
12981 : AAInvariantLoadPointerImpl {
12982 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12983 : AAInvariantLoadPointerImpl(IRP, A) {}
12984
12985 void initialize(Attributor &A) override {
12986 const Function *F = getAssociatedFunction();
12987 assert(F && "no associated function for return from call");
12988
12989 if (!F->isDeclaration() && !F->isIntrinsic())
12990 return AAInvariantLoadPointerImpl::initialize(A);
12991
12992 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12993 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12994 Call: &CB, /*MustPreserveOffset=*/false))
12995 return AAInvariantLoadPointerImpl::initialize(A);
12996
12997 if (F->onlyReadsMemory() && F->hasNoSync())
12998 return AAInvariantLoadPointerImpl::initialize(A);
12999
13000 // At this point, the function is opaque, so we conservatively assume
13001 // non-invariance.
13002 indicatePessimisticFixpoint();
13003 }
13004};
13005
13006struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
13007 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
13008 : AAInvariantLoadPointerImpl(IRP, A) {}
13009
13010 void initialize(Attributor &) override {
13011 const Function *F = getAssociatedFunction();
13012 assert(F && "no associated function for argument");
13013
13014 if (!isCallableCC(CC: F->getCallingConv())) {
13015 addKnownBits(Bits: IS_LOCALLY_CONSTRAINED);
13016 return;
13017 }
13018
13019 if (!F->hasLocalLinkage())
13020 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
13021 }
13022};
13023
13024struct AAInvariantLoadPointerCallSiteArgument final
13025 : AAInvariantLoadPointerImpl {
13026 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
13027 : AAInvariantLoadPointerImpl(IRP, A) {}
13028};
13029} // namespace
13030
13031/// ------------------------ Address Space ------------------------------------
13032namespace {
13033
13034template <typename InstType>
13035static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
13036 Value *OriginalValue, PointerType *NewPtrTy,
13037 bool UseOriginalValue) {
13038 if (U.getOperandNo() != InstType::getPointerOperandIndex())
13039 return false;
13040
13041 if (MemInst->isVolatile()) {
13042 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
13043 *MemInst->getFunction());
13044 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
13045 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
13046 return false;
13047 }
13048
13049 if (UseOriginalValue) {
13050 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *OriginalValue);
13051 return true;
13052 }
13053
13054 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
13055 CastInst->insertBefore(MemInst->getIterator());
13056 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *CastInst);
13057 return true;
13058}
13059
13060struct AAAddressSpaceImpl : public AAAddressSpace {
13061 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
13062 : AAAddressSpace(IRP, A) {}
13063
13064 uint32_t getAddressSpace() const override {
13065 assert(isValidState() && "the AA is invalid");
13066 return AssumedAddressSpace;
13067 }
13068
13069 /// See AbstractAttribute::initialize(...).
13070 void initialize(Attributor &A) override {
13071 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13072 "Associated value is not a pointer");
13073
13074 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
13075 indicatePessimisticFixpoint();
13076 return;
13077 }
13078
13079 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13080 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13081 if (AS != FlatAS) {
13082 [[maybe_unused]] bool R = takeAddressSpace(AS);
13083 assert(R && "The take should happen");
13084 indicateOptimisticFixpoint();
13085 }
13086 }
13087
13088 ChangeStatus updateImpl(Attributor &A) override {
13089 uint32_t OldAddressSpace = AssumedAddressSpace;
13090 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13091
13092 auto CheckAddressSpace = [&](Value &Obj) {
13093 // Ignore undef.
13094 if (isa<UndefValue>(Val: &Obj))
13095 return true;
13096
13097 // If the object already has a non-flat address space, we simply take it.
13098 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
13099 if (ObjAS != FlatAS)
13100 return takeAddressSpace(AS: ObjAS);
13101
13102 // At this point, we know Obj is in the flat address space. For a final
13103 // attempt, we want to use getAssumedAddrSpace, but first we must get the
13104 // associated function, if possible.
13105 Function *F = nullptr;
13106 if (auto *Arg = dyn_cast<Argument>(Val: &Obj))
13107 F = Arg->getParent();
13108 else if (auto *I = dyn_cast<Instruction>(Val: &Obj))
13109 F = I->getFunction();
13110
13111 // Use getAssumedAddrSpace if the associated function exists.
13112 if (F) {
13113 auto *TTI =
13114 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: *F);
13115 unsigned AssumedAS = TTI->getAssumedAddrSpace(V: &Obj);
13116 if (AssumedAS != ~0U)
13117 return takeAddressSpace(AS: AssumedAS);
13118 }
13119
13120 // Now we can't do anything else but to take the flat AS.
13121 return takeAddressSpace(AS: FlatAS);
13122 };
13123
13124 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
13125 DepClass: DepClassTy::REQUIRED);
13126 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13127 return indicatePessimisticFixpoint();
13128
13129 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13130 : ChangeStatus::CHANGED;
13131 }
13132
13133 /// See AbstractAttribute::manifest(...).
13134 ChangeStatus manifest(Attributor &A) override {
13135 unsigned NewAS = getAddressSpace();
13136
13137 if (NewAS == InvalidAddressSpace ||
13138 NewAS == getAssociatedType()->getPointerAddressSpace())
13139 return ChangeStatus::UNCHANGED;
13140
13141 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13142
13143 Value *AssociatedValue = &getAssociatedValue();
13144 Value *OriginalValue = peelAddrspacecast(V: AssociatedValue, FlatAS);
13145
13146 PointerType *NewPtrTy =
13147 PointerType::get(C&: getAssociatedType()->getContext(), AddressSpace: NewAS);
13148 bool UseOriginalValue =
13149 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13150
13151 bool Changed = false;
13152
13153 auto Pred = [&](const Use &U, bool &) {
13154 if (U.get() != AssociatedValue)
13155 return true;
13156 auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
13157 if (!Inst)
13158 return true;
13159 // This is a WA to make sure we only change uses from the corresponding
13160 // CGSCC if the AA is run on CGSCC instead of the entire module.
13161 if (!A.isRunOn(Fn: Inst->getFunction()))
13162 return true;
13163 if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
13164 Changed |=
13165 makeChange(A, MemInst: LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13166 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
13167 Changed |=
13168 makeChange(A, MemInst: SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13169 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
13170 Changed |=
13171 makeChange(A, MemInst: RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13172 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
13173 Changed |=
13174 makeChange(A, MemInst: CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13175 }
13176 return true;
13177 };
13178
13179 // It doesn't matter if we can't check all uses as we can simply
13180 // conservatively ignore those that can not be visited.
13181 (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
13182 /* CheckBBLivenessOnly */ true);
13183
13184 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13185 }
13186
13187 /// See AbstractAttribute::getAsStr().
13188 const std::string getAsStr(Attributor *A) const override {
13189 if (!isValidState())
13190 return "addrspace(<invalid>)";
13191 return "addrspace(" +
13192 (AssumedAddressSpace == InvalidAddressSpace
13193 ? "none"
13194 : std::to_string(val: AssumedAddressSpace)) +
13195 ")";
13196 }
13197
13198private:
13199 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13200
13201 bool takeAddressSpace(uint32_t AS) {
13202 if (AssumedAddressSpace == InvalidAddressSpace) {
13203 AssumedAddressSpace = AS;
13204 return true;
13205 }
13206 return AssumedAddressSpace == AS;
13207 }
13208
13209 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13210 if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) {
13211 assert(I->getSrcAddressSpace() != FlatAS &&
13212 "there should not be flat AS -> non-flat AS");
13213 return I->getPointerOperand();
13214 }
13215 if (auto *C = dyn_cast<ConstantExpr>(Val: V))
13216 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13217 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13218 FlatAS &&
13219 "there should not be flat AS -> non-flat AS X");
13220 return C->getOperand(i_nocapture: 0);
13221 }
13222 return V;
13223 }
13224};
13225
13226struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13227 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13228 : AAAddressSpaceImpl(IRP, A) {}
13229
13230 void trackStatistics() const override {
13231 STATS_DECLTRACK_FLOATING_ATTR(addrspace);
13232 }
13233};
13234
13235struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13236 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13237 : AAAddressSpaceImpl(IRP, A) {}
13238
13239 /// See AbstractAttribute::initialize(...).
13240 void initialize(Attributor &A) override {
13241 // TODO: we don't rewrite function argument for now because it will need to
13242 // rewrite the function signature and all call sites.
13243 (void)indicatePessimisticFixpoint();
13244 }
13245
13246 void trackStatistics() const override {
13247 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13248 }
13249};
13250
13251struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13252 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13253 : AAAddressSpaceImpl(IRP, A) {}
13254
13255 void trackStatistics() const override {
13256 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13257 }
13258};
13259
13260struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13261 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13262 : AAAddressSpaceImpl(IRP, A) {}
13263
13264 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13265};
13266
13267struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13268 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13269 : AAAddressSpaceImpl(IRP, A) {}
13270
13271 /// See AbstractAttribute::initialize(...).
13272 void initialize(Attributor &A) override {
13273 // TODO: we don't rewrite call site argument for now because it will need to
13274 // rewrite the function signature of the callee.
13275 (void)indicatePessimisticFixpoint();
13276 }
13277
13278 void trackStatistics() const override {
13279 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13280 }
13281};
13282} // namespace
13283
13284/// ------------------------ No Alias Address Space ---------------------------
13285// This attribute assumes flat address space can alias all other address space
13286
13287// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13288// But merging it created failing cased on gateway test that cannot be
13289// reproduced locally. So should open a separated PR to handle the merge of
13290// AANoAliasAddrSpace and AAAddressSpace attribute
13291
13292namespace {
13293struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13294 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13295 : AANoAliasAddrSpace(IRP, A) {}
13296
13297 void initialize(Attributor &A) override {
13298 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13299 "Associated value is not a pointer");
13300
13301 resetASRanges(A);
13302
13303 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13304 if (!FlatAS.has_value()) {
13305 indicatePessimisticFixpoint();
13306 return;
13307 }
13308
13309 removeAS(AS: *FlatAS);
13310
13311 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13312 if (AS != *FlatAS) {
13313 removeAS(AS);
13314 indicateOptimisticFixpoint();
13315 }
13316 }
13317
13318 ChangeStatus updateImpl(Attributor &A) override {
13319 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13320 uint32_t OldAssumed = getAssumed();
13321
13322 auto CheckAddressSpace = [&](Value &Obj) {
13323 if (isa<PoisonValue>(Val: &Obj))
13324 return true;
13325
13326 unsigned AS = Obj.getType()->getPointerAddressSpace();
13327 if (AS == FlatAS)
13328 return false;
13329
13330 removeAS(AS: Obj.getType()->getPointerAddressSpace());
13331 return true;
13332 };
13333
13334 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13335 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
13336 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13337 return indicatePessimisticFixpoint();
13338
13339 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13340 : ChangeStatus::CHANGED;
13341 }
13342
13343 /// See AbstractAttribute::manifest(...).
13344 ChangeStatus manifest(Attributor &A) override {
13345 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13346
13347 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13348 if (AS != FlatAS || Map.empty())
13349 return ChangeStatus::UNCHANGED;
13350
13351 LLVMContext &Ctx = getAssociatedValue().getContext();
13352 MDNode *NoAliasASNode = nullptr;
13353 MDBuilder MDB(Ctx);
13354 // Has to use iterator to get the range info.
13355 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13356 if (!I.value())
13357 continue;
13358 unsigned Upper = I.stop();
13359 unsigned Lower = I.start();
13360 if (!NoAliasASNode) {
13361 NoAliasASNode = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13362 continue;
13363 }
13364 MDNode *ASRange = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13365 NoAliasASNode = MDNode::getMostGenericRange(A: NoAliasASNode, B: ASRange);
13366 }
13367
13368 Value *AssociatedValue = &getAssociatedValue();
13369 bool Changed = false;
13370
13371 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13372 if (U.get() != AssociatedValue)
13373 return true;
13374 Instruction *Inst = dyn_cast<Instruction>(Val: U.getUser());
13375 if (!Inst || Inst->hasMetadata(KindID: LLVMContext::MD_noalias_addrspace))
13376 return true;
13377 if (!isa<LoadInst>(Val: Inst) && !isa<StoreInst>(Val: Inst) &&
13378 !isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
13379 return true;
13380 if (!A.isRunOn(Fn: Inst->getFunction()))
13381 return true;
13382 Inst->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: NoAliasASNode);
13383 Changed = true;
13384 return true;
13385 };
13386 (void)A.checkForAllUses(Pred: AddNoAliasAttr, QueryingAA: *this, V: *AssociatedValue,
13387 /*CheckBBLivenessOnly=*/true);
13388 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13389 }
13390
13391 /// See AbstractAttribute::getAsStr().
13392 const std::string getAsStr(Attributor *A) const override {
13393 if (!isValidState())
13394 return "<invalid>";
13395 std::string Str;
13396 raw_string_ostream OS(Str);
13397 OS << "CanNotBeAddrSpace(";
13398 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13399 unsigned Upper = I.stop();
13400 unsigned Lower = I.start();
13401 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13402 }
13403 OS << " )";
13404 return OS.str();
13405 }
13406
13407private:
13408 void removeAS(unsigned AS) {
13409 RangeMap::iterator I = Map.find(x: AS);
13410
13411 if (I != Map.end()) {
13412 unsigned Upper = I.stop();
13413 unsigned Lower = I.start();
13414 I.erase();
13415 if (Upper == Lower)
13416 return;
13417 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13418 Map.insert(a: AS + 1, b: Upper, /*what ever this variable name is=*/y: true);
13419 if (AS != 0 && Lower <= AS - 1)
13420 Map.insert(a: Lower, b: AS - 1, y: true);
13421 }
13422 }
13423
13424 void resetASRanges(Attributor &A) {
13425 Map.clear();
13426 Map.insert(a: 0, b: A.getInfoCache().getMaxAddrSpace(), y: true);
13427 }
13428};
13429
13430struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13431 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13432 : AANoAliasAddrSpaceImpl(IRP, A) {}
13433
13434 void trackStatistics() const override {
13435 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13436 }
13437};
13438
13439struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13440 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13441 : AANoAliasAddrSpaceImpl(IRP, A) {}
13442
13443 void trackStatistics() const override {
13444 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13445 }
13446};
13447
13448struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13449 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13450 : AANoAliasAddrSpaceImpl(IRP, A) {}
13451
13452 void trackStatistics() const override {
13453 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13454 }
13455};
13456
13457struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13458 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13459 : AANoAliasAddrSpaceImpl(IRP, A) {}
13460
13461 void trackStatistics() const override {
13462 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13463 }
13464};
13465
13466struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13467 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13468 : AANoAliasAddrSpaceImpl(IRP, A) {}
13469
13470 void trackStatistics() const override {
13471 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13472 }
13473};
13474} // namespace
13475/// ----------- Allocation Info ----------
13476namespace {
13477struct AAAllocationInfoImpl : public AAAllocationInfo {
13478 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13479 : AAAllocationInfo(IRP, A) {}
13480
13481 std::optional<TypeSize> getAllocatedSize() const override {
13482 assert(isValidState() && "the AA is invalid");
13483 return AssumedAllocatedSize;
13484 }
13485
13486 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13487 const DataLayout &DL) {
13488
13489 // TODO: implement case for malloc like instructions
13490 switch (I->getOpcode()) {
13491 case Instruction::Alloca: {
13492 AllocaInst *AI = cast<AllocaInst>(Val: I);
13493 return AI->getAllocationSize(DL);
13494 }
13495 default:
13496 return std::nullopt;
13497 }
13498 }
13499
13500 ChangeStatus updateImpl(Attributor &A) override {
13501
13502 const IRPosition &IRP = getIRPosition();
13503 Instruction *I = IRP.getCtxI();
13504
13505 // TODO: update check for malloc like calls
13506 if (!isa<AllocaInst>(Val: I))
13507 return indicatePessimisticFixpoint();
13508
13509 bool IsKnownNoCapture;
13510 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
13511 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
13512 return indicatePessimisticFixpoint();
13513
13514 const AAPointerInfo *PI =
13515 A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
13516
13517 if (!PI)
13518 return indicatePessimisticFixpoint();
13519
13520 if (!PI->getState().isValidState() || PI->reachesReturn())
13521 return indicatePessimisticFixpoint();
13522
13523 const DataLayout &DL = A.getDataLayout();
13524 const auto AllocationSize = findInitialAllocationSize(I, DL);
13525
13526 // If allocation size is nullopt, we give up.
13527 if (!AllocationSize)
13528 return indicatePessimisticFixpoint();
13529
13530 // For zero sized allocations, we give up.
13531 // Since we can't reduce further
13532 if (*AllocationSize == 0)
13533 return indicatePessimisticFixpoint();
13534
13535 int64_t BinSize = PI->numOffsetBins();
13536
13537 // TODO: implement for multiple bins
13538 if (BinSize > 1)
13539 return indicatePessimisticFixpoint();
13540
13541 if (BinSize == 0) {
13542 auto NewAllocationSize = std::make_optional<TypeSize>(args: 0, args: false);
13543 if (!changeAllocationSize(Size: NewAllocationSize))
13544 return ChangeStatus::UNCHANGED;
13545 return ChangeStatus::CHANGED;
13546 }
13547
13548 // TODO: refactor this to be part of multiple bin case
13549 const auto &It = PI->begin();
13550
13551 // TODO: handle if Offset is not zero
13552 if (It->first.Offset != 0)
13553 return indicatePessimisticFixpoint();
13554
13555 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13556
13557 if (SizeOfBin >= *AllocationSize)
13558 return indicatePessimisticFixpoint();
13559
13560 auto NewAllocationSize = std::make_optional<TypeSize>(args: SizeOfBin * 8, args: false);
13561
13562 if (!changeAllocationSize(Size: NewAllocationSize))
13563 return ChangeStatus::UNCHANGED;
13564
13565 return ChangeStatus::CHANGED;
13566 }
13567
13568 /// See AbstractAttribute::manifest(...).
13569 ChangeStatus manifest(Attributor &A) override {
13570
13571 assert(isValidState() &&
13572 "Manifest should only be called if the state is valid.");
13573
13574 Instruction *I = getIRPosition().getCtxI();
13575
13576 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13577
13578 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13579
13580 switch (I->getOpcode()) {
13581 // TODO: add case for malloc like calls
13582 case Instruction::Alloca: {
13583
13584 AllocaInst *AI = cast<AllocaInst>(Val: I);
13585
13586 Type *CharType = Type::getInt8Ty(C&: I->getContext());
13587
13588 auto *NumBytesToValue =
13589 ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
13590
13591 BasicBlock::iterator insertPt = AI->getIterator();
13592 insertPt = std::next(x: insertPt);
13593 AllocaInst *NewAllocaInst =
13594 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13595 AI->getAlign(), AI->getName(), insertPt);
13596
13597 if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
13598 return ChangeStatus::CHANGED;
13599
13600 break;
13601 }
13602 default:
13603 break;
13604 }
13605
13606 return ChangeStatus::UNCHANGED;
13607 }
13608
13609 /// See AbstractAttribute::getAsStr().
13610 const std::string getAsStr(Attributor *A) const override {
13611 if (!isValidState())
13612 return "allocationinfo(<invalid>)";
13613 return "allocationinfo(" +
13614 (AssumedAllocatedSize == HasNoAllocationSize
13615 ? "none"
13616 : std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
13617 ")";
13618 }
13619
13620private:
13621 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13622
13623 // Maintain the computed allocation size of the object.
13624 // Returns (bool) weather the size of the allocation was modified or not.
13625 bool changeAllocationSize(std::optional<TypeSize> Size) {
13626 if (AssumedAllocatedSize == HasNoAllocationSize ||
13627 AssumedAllocatedSize != Size) {
13628 AssumedAllocatedSize = Size;
13629 return true;
13630 }
13631 return false;
13632 }
13633};
13634
13635struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13636 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13637 : AAAllocationInfoImpl(IRP, A) {}
13638
13639 void trackStatistics() const override {
13640 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13641 }
13642};
13643
13644struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13645 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13646 : AAAllocationInfoImpl(IRP, A) {}
13647
13648 /// See AbstractAttribute::initialize(...).
13649 void initialize(Attributor &A) override {
13650 // TODO: we don't rewrite function argument for now because it will need to
13651 // rewrite the function signature and all call sites
13652 (void)indicatePessimisticFixpoint();
13653 }
13654
13655 void trackStatistics() const override {
13656 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13657 }
13658};
13659
13660struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13661 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13662 : AAAllocationInfoImpl(IRP, A) {}
13663
13664 void trackStatistics() const override {
13665 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13666 }
13667};
13668
13669struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13670 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13671 : AAAllocationInfoImpl(IRP, A) {}
13672
13673 void trackStatistics() const override {
13674 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13675 }
13676};
13677
13678struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13679 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13680 : AAAllocationInfoImpl(IRP, A) {}
13681
13682 /// See AbstractAttribute::initialize(...).
13683 void initialize(Attributor &A) override {
13684
13685 (void)indicatePessimisticFixpoint();
13686 }
13687
13688 void trackStatistics() const override {
13689 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13690 }
13691};
13692} // namespace
13693
13694const char AANoUnwind::ID = 0;
13695const char AANoSync::ID = 0;
13696const char AANoFree::ID = 0;
13697const char AANonNull::ID = 0;
13698const char AAMustProgress::ID = 0;
13699const char AANoRecurse::ID = 0;
13700const char AANonConvergent::ID = 0;
13701const char AAWillReturn::ID = 0;
13702const char AAUndefinedBehavior::ID = 0;
13703const char AANoAlias::ID = 0;
13704const char AAIntraFnReachability::ID = 0;
13705const char AANoReturn::ID = 0;
13706const char AAIsDead::ID = 0;
13707const char AADereferenceable::ID = 0;
13708const char AAAlign::ID = 0;
13709const char AAInstanceInfo::ID = 0;
13710const char AANoCapture::ID = 0;
13711const char AAValueSimplify::ID = 0;
13712const char AAHeapToStack::ID = 0;
13713const char AAPrivatizablePtr::ID = 0;
13714const char AAMemoryBehavior::ID = 0;
13715const char AAMemoryLocation::ID = 0;
13716const char AAValueConstantRange::ID = 0;
13717const char AAPotentialConstantValues::ID = 0;
13718const char AAPotentialValues::ID = 0;
13719const char AANoUndef::ID = 0;
13720const char AANoFPClass::ID = 0;
13721const char AACallEdges::ID = 0;
13722const char AAInterFnReachability::ID = 0;
13723const char AAPointerInfo::ID = 0;
13724const char AAAssumptionInfo::ID = 0;
13725const char AAUnderlyingObjects::ID = 0;
13726const char AAInvariantLoadPointer::ID = 0;
13727const char AAAddressSpace::ID = 0;
13728const char AANoAliasAddrSpace::ID = 0;
13729const char AAAllocationInfo::ID = 0;
13730const char AAIndirectCallInfo::ID = 0;
13731const char AAGlobalValueInfo::ID = 0;
13732const char AADenormalFPMath::ID = 0;
13733
13734// Macro magic to create the static generator function for attributes that
13735// follow the naming scheme.
13736
13737#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13738 case IRPosition::PK: \
13739 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13740
13741#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13742 case IRPosition::PK: \
13743 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13744 ++NumAAs; \
13745 break;
13746
13747#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13748 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13749 CLASS *AA = nullptr; \
13750 switch (IRP.getPositionKind()) { \
13751 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13752 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13753 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13754 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13755 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13756 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13757 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13758 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13759 } \
13760 return *AA; \
13761 }
13762
13763#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13764 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13765 CLASS *AA = nullptr; \
13766 switch (IRP.getPositionKind()) { \
13767 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13768 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13769 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13770 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13771 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13772 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13773 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13774 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13775 } \
13776 return *AA; \
13777 }
13778
13779#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13780 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13781 CLASS *AA = nullptr; \
13782 switch (IRP.getPositionKind()) { \
13783 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13784 default: \
13785 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13786 " position!"); \
13787 } \
13788 return *AA; \
13789 }
13790
13791#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13792 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13793 CLASS *AA = nullptr; \
13794 switch (IRP.getPositionKind()) { \
13795 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13796 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13797 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13798 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13799 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13800 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13801 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13802 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13803 } \
13804 return *AA; \
13805 }
13806
13807#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13808 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13809 CLASS *AA = nullptr; \
13810 switch (IRP.getPositionKind()) { \
13811 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13812 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13813 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13814 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13815 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13816 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13817 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13818 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13819 } \
13820 return *AA; \
13821 }
13822
13823#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13824 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13825 CLASS *AA = nullptr; \
13826 switch (IRP.getPositionKind()) { \
13827 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13828 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13829 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13830 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13831 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13832 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13833 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13834 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13835 } \
13836 return *AA; \
13837 }
13838
13839CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13840CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13841CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13842CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13843CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13844CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13845CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13846CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13847CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13848
13849CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13850CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13851CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13852CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13853CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13854CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13855CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13856CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13857CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13858CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13859CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13860CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13861CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13862CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
13863CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13864CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
13865CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13866
13867CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13868CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13869CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13870CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13871
13872CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13873 AAIndirectCallInfo)
13874CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13875 AAGlobalValueInfo)
13876
13877CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13878CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13879CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13880CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13881CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13882CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13883
13884CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13885
13886#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13887#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13888#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13889#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13890#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13891#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13892#undef SWITCH_PK_CREATE
13893#undef SWITCH_PK_INV
13894