1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "IPOOptions.h"
15#include "llvm/IR/BundleAttributes.h"
16#include "llvm/Transforms/IPO/Attributor.h"
17
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseMapInfo.h"
22#include "llvm/ADT/MapVector.h"
23#include "llvm/ADT/SCCIterator.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SetOperations.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/ADT/Statistic.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/Analysis/AliasAnalysis.h"
32#include "llvm/Analysis/AssumeBundleQueries.h"
33#include "llvm/Analysis/AssumptionCache.h"
34#include "llvm/Analysis/CaptureTracking.h"
35#include "llvm/Analysis/CycleAnalysis.h"
36#include "llvm/Analysis/InstructionSimplify.h"
37#include "llvm/Analysis/LazyValueInfo.h"
38#include "llvm/Analysis/MemoryBuiltins.h"
39#include "llvm/Analysis/ScalarEvolution.h"
40#include "llvm/Analysis/TargetTransformInfo.h"
41#include "llvm/Analysis/ValueTracking.h"
42#include "llvm/IR/Argument.h"
43#include "llvm/IR/Assumptions.h"
44#include "llvm/IR/AttributeMask.h"
45#include "llvm/IR/Attributes.h"
46#include "llvm/IR/BasicBlock.h"
47#include "llvm/IR/Constant.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DataLayout.h"
50#include "llvm/IR/DerivedTypes.h"
51#include "llvm/IR/GlobalValue.h"
52#include "llvm/IR/IRBuilder.h"
53#include "llvm/IR/InlineAsm.h"
54#include "llvm/IR/InstrTypes.h"
55#include "llvm/IR/Instruction.h"
56#include "llvm/IR/Instructions.h"
57#include "llvm/IR/IntrinsicInst.h"
58#include "llvm/IR/IntrinsicsAMDGPU.h"
59#include "llvm/IR/IntrinsicsNVPTX.h"
60#include "llvm/IR/LLVMContext.h"
61#include "llvm/IR/MDBuilder.h"
62#include "llvm/IR/NoFolder.h"
63#include "llvm/IR/Value.h"
64#include "llvm/IR/ValueHandle.h"
65#include "llvm/Support/Alignment.h"
66#include "llvm/Support/Casting.h"
67#include "llvm/Support/ErrorHandling.h"
68#include "llvm/Support/GraphWriter.h"
69#include "llvm/Support/InterleavedRange.h"
70#include "llvm/Support/KnownFPClass.h"
71#include "llvm/Support/MathExtras.h"
72#include "llvm/Support/TypeSize.h"
73#include "llvm/Support/raw_ostream.h"
74#include "llvm/Transforms/Utils/BasicBlockUtils.h"
75#include "llvm/Transforms/Utils/CallPromotionUtils.h"
76#include "llvm/Transforms/Utils/Local.h"
77#include "llvm/Transforms/Utils/ValueMapper.h"
78#include <cassert>
79#include <numeric>
80#include <optional>
81#include <string>
82
83using namespace llvm;
84
85#define DEBUG_TYPE "attributor"
86
87STATISTIC(NumAAs, "Number of abstract attributes created");
88STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
89
90// Some helper macros to deal with statistics tracking.
91//
92// Usage:
93// For simple IR attribute tracking overload trackStatistics in the abstract
94// attribute and choose the right STATS_DECLTRACK_********* macro,
95// e.g.,:
96// void trackStatistics() const override {
97// STATS_DECLTRACK_ARG_ATTR(returned)
98// }
99// If there is a single "increment" side one can use the macro
100// STATS_DECLTRACK with a custom message. If there are multiple increment
101// sides, STATS_DECL and STATS_TRACK can also be used separately.
102//
103#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
104 ("Number of " #TYPE " marked '" #NAME "'")
105#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
106#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
107#define STATS_DECL(NAME, TYPE, MSG) \
108 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
109#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
110#define STATS_DECLTRACK(NAME, TYPE, MSG) \
111 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
112#define STATS_DECLTRACK_ARG_ATTR(NAME) \
113 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
114#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
115 STATS_DECLTRACK(NAME, CSArguments, \
116 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
117#define STATS_DECLTRACK_FN_ATTR(NAME) \
118 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
119#define STATS_DECLTRACK_CS_ATTR(NAME) \
120 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
121#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
122 STATS_DECLTRACK(NAME, FunctionReturn, \
123 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
124#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
125 STATS_DECLTRACK(NAME, CSReturn, \
126 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
127#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
128 STATS_DECLTRACK(NAME, Floating, \
129 ("Number of floating values known to be '" #NAME "'"))
130
131// Specialization of the operator<< for abstract attributes subclasses. This
132// disambiguates situations where multiple operators are applicable.
133namespace llvm {
134#define PIPE_OPERATOR(CLASS) \
135 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
136 return OS << static_cast<const AbstractAttribute &>(AA); \
137 }
138
139PIPE_OPERATOR(AAIsDead)
140PIPE_OPERATOR(AANoUnwind)
141PIPE_OPERATOR(AANoSync)
142PIPE_OPERATOR(AANoRecurse)
143PIPE_OPERATOR(AANonConvergent)
144PIPE_OPERATOR(AAWillReturn)
145PIPE_OPERATOR(AANoReturn)
146PIPE_OPERATOR(AANonNull)
147PIPE_OPERATOR(AAMustProgress)
148PIPE_OPERATOR(AANoAlias)
149PIPE_OPERATOR(AADereferenceable)
150PIPE_OPERATOR(AAAlign)
151PIPE_OPERATOR(AAInstanceInfo)
152PIPE_OPERATOR(AANoCapture)
153PIPE_OPERATOR(AAValueSimplify)
154PIPE_OPERATOR(AANoFree)
155PIPE_OPERATOR(AAHeapToStack)
156PIPE_OPERATOR(AAIntraFnReachability)
157PIPE_OPERATOR(AAMemoryBehavior)
158PIPE_OPERATOR(AAMemoryLocation)
159PIPE_OPERATOR(AAValueConstantRange)
160PIPE_OPERATOR(AAPrivatizablePtr)
161PIPE_OPERATOR(AAUndefinedBehavior)
162PIPE_OPERATOR(AAPotentialConstantValues)
163PIPE_OPERATOR(AAPotentialValues)
164PIPE_OPERATOR(AANoUndef)
165PIPE_OPERATOR(AANoFPClass)
166PIPE_OPERATOR(AACallEdges)
167PIPE_OPERATOR(AAInterFnReachability)
168PIPE_OPERATOR(AAPointerInfo)
169PIPE_OPERATOR(AAAssumptionInfo)
170PIPE_OPERATOR(AAUnderlyingObjects)
171PIPE_OPERATOR(AAInvariantLoadPointer)
172PIPE_OPERATOR(AAAddressSpace)
173PIPE_OPERATOR(AANoAliasAddrSpace)
174PIPE_OPERATOR(AAAllocationInfo)
175PIPE_OPERATOR(AAIndirectCallInfo)
176PIPE_OPERATOR(AAGlobalValueInfo)
177PIPE_OPERATOR(AADenormalFPMath)
178
179#undef PIPE_OPERATOR
180
181template <>
182ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
183 const DerefState &R) {
184 ChangeStatus CS0 =
185 clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
186 ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
187 return CS0 | CS1;
188}
189
190} // namespace llvm
191
192static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
193 bool HeaderOnly, CycleRef *CPtr = nullptr) {
194 if (!CI)
195 return true;
196 auto *BB = I->getParent();
197 CycleRef C = CI->getCycle(Block: BB);
198 if (!C)
199 return false;
200 if (CPtr)
201 *CPtr = C;
202 return !HeaderOnly || BB == CI->getHeader(C);
203}
204
205/// Checks if a type could have padding bytes.
206static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
207 // There is no size information, so be conservative.
208 if (!Ty->isSized())
209 return false;
210
211 // If the alloc size is not equal to the storage size, then there are padding
212 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
213 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
214 return false;
215
216 // FIXME: This isn't the right way to check for padding in vectors with
217 // non-byte-size elements.
218 if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
219 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
220
221 // For array types, check for padding within members.
222 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
223 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
224
225 if (!isa<StructType>(Val: Ty))
226 return true;
227
228 // Check for padding within and between elements of a struct.
229 StructType *StructTy = cast<StructType>(Val: Ty);
230 const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
231 uint64_t StartPos = 0;
232 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
233 Type *ElTy = StructTy->getElementType(N: I);
234 if (!isDenselyPacked(Ty: ElTy, DL))
235 return false;
236 if (StartPos != Layout->getElementOffsetInBits(Idx: I))
237 return false;
238 StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
239 }
240
241 // Check for tail padding.
242 return StartPos == Layout->getSizeInBits();
243}
244
245/// Get pointer operand of memory accessing instruction. If \p I is
246/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
247/// is set to false and the instruction is volatile, return nullptr.
248static const Value *getPointerOperand(const Instruction *I,
249 bool AllowVolatile) {
250 if (!AllowVolatile && I->isVolatile())
251 return nullptr;
252
253 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
254 return LI->getPointerOperand();
255 }
256
257 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
258 return SI->getPointerOperand();
259 }
260
261 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
262 return CXI->getPointerOperand();
263 }
264
265 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
266 return RMWI->getPointerOperand();
267 }
268
269 return nullptr;
270}
271
272/// Helper function to create a pointer based on \p Ptr, and advanced by \p
273/// Offset bytes.
274static Value *constructPointer(Value *Ptr, int64_t Offset,
275 IRBuilder<NoFolder> &IRB) {
276 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
277 << "-bytes\n");
278
279 if (Offset)
280 Ptr = IRB.CreatePtrAddWithSignedOffset(
281 Ptr, Offset, Name: Ptr->getName() + ".b" + Twine(Offset));
282 return Ptr;
283}
284
285static const Value *
286stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
287 const Value *Val, const DataLayout &DL, APInt &Offset,
288 bool GetMinOffset, bool AllowNonInbounds,
289 bool UseAssumed = false) {
290
291 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
292 const IRPosition &Pos = IRPosition::value(V);
293 // Only track dependence if we are going to use the assumed info.
294 const AAValueConstantRange *ValueConstantRangeAA =
295 A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
296 DepClass: UseAssumed ? DepClassTy::OPTIONAL
297 : DepClassTy::NONE);
298 if (!ValueConstantRangeAA)
299 return false;
300 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
301 : ValueConstantRangeAA->getKnown();
302 if (Range.isFullSet())
303 return false;
304
305 // We can only use the lower part of the range because the upper part can
306 // be higher than what the value can really be.
307 if (GetMinOffset)
308 ROffset = Range.getSignedMin();
309 else
310 ROffset = Range.getSignedMax();
311 return true;
312 };
313
314 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
315 /* AllowInvariant */ AllowInvariantGroup: true,
316 ExternalAnalysis: AttributorAnalysis);
317}
318
319static const Value *
320getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
321 const Value *Ptr, int64_t &BytesOffset,
322 const DataLayout &DL, bool AllowNonInbounds = false) {
323 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
324 const Value *Base =
325 stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
326 /* GetMinOffset */ true, AllowNonInbounds);
327
328 BytesOffset = OffsetAPInt.getSExtValue();
329 return Base;
330}
331
332/// Clamp the information known for all returned values of a function
333/// (identified by \p QueryingAA) into \p S.
334template <typename AAType, typename StateType = typename AAType::StateType,
335 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
336 bool RecurseForSelectAndPHI = true>
337static void clampReturnedValueStates(
338 Attributor &A, const AAType &QueryingAA, StateType &S,
339 const IRPosition::CallBaseContext *CBContext = nullptr) {
340 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
341 << QueryingAA << " into " << S << "\n");
342
343 assert((QueryingAA.getIRPosition().getPositionKind() ==
344 IRPosition::IRP_RETURNED ||
345 QueryingAA.getIRPosition().getPositionKind() ==
346 IRPosition::IRP_CALL_SITE_RETURNED) &&
347 "Can only clamp returned value states for a function returned or call "
348 "site returned position!");
349
350 // Use an optional state as there might not be any return values and we want
351 // to join (IntegerState::operator&) the state of all there are.
352 std::optional<StateType> T;
353
354 // Callback for each possibly returned value.
355 auto CheckReturnValue = [&](Value &RV) -> bool {
356 const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
357 // If possible, use the hasAssumedIRAttr interface.
358 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
359 bool IsKnown;
360 return AA::hasAssumedIRAttr<IRAttributeKind>(
361 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
362 }
363
364 const AAType *AA =
365 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
366 if (!AA)
367 return false;
368 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
369 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
370 const StateType &AAS = AA->getState();
371 if (!T)
372 T = StateType::getBestState(AAS);
373 *T &= AAS;
374 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
375 << "\n");
376 return T->isValidState();
377 };
378
379 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
380 S: AA::ValueScope::Intraprocedural,
381 RecurseForSelectAndPHI))
382 S.indicatePessimisticFixpoint();
383 else if (T)
384 S ^= *T;
385}
386
387namespace {
388/// Helper class for generic deduction: return value -> returned position.
389template <typename AAType, typename BaseType,
390 typename StateType = typename BaseType::StateType,
391 bool PropagateCallBaseContext = false,
392 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
393 bool RecurseForSelectAndPHI = true>
394struct AAReturnedFromReturnedValues : public BaseType {
395 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
396 : BaseType(IRP, A) {}
397
398 ChangeStatus updateImpl(Attributor &A) override {
399 StateType S(StateType::getBestState(this->getState()));
400 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
401 RecurseForSelectAndPHI>(
402 A, *this, S,
403 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
404 // TODO: If we know we visited all returned values, thus no are assumed
405 // dead, we can take the known information from the state T.
406 return clampStateAndIndicateChange<StateType>(this->getState(), S);
407 }
408};
409
410/// Clamp the information known at all call sites for a given argument
411/// (identified by \p QueryingAA) into \p S.
412template <typename AAType, typename StateType = typename AAType::StateType,
413 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
414static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
415 StateType &S) {
416 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
417 << QueryingAA << " into " << S << "\n");
418
419 assert(QueryingAA.getIRPosition().getPositionKind() ==
420 IRPosition::IRP_ARGUMENT &&
421 "Can only clamp call site argument states for an argument position!");
422
423 // Use an optional state as there might not be any return values and we want
424 // to join (IntegerState::operator&) the state of all there are.
425 std::optional<StateType> T;
426
427 // The argument number which is also the call site argument number.
428 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
429
430 auto CallSiteCheck = [&](AbstractCallSite ACS) {
431 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
432 // Check if a coresponding argument was found or if it is on not associated
433 // (which can happen for callback calls).
434 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
435 return false;
436
437 // If possible, use the hasAssumedIRAttr interface.
438 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
439 bool IsKnown;
440 return AA::hasAssumedIRAttr<IRAttributeKind>(
441 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
442 }
443
444 const AAType *AA =
445 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
446 if (!AA)
447 return false;
448 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
449 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
450 << "\n");
451 const StateType &AAS = AA->getState();
452 if (!T)
453 T = StateType::getBestState(AAS);
454 *T &= AAS;
455 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
456 << "\n");
457 return T->isValidState();
458 };
459
460 bool UsedAssumedInformation = false;
461 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
462 UsedAssumedInformation))
463 S.indicatePessimisticFixpoint();
464 else if (T)
465 S ^= *T;
466}
467
468/// This function is the bridge between argument position and the call base
469/// context.
470template <typename AAType, typename BaseType,
471 typename StateType = typename AAType::StateType,
472 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
473bool getArgumentStateFromCallBaseContext(Attributor &A,
474 BaseType &QueryingAttribute,
475 IRPosition &Pos, StateType &State) {
476 assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
477 "Expected an 'argument' position !");
478 const CallBase *CBContext = Pos.getCallBaseContext();
479 if (!CBContext)
480 return false;
481
482 int ArgNo = Pos.getCallSiteArgNo();
483 assert(ArgNo >= 0 && "Invalid Arg No!");
484 const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
485
486 // If possible, use the hasAssumedIRAttr interface.
487 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
488 bool IsKnown;
489 return AA::hasAssumedIRAttr<IRAttributeKind>(
490 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
491 }
492
493 const auto *AA =
494 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
495 if (!AA)
496 return false;
497 const StateType &CBArgumentState =
498 static_cast<const StateType &>(AA->getState());
499
500 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
501 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
502 << "\n");
503
504 // NOTE: If we want to do call site grouping it should happen here.
505 State ^= CBArgumentState;
506 return true;
507}
508
509/// Helper class for generic deduction: call site argument -> argument position.
510template <typename AAType, typename BaseType,
511 typename StateType = typename AAType::StateType,
512 bool BridgeCallBaseContext = false,
513 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
514struct AAArgumentFromCallSiteArguments : public BaseType {
515 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
516 : BaseType(IRP, A) {}
517
518 ChangeStatus updateImpl(Attributor &A) override {
519 StateType S = StateType::getBestState(this->getState());
520
521 if (BridgeCallBaseContext) {
522 bool Success =
523 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
524 IRAttributeKind>(
525 A, *this, this->getIRPosition(), S);
526 if (Success)
527 return clampStateAndIndicateChange<StateType>(this->getState(), S);
528 }
529 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
530 S);
531
532 // TODO: If we know we visited all incoming values, thus no are assumed
533 // dead, we can take the known information from the state T.
534 return clampStateAndIndicateChange<StateType>(this->getState(), S);
535 }
536};
537
538/// Helper class for generic replication: function returned -> cs returned.
539template <typename AAType, typename BaseType,
540 typename StateType = typename BaseType::StateType,
541 bool IntroduceCallBaseContext = false,
542 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
543struct AACalleeToCallSite : public BaseType {
544 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
545
546 ChangeStatus updateImpl(Attributor &A) override {
547 auto IRPKind = this->getIRPosition().getPositionKind();
548 assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
549 IRPKind == IRPosition::IRP_CALL_SITE) &&
550 "Can only wrap function returned positions for call site "
551 "returned positions!");
552 auto &S = this->getState();
553
554 CallBase &CB = cast<CallBase>(this->getAnchorValue());
555 if (IntroduceCallBaseContext)
556 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
557 << "\n");
558
559 ChangeStatus Changed = ChangeStatus::UNCHANGED;
560 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
561 for (const Function *Callee : Callees) {
562 // The returned position of the callee does not describe the call if
563 // the call uses a different return type.
564 if (IRPKind == IRPosition::IRP_CALL_SITE_RETURNED &&
565 Callee->getReturnType() != CB.getType())
566 return false;
567 IRPosition FnPos =
568 IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
569 ? IRPosition::returned(F: *Callee,
570 CBContext: IntroduceCallBaseContext ? &CB : nullptr)
571 : IRPosition::function(
572 F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
573 // If possible, use the hasAssumedIRAttr interface.
574 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
575 bool IsKnown;
576 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
577 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
578 return false;
579 continue;
580 }
581
582 const AAType *AA =
583 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
584 if (!AA)
585 return false;
586 Changed |= clampStateAndIndicateChange(S, AA->getState());
587 if (S.isAtFixpoint())
588 return S.isValidState();
589 }
590 return true;
591 };
592 if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
593 return S.indicatePessimisticFixpoint();
594 return Changed;
595 }
596};
597
598/// Helper function to accumulate uses.
599template <class AAType, typename StateType = typename AAType::StateType>
600static void followUsesInContext(AAType &AA, Attributor &A,
601 MustBeExecutedContextExplorer &Explorer,
602 const Instruction *CtxI,
603 SetVector<const Use *> &Uses,
604 StateType &State) {
605 for (unsigned u = 0; u < Uses.size(); ++u) {
606 const Use *U = Uses[u];
607 if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
608 bool Found = Explorer.isInContextOf(I: UserI, PP: CtxI);
609 if (Found && AA.followUseInMBEC(A, U, UserI, State))
610 Uses.insert_range(R: llvm::make_pointer_range(Range: UserI->uses()));
611 }
612 }
613}
614
615/// Use the must-be-executed-context around \p I to add information into \p S.
616/// The AAType class is required to have `followUseInMBEC` method with the
617/// following signature and behaviour:
618///
619/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
620/// U - Underlying use.
621/// I - The user of the \p U.
622/// Returns true if the value should be tracked transitively.
623///
624template <class AAType, typename StateType = typename AAType::StateType>
625static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
626 Instruction &CtxI) {
627 const Value &Val = AA.getIRPosition().getAssociatedValue();
628 if (isa<ConstantData>(Val))
629 return;
630
631 MustBeExecutedContextExplorer *Explorer =
632 A.getInfoCache().getMustBeExecutedContextExplorer();
633 if (!Explorer)
634 return;
635
636 // Container for (transitive) uses of the associated value.
637 SetVector<const Use *> Uses;
638 for (const Use &U : Val.uses())
639 Uses.insert(X: &U);
640
641 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
642
643 if (S.isAtFixpoint())
644 return;
645
646 SmallVector<const CondBrInst *, 4> BrInsts;
647 auto Pred = [&](const Instruction *I) {
648 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: I))
649 BrInsts.push_back(Elt: Br);
650 return true;
651 };
652
653 // Here, accumulate conditional branch instructions in the context. We
654 // explore the child paths and collect the known states. The disjunction of
655 // those states can be merged to its own state. Let ParentState_i be a state
656 // to indicate the known information for an i-th branch instruction in the
657 // context. ChildStates are created for its successors respectively.
658 //
659 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
660 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
661 // ...
662 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
663 //
664 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
665 //
666 // FIXME: Currently, recursive branches are not handled. For example, we
667 // can't deduce that ptr must be dereferenced in below function.
668 //
669 // void f(int a, int c, int *ptr) {
670 // if(a)
671 // if (b) {
672 // *ptr = 0;
673 // } else {
674 // *ptr = 1;
675 // }
676 // else {
677 // if (b) {
678 // *ptr = 0;
679 // } else {
680 // *ptr = 1;
681 // }
682 // }
683 // }
684
685 Explorer->checkForAllTerminatorsInContext(PP: &CtxI, Pred);
686 for (const CondBrInst *Br : BrInsts) {
687 StateType ParentState;
688
689 // The known state of the parent state is a conjunction of children's
690 // known states so it is initialized with a best state.
691 ParentState.indicateOptimisticFixpoint();
692
693 for (const BasicBlock *BB : Br->successors()) {
694 StateType ChildState;
695
696 size_t BeforeSize = Uses.size();
697 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
698
699 // Erase uses which only appear in the child.
700 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
701 It = Uses.erase(I: It);
702
703 ParentState &= ChildState;
704 }
705
706 // Use only known state.
707 S += ParentState;
708 }
709}
710} // namespace
711
712/// ------------------------ PointerInfo ---------------------------------------
713
714namespace llvm {
715namespace AA {
716namespace PointerInfo {
717
718struct State;
719
720} // namespace PointerInfo
721} // namespace AA
722
723/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
724template <>
725struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
726 using Access = AAPointerInfo::Access;
727 static unsigned getHashValue(const Access &A);
728 static bool isEqual(const Access &LHS, const Access &RHS);
729};
730
731/// Helper that allows RangeTy as a key in a DenseMap.
732template <> struct DenseMapInfo<AA::RangeTy> {
733 static unsigned getHashValue(const AA::RangeTy &Range) {
734 return detail::combineHashValue(
735 a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
736 b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
737 }
738
739 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
740 return A == B;
741 }
742};
743
744} // namespace llvm
745
746/// A type to track pointer/struct usage and accesses for AAPointerInfo.
747struct AA::PointerInfo::State : public AbstractState {
748 /// Return the best possible representable state.
749 static State getBestState(const State &SIS) { return State(); }
750
751 /// Return the worst possible representable state.
752 static State getWorstState(const State &SIS) {
753 State R;
754 R.indicatePessimisticFixpoint();
755 return R;
756 }
757
758 State() = default;
759 State(State &&SIS) = default;
760
761 const State &getAssumed() const { return *this; }
762
763 bool isValidState() const override { return BS.isValidState(); }
764
765 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
766
767 ChangeStatus indicateOptimisticFixpoint() override {
768 BS.indicateOptimisticFixpoint();
769 return ChangeStatus::UNCHANGED;
770 }
771
772 ChangeStatus indicatePessimisticFixpoint() override {
773 BS.indicatePessimisticFixpoint();
774 return ChangeStatus::CHANGED;
775 }
776
777 State &operator=(const State &R) {
778 if (this == &R)
779 return *this;
780 BS = R.BS;
781 AccessList = R.AccessList;
782 OffsetBins = R.OffsetBins;
783 RemoteIMap = R.RemoteIMap;
784 ReturnedOffsets = R.ReturnedOffsets;
785 return *this;
786 }
787
788 State &operator=(State &&R) {
789 if (this == &R)
790 return *this;
791 std::swap(a&: BS, b&: R.BS);
792 std::swap(LHS&: AccessList, RHS&: R.AccessList);
793 std::swap(a&: OffsetBins, b&: R.OffsetBins);
794 std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
795 std::swap(a&: ReturnedOffsets, b&: R.ReturnedOffsets);
796 return *this;
797 }
798
799 /// Add a new Access to the state at offset \p Offset and with size \p Size.
800 /// The access is associated with \p I, writes \p Content (if anything), and
801 /// is of kind \p Kind. If an Access already exists for the same \p I and same
802 /// \p RemoteI, the two are combined, potentially losing information about
803 /// offset and size. The resulting access must now be moved from its original
804 /// OffsetBin to the bin for its new offset.
805 ///
806 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
807 ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
808 Instruction &I, std::optional<Value *> Content,
809 AAPointerInfo::AccessKind Kind, Type *Ty,
810 Instruction *RemoteI = nullptr);
811
812 AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
813 AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
814 int64_t numOffsetBins() const { return OffsetBins.size(); }
815
816 const AAPointerInfo::Access &getAccess(unsigned Index) const {
817 return AccessList[Index];
818 }
819
820protected:
821 // Every memory instruction results in an Access object. We maintain a list of
822 // all Access objects that we own, along with the following maps:
823 //
824 // - OffsetBins: RangeTy -> { Access }
825 // - RemoteIMap: RemoteI x LocalI -> Access
826 //
827 // A RemoteI is any instruction that accesses memory. RemoteI is different
828 // from LocalI if and only if LocalI is a call; then RemoteI is some
829 // instruction in the callgraph starting from LocalI. Multiple paths in the
830 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
831 // are all combined into a single Access object. This may result in loss of
832 // information in RangeTy in the Access object.
833 SmallVector<AAPointerInfo::Access> AccessList;
834 AAPointerInfo::OffsetBinsTy OffsetBins;
835 DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
836
837 /// Flag to determine if the underlying pointer is reaching a return statement
838 /// in the associated function or not. Returns in other functions cause
839 /// invalidation.
840 AAPointerInfo::OffsetInfo ReturnedOffsets;
841
842 /// See AAPointerInfo::forallInterferingAccesses.
843 template <typename F>
844 bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {
845 if (!isValidState() || !ReturnedOffsets.isUnassigned())
846 return false;
847
848 for (const auto &It : OffsetBins) {
849 AA::RangeTy ItRange = It.getFirst();
850 if (!Range.mayOverlap(Range: ItRange))
851 continue;
852 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
853 for (auto Index : It.getSecond()) {
854 auto &Access = AccessList[Index];
855 if (!CB(Access, IsExact))
856 return false;
857 }
858 }
859 return true;
860 }
861
862 /// See AAPointerInfo::forallInterferingAccesses.
863 template <typename F>
864 bool forallInterferingAccesses(Instruction &I, F CB,
865 AA::RangeTy &Range) const {
866 if (!isValidState() || !ReturnedOffsets.isUnassigned())
867 return false;
868
869 auto LocalList = RemoteIMap.find(Val: &I);
870 if (LocalList == RemoteIMap.end()) {
871 return true;
872 }
873
874 for (unsigned Index : LocalList->getSecond()) {
875 for (auto &R : AccessList[Index]) {
876 Range &= R;
877 if (Range.offsetAndSizeAreUnknown())
878 break;
879 }
880 }
881 return forallInterferingAccesses(Range, CB);
882 }
883
884private:
885 /// State to track fixpoint and validity.
886 BooleanState BS;
887};
888
889ChangeStatus AA::PointerInfo::State::addAccess(
890 Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
891 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
892 Instruction *RemoteI) {
893 RemoteI = RemoteI ? RemoteI : &I;
894
895 // Check if we have an access for this instruction, if not, simply add it.
896 auto &LocalList = RemoteIMap[RemoteI];
897 bool AccExists = false;
898 unsigned AccIndex = AccessList.size();
899 for (auto Index : LocalList) {
900 auto &A = AccessList[Index];
901 if (A.getLocalInst() == &I) {
902 AccExists = true;
903 AccIndex = Index;
904 break;
905 }
906 }
907
908 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
909 LLVM_DEBUG(if (ToAdd.size()) dbgs()
910 << "[AAPointerInfo] Inserting access in new offset bins\n";);
911
912 for (auto Key : ToAdd) {
913 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
914 OffsetBins[Key].insert(V: AccIndex);
915 }
916 };
917
918 if (!AccExists) {
919 AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
920 assert((AccessList.size() == AccIndex + 1) &&
921 "New Access should have been at AccIndex");
922 LocalList.push_back(Elt: AccIndex);
923 AddToBins(AccessList[AccIndex].getRanges());
924 return ChangeStatus::CHANGED;
925 }
926
927 // Combine the new Access with the existing Access, and then update the
928 // mapping in the offset bins.
929 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
930 auto &Current = AccessList[AccIndex];
931 auto Before = Current;
932 Current &= Acc;
933 if (Current == Before)
934 return ChangeStatus::UNCHANGED;
935
936 auto &ExistingRanges = Before.getRanges();
937 auto &NewRanges = Current.getRanges();
938
939 // Ranges that are in the old access but not the new access need to be removed
940 // from the offset bins.
941 AAPointerInfo::RangeList ToRemove;
942 AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
943 LLVM_DEBUG(if (ToRemove.size()) dbgs()
944 << "[AAPointerInfo] Removing access from old offset bins\n";);
945
946 for (auto Key : ToRemove) {
947 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
948 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
949 auto &Bin = OffsetBins[Key];
950 assert(Bin.count(AccIndex) &&
951 "Expected bin to actually contain the Access.");
952 Bin.erase(V: AccIndex);
953 }
954
955 // Ranges that are in the new access but not the old access need to be added
956 // to the offset bins.
957 AAPointerInfo::RangeList ToAdd;
958 AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
959 AddToBins(ToAdd);
960 return ChangeStatus::CHANGED;
961}
962
963namespace {
964
965#ifndef NDEBUG
966static raw_ostream &operator<<(raw_ostream &OS,
967 const AAPointerInfo::OffsetInfo &OI) {
968 OS << llvm::interleaved_array(OI);
969 return OS;
970}
971#endif // NDEBUG
972
973struct AAPointerInfoImpl
974 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
975 using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
976 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
977
978 const std::string getAsStr(Attributor *A) const override {
979 return std::string("PointerInfo ") +
980 (isValidState() ? (std::string("#") +
981 std::to_string(val: OffsetBins.size()) + " bins")
982 : "<invalid>") +
983 (reachesReturn()
984 ? (" (returned:" +
985 join(R: map_range(C: ReturnedOffsets,
986 F: [](int64_t O) { return std::to_string(val: O); }),
987 Separator: ", ") +
988 ")")
989 : "");
990 }
991
992 ChangeStatus manifest(Attributor &A) override {
993 return AAPointerInfo::manifest(A);
994 }
995
996 const_bin_iterator begin() const override { return State::begin(); }
997 const_bin_iterator end() const override { return State::end(); }
998 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
999 bool reachesReturn() const override {
1000 return !ReturnedOffsets.isUnassigned();
1001 }
1002 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1003 if (ReturnedOffsets.isUnknown()) {
1004 OI.setUnknown();
1005 return;
1006 }
1007
1008 OffsetInfo MergedOI;
1009 for (auto Offset : ReturnedOffsets) {
1010 OffsetInfo TmpOI = OI;
1011 TmpOI.addToAll(Inc: Offset);
1012 MergedOI.merge(R: TmpOI);
1013 }
1014 OI = std::move(MergedOI);
1015 }
1016
1017 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1018 if (ReturnedOffsets.isUnknown())
1019 return ChangeStatus::UNCHANGED;
1020 if (ReachedReturnedOffsets.isUnknown()) {
1021 ReturnedOffsets.setUnknown();
1022 return ChangeStatus::CHANGED;
1023 }
1024 if (ReturnedOffsets.merge(R: ReachedReturnedOffsets))
1025 return ChangeStatus::CHANGED;
1026 return ChangeStatus::UNCHANGED;
1027 }
1028
1029 bool forallInterferingAccesses(
1030 AA::RangeTy Range,
1031 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1032 const override {
1033 return State::forallInterferingAccesses(Range, CB);
1034 }
1035
1036 bool forallInterferingAccesses(
1037 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1038 bool FindInterferingWrites, bool FindInterferingReads,
1039 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1040 AA::RangeTy &Range,
1041 function_ref<bool(const Access &)> SkipCB) const override {
1042 HasBeenWrittenTo = false;
1043
1044 SmallPtrSet<const Access *, 8> DominatingWrites;
1045 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1046
1047 Function &Scope = *I.getFunction();
1048 bool IsKnownNoSync;
1049 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1050 A, QueryingAA: &QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1051 IsKnown&: IsKnownNoSync);
1052 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1053 IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1054 bool AllInSameNoSyncFn = IsAssumedNoSync;
1055 bool InstIsExecutedByInitialThreadOnly =
1056 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1057
1058 // If the function is not ending in aligned barriers, we need the stores to
1059 // be in aligned barriers. The load being in one is not sufficient since the
1060 // store might be executed by a thread that disappears after, causing the
1061 // aligned barrier guarding the load to unblock and the load to read a value
1062 // that has no CFG path to the load.
1063 bool InstIsExecutedInAlignedRegion =
1064 FindInterferingReads && ExecDomainAA &&
1065 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1066
1067 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1068 A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1069
1070 InformationCache &InfoCache = A.getInfoCache();
1071 bool IsThreadLocalObj =
1072 AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1073
1074 // Helper to determine if we need to consider threading, which we cannot
1075 // right now. However, if the function is (assumed) nosync or the thread
1076 // executing all instructions is the main thread only we can ignore
1077 // threading. Also, thread-local objects do not require threading reasoning.
1078 // Finally, we can ignore threading if either access is executed in an
1079 // aligned region.
1080 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1081 if (IsThreadLocalObj || AllInSameNoSyncFn)
1082 return true;
1083 const auto *FnExecDomainAA =
1084 I.getFunction() == &Scope
1085 ? ExecDomainAA
1086 : A.lookupAAFor<AAExecutionDomain>(
1087 IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1088 DepClass: DepClassTy::NONE);
1089 if (!FnExecDomainAA)
1090 return false;
1091 if (InstIsExecutedInAlignedRegion ||
1092 (FindInterferingWrites &&
1093 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1094 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1095 return true;
1096 }
1097 if (InstIsExecutedByInitialThreadOnly &&
1098 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1099 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1100 return true;
1101 }
1102 return false;
1103 };
1104
1105 // Helper to determine if the access is executed by the same thread as the
1106 // given instruction, for now it is sufficient to avoid any potential
1107 // threading effects as we cannot deal with them anyway.
1108 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1109 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1110 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1111 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1112 };
1113
1114 // TODO: Use inter-procedural reachability and dominance.
1115 bool IsKnownNoRecurse;
1116 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1117 A, QueryingAA: this, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1118 IsKnown&: IsKnownNoRecurse);
1119
1120 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1121 // AAExecutionDomain) such that we allow scopes other than kernels as long
1122 // as the reaching kernels are disjoint.
1123 bool InstInKernel = A.getInfoCache().isKernel(F: Scope);
1124 bool ObjHasKernelLifetime = false;
1125 const bool UseDominanceReasoning =
1126 FindInterferingWrites && IsKnownNoRecurse;
1127 const DominatorTree *DT =
1128 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1129
1130 // Helper to check if a value has "kernel lifetime", that is it will not
1131 // outlive a GPU kernel. This is true for shared, constant, and local
1132 // globals on AMD and NVIDIA GPUs.
1133 auto HasKernelLifetime = [&](Value *V, Module &M) {
1134 if (!AA::isGPU(M))
1135 return false;
1136 unsigned VAS = V->getType()->getPointerAddressSpace();
1137 return AA::isGPUSharedAddressSpace(M, AS: VAS) ||
1138 AA::isGPUConstantAddressSpace(M, AS: VAS) ||
1139 AA::isGPULocalAddressSpace(M, AS: VAS);
1140 };
1141
1142 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1143 // to determine if we should look at reachability from the callee. For
1144 // certain pointers we know the lifetime and we do not have to step into the
1145 // callee to determine reachability as the pointer would be dead in the
1146 // callee. See the conditional initialization below.
1147 std::function<bool(const Function &)> IsLiveInCalleeCB;
1148
1149 if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1150 // If the alloca containing function is not recursive the alloca
1151 // must be dead in the callee.
1152 const Function *AIFn = AI->getFunction();
1153 ObjHasKernelLifetime = A.getInfoCache().isKernel(F: *AIFn);
1154 bool IsKnownNoRecurse;
1155 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1156 A, QueryingAA: this, IRP: IRPosition::function(F: *AIFn), DepClass: DepClassTy::OPTIONAL,
1157 IsKnown&: IsKnownNoRecurse)) {
1158 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1159 }
1160 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1161 // If the global has kernel lifetime we can stop if we reach a kernel
1162 // as it is "dead" in the (unknown) callees.
1163 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1164 if (ObjHasKernelLifetime)
1165 IsLiveInCalleeCB = [&A](const Function &Fn) {
1166 return !A.getInfoCache().isKernel(F: Fn);
1167 };
1168 }
1169
1170 // Set of accesses/instructions that will overwrite the result and are
1171 // therefore blockers in the reachability traversal.
1172 AA::InstExclusionSetTy ExclusionSet;
1173
1174 auto AccessCB = [&](const Access &Acc, bool Exact) {
1175 Function *AccScope = Acc.getRemoteInst()->getFunction();
1176 bool AccInSameScope = AccScope == &Scope;
1177
1178 // If the object has kernel lifetime we can ignore accesses only reachable
1179 // by other kernels. For now we only skip accesses *in* other kernels.
1180 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1181 A.getInfoCache().isKernel(F: *AccScope))
1182 return true;
1183
1184 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1185 if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1186 ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1187 }
1188
1189 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1190 (!FindInterferingReads || !Acc.isRead()))
1191 return true;
1192
1193 bool Dominates = FindInterferingWrites && DT && Exact &&
1194 Acc.isMustAccess() && AccInSameScope &&
1195 DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1196 if (Dominates)
1197 DominatingWrites.insert(Ptr: &Acc);
1198
1199 // Track if all interesting accesses are in the same `nosync` function as
1200 // the given instruction.
1201 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1202
1203 InterferingAccesses.push_back(Elt: {&Acc, Exact});
1204 return true;
1205 };
1206 if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1207 return false;
1208
1209 HasBeenWrittenTo = !DominatingWrites.empty();
1210
1211 // Dominating writes form a chain, find the least/lowest member.
1212 Instruction *LeastDominatingWriteInst = nullptr;
1213 for (const Access *Acc : DominatingWrites) {
1214 if (!LeastDominatingWriteInst) {
1215 LeastDominatingWriteInst = Acc->getRemoteInst();
1216 } else if (DT->dominates(Def: LeastDominatingWriteInst,
1217 User: Acc->getRemoteInst())) {
1218 LeastDominatingWriteInst = Acc->getRemoteInst();
1219 }
1220 }
1221
1222 // Helper to determine if we can skip a specific write access.
1223 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1224 if (SkipCB && SkipCB(Acc))
1225 return true;
1226 if (!CanIgnoreThreading(Acc))
1227 return false;
1228
1229 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1230 // If we successfully excluded all effects we are interested in, the
1231 // access can be skipped.
1232 bool ReadChecked = !FindInterferingReads;
1233 bool WriteChecked = !FindInterferingWrites;
1234
1235 // If the instruction cannot reach the access, the former does not
1236 // interfere with what the access reads.
1237 if (!ReadChecked) {
1238 if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1239 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1240 ReadChecked = true;
1241 }
1242 // If the instruction cannot be reach from the access, the latter does not
1243 // interfere with what the instruction reads.
1244 if (!WriteChecked) {
1245 if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1246 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1247 WriteChecked = true;
1248 }
1249
1250 // If we still might be affected by the write of the access but there are
1251 // dominating writes in the function of the instruction
1252 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1253 // by them. This would have happend above if they are all in the same
1254 // function, so we only check the inter-procedural case. Effectively, we
1255 // want to show that there is no call after the dominting write that might
1256 // reach the access, and when it returns reach the instruction with the
1257 // updated value. To this end, we iterate all call sites, check if they
1258 // might reach the instruction without going through another access
1259 // (ExclusionSet) and at the same time might reach the access. However,
1260 // that is all part of AAInterFnReachability.
1261 if (!WriteChecked && HasBeenWrittenTo &&
1262 Acc.getRemoteInst()->getFunction() != &Scope) {
1263
1264 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1265 QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1266 if (FnReachabilityAA) {
1267 // Without going backwards in the call tree, can we reach the access
1268 // from the least dominating write. Do not allow to pass the
1269 // instruction itself either.
1270 bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1271
1272 if (!FnReachabilityAA->instructionCanReach(
1273 A, Inst: *LeastDominatingWriteInst,
1274 Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1275 WriteChecked = true;
1276
1277 if (Inserted)
1278 ExclusionSet.erase(Ptr: &I);
1279 }
1280 }
1281
1282 if (ReadChecked && WriteChecked)
1283 return true;
1284
1285 if (!DT || !UseDominanceReasoning)
1286 return false;
1287 if (!DominatingWrites.count(Ptr: &Acc))
1288 return false;
1289 return LeastDominatingWriteInst != Acc.getRemoteInst();
1290 };
1291
1292 // Run the user callback on all accesses we cannot skip and return if
1293 // that succeeded for all or not.
1294 for (auto &It : InterferingAccesses) {
1295 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1296 !CanSkipAccess(*It.first, It.second)) {
1297 if (!UserCB(*It.first, It.second))
1298 return false;
1299 }
1300 }
1301 return true;
1302 }
1303
1304 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1305 const AAPointerInfo &OtherAA,
1306 CallBase &CB) {
1307 using namespace AA::PointerInfo;
1308 if (!OtherAA.getState().isValidState() || !isValidState())
1309 return indicatePessimisticFixpoint();
1310
1311 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1312 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1313 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1314 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1315
1316 // Combine the accesses bin by bin.
1317 const auto &State = OtherAAImpl.getState();
1318 for (const auto &It : State) {
1319 for (auto Index : It.getSecond()) {
1320 const auto &RAcc = State.getAccess(Index);
1321 if (IsByval && !RAcc.isRead())
1322 continue;
1323 bool UsedAssumedInformation = false;
1324 AccessKind AK = RAcc.getKind();
1325 auto Content = A.translateArgumentToCallSiteContent(
1326 V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1327 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1328 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1329
1330 Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1331 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1332 }
1333 }
1334 return Changed;
1335 }
1336
1337 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1338 const OffsetInfo &Offsets, CallBase &CB,
1339 bool IsMustAcc) {
1340 using namespace AA::PointerInfo;
1341 if (!OtherAA.getState().isValidState() || !isValidState())
1342 return indicatePessimisticFixpoint();
1343
1344 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1345
1346 // Combine the accesses bin by bin.
1347 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1348 const auto &State = OtherAAImpl.getState();
1349 for (const auto &It : State) {
1350 for (auto Index : It.getSecond()) {
1351 const auto &RAcc = State.getAccess(Index);
1352 if (!IsMustAcc && RAcc.isAssumption())
1353 continue;
1354 for (auto Offset : Offsets) {
1355 auto NewRanges = Offset == AA::RangeTy::Unknown
1356 ? AA::RangeTy::getUnknown()
1357 : RAcc.getRanges();
1358 if (!NewRanges.isUnknown()) {
1359 NewRanges.addToAllOffsets(Inc: Offset);
1360 }
1361 AccessKind AK = RAcc.getKind();
1362 if (!IsMustAcc)
1363 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1364 Changed |= addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: AK,
1365 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1366 }
1367 }
1368 }
1369 return Changed;
1370 }
1371
1372 /// Statistic tracking for all AAPointerInfo implementations.
1373 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1374
1375 /// Dump the state into \p O.
1376 void dumpState(raw_ostream &O) {
1377 for (auto &It : OffsetBins) {
1378 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1379 << "] : " << It.getSecond().size() << "\n";
1380 for (auto AccIndex : It.getSecond()) {
1381 auto &Acc = AccessList[AccIndex];
1382 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1383 if (Acc.getLocalInst() != Acc.getRemoteInst())
1384 O << " --> " << *Acc.getRemoteInst()
1385 << "\n";
1386 if (!Acc.isWrittenValueYetUndetermined()) {
1387 if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1388 O << " - c: func " << Acc.getWrittenValue()->getName()
1389 << "\n";
1390 else if (Acc.getWrittenValue())
1391 O << " - c: " << *Acc.getWrittenValue() << "\n";
1392 else
1393 O << " - c: <unknown>\n";
1394 }
1395 }
1396 }
1397 }
1398};
1399
1400struct AAPointerInfoFloating : public AAPointerInfoImpl {
1401 using AccessKind = AAPointerInfo::AccessKind;
1402 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1403 : AAPointerInfoImpl(IRP, A) {}
1404
1405 /// Deal with an access and signal if it was handled successfully.
1406 bool handleAccess(Attributor &A, Instruction &I,
1407 std::optional<Value *> Content, AccessKind Kind,
1408 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1409 Type &Ty) {
1410 using namespace AA::PointerInfo;
1411 auto Size = AA::RangeTy::Unknown;
1412 const DataLayout &DL = A.getDataLayout();
1413 TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1414 if (!AccessSize.isScalable())
1415 Size = AccessSize.getFixedValue();
1416
1417 // Make a strictly ascending list of offsets as required by addAccess()
1418 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1419 llvm::sort(C&: OffsetsSorted);
1420
1421 VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1422 if (!VT || VT->getElementCount().isScalable() ||
1423 !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1424 (*Content)->getType() != VT ||
1425 DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1426 Changed =
1427 Changed | addAccess(A, Ranges: {OffsetsSorted, Size}, I, Content, Kind, Ty: &Ty);
1428 } else {
1429 // Handle vector stores with constant content element-wise.
1430 // TODO: We could look for the elements or create instructions
1431 // representing them.
1432 // TODO: We need to push the Content into the range abstraction
1433 // (AA::RangeTy) to allow different content values for different
1434 // ranges. ranges. Hence, support vectors storing different values.
1435 Type *ElementType = VT->getElementType();
1436 int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1437 auto *ConstContent = cast<Constant>(Val: *Content);
1438 Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1439 SmallVector<int64_t> ElementOffsets(OffsetsSorted.begin(),
1440 OffsetsSorted.end());
1441
1442 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1443 Value *ElementContent = ConstantExpr::getExtractElement(
1444 Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1445
1446 // Add the element access.
1447 Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1448 Content: ElementContent, Kind, Ty: ElementType);
1449
1450 // Advance the offsets for the next element.
1451 for (auto &ElementOffset : ElementOffsets)
1452 ElementOffset += ElementSize;
1453 }
1454 }
1455 return true;
1456 };
1457
1458 ChangeStatus updateImpl(Attributor &A) override;
1459
1460 /// If the indices to \p GEP can be traced to constants, incorporate all
1461 /// of these into \p UsrOI.
1462 ///
1463 /// \return true iff \p UsrOI is updated.
1464 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1465 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1466 const GEPOperator *GEP);
1467
1468 void trackStatistics() const override {
1469 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1470 }
1471};
1472
1473bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1474 const DataLayout &DL,
1475 OffsetInfo &UsrOI,
1476 const OffsetInfo &PtrOI,
1477 const GEPOperator *GEP) {
1478 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1479 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1480 APInt ConstantOffset(BitWidth, 0);
1481
1482 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1483 "Don't look for constant values if the offset has already been "
1484 "determined to be unknown.");
1485
1486 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1487 UsrOI.setUnknown();
1488 return true;
1489 }
1490
1491 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1492 << (VariableOffsets.empty() ? "" : "not") << " constant "
1493 << *GEP << "\n");
1494
1495 auto Union = PtrOI;
1496 Union.addToAll(Inc: ConstantOffset.getSExtValue());
1497
1498 // Each VI in VariableOffsets has a set of potential constant values. Every
1499 // combination of elements, picked one each from these sets, is separately
1500 // added to the original set of offsets, thus resulting in more offsets.
1501 for (const auto &VI : VariableOffsets) {
1502 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1503 QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1504 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1505 UsrOI.setUnknown();
1506 return true;
1507 }
1508
1509 // UndefValue is treated as a zero, which leaves Union as is.
1510 if (PotentialConstantsAA->undefIsContained())
1511 continue;
1512
1513 // We need at least one constant in every set to compute an actual offset.
1514 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1515 // don't actually exist. In other words, the absence of constant values
1516 // implies that the operation can be assumed dead for now.
1517 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1518 if (AssumedSet.empty())
1519 return false;
1520
1521 OffsetInfo Product;
1522 for (const auto &ConstOffset : AssumedSet) {
1523 auto CopyPerOffset = Union;
1524 CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1525 VI.second.getZExtValue());
1526 Product.merge(R: CopyPerOffset);
1527 }
1528 Union = Product;
1529 }
1530
1531 UsrOI = std::move(Union);
1532 return true;
1533}
1534
1535ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1536 using namespace AA::PointerInfo;
1537 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1538 const DataLayout &DL = A.getDataLayout();
1539 Value &AssociatedValue = getAssociatedValue();
1540
1541 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1542 OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1543
1544 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1545 // One does not simply walk into a map and assign a reference to a possibly
1546 // new location. That can cause an invalidation before the assignment
1547 // happens, like so:
1548 //
1549 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1550 //
1551 // The RHS is a reference that may be invalidated by an insertion caused by
1552 // the LHS. So we ensure that the side-effect of the LHS happens first.
1553
1554 assert(OffsetInfoMap.contains(CurPtr) &&
1555 "CurPtr does not exist in the map!");
1556
1557 auto &UsrOI = OffsetInfoMap[Usr];
1558 auto &PtrOI = OffsetInfoMap[CurPtr];
1559 assert(!PtrOI.isUnassigned() &&
1560 "Cannot pass through if the input Ptr was not visited!");
1561 UsrOI.merge(R: PtrOI);
1562 Follow = true;
1563 return true;
1564 };
1565
1566 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1567 Value *CurPtr = U.get();
1568 User *Usr = U.getUser();
1569 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1570 << "\n");
1571 assert(OffsetInfoMap.count(CurPtr) &&
1572 "The current pointer offset should have been seeded!");
1573 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1574 "Current pointer should be assigned");
1575
1576 // Only address space casts are followed; other casts of a pointer (e.g.,
1577 // ptrtoint, ptrtoaddr, or a bitcast to a byte type) do not produce a
1578 // pointer that can be tracked.
1579 if (isa<AddrSpaceCastOperator>(Val: Usr))
1580 return HandlePassthroughUser(Usr, CurPtr, Follow);
1581
1582 if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1583 // Note the order here, the Usr access might change the map, CurPtr is
1584 // already in it though.
1585 auto &UsrOI = OffsetInfoMap[Usr];
1586 auto &PtrOI = OffsetInfoMap[CurPtr];
1587
1588 if (UsrOI.isUnknown())
1589 return true;
1590
1591 if (PtrOI.isUnknown()) {
1592 Follow = true;
1593 UsrOI.setUnknown();
1594 return true;
1595 }
1596
1597 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1598 return true;
1599 }
1600 if (isa<SelectInst>(Val: Usr))
1601 return HandlePassthroughUser(Usr, CurPtr, Follow);
1602 // Returns are allowed if they are in the associated functions. Users can
1603 // then check the call site return. Returns from other functions can't be
1604 // tracked and are cause for invalidation.
1605 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
1606 if (RI->getFunction() == getAssociatedFunction()) {
1607 auto &PtrOI = OffsetInfoMap[CurPtr];
1608 Changed |= setReachesReturn(PtrOI);
1609 return true;
1610 }
1611 return false;
1612 }
1613
1614 // For PHIs we need to take care of the recurrence explicitly as the value
1615 // might change while we iterate through a loop. For now, we give up if
1616 // the PHI is not invariant.
1617 if (auto *PHI = dyn_cast<PHINode>(Val: Usr)) {
1618 // Note the order here, the Usr access might change the map, CurPtr is
1619 // already in it though.
1620 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(Key: PHI);
1621 auto &UsrOI = PhiIt->second;
1622 auto &PtrOI = OffsetInfoMap[CurPtr];
1623
1624 // Check if the PHI operand has already an unknown offset as we can't
1625 // improve on that anymore.
1626 if (PtrOI.isUnknown()) {
1627 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1628 << *CurPtr << " in " << *PHI << "\n");
1629 Follow = !UsrOI.isUnknown();
1630 UsrOI.setUnknown();
1631 return true;
1632 }
1633
1634 // Check if the PHI is invariant (so far).
1635 if (UsrOI == PtrOI) {
1636 assert(!PtrOI.isUnassigned() &&
1637 "Cannot assign if the current Ptr was not visited!");
1638 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1639 return true;
1640 }
1641
1642 // Check if the PHI operand can be traced back to AssociatedValue.
1643 APInt Offset(
1644 DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1645 0);
1646 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1647 DL, Offset, /* AllowNonInbounds */ true);
1648 auto It = OffsetInfoMap.find(Val: CurPtrBase);
1649 if (It == OffsetInfoMap.end()) {
1650 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1651 << *CurPtr << " in " << *PHI
1652 << " (base: " << *CurPtrBase << ")\n");
1653 UsrOI.setUnknown();
1654 Follow = true;
1655 return true;
1656 }
1657
1658 // Check if the PHI operand is not dependent on the PHI itself. Every
1659 // recurrence is a cyclic net of PHIs in the data flow, and has an
1660 // equivalent Cycle in the control flow. One of those PHIs must be in the
1661 // header of that control flow Cycle. This is independent of the choice of
1662 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1663 // every Cycle header; if such a node is marked unknown, this will
1664 // eventually propagate through the whole net of PHIs in the recurrence.
1665 const auto *CI =
1666 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1667 F: *PHI->getFunction());
1668 if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1669 auto BaseOI = It->getSecond();
1670 BaseOI.addToAll(Inc: Offset.getZExtValue());
1671 if (IsFirstPHIUser || BaseOI == UsrOI) {
1672 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1673 << " in " << *Usr << "\n");
1674 return HandlePassthroughUser(Usr, CurPtr, Follow);
1675 }
1676
1677 LLVM_DEBUG(
1678 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1679 << *CurPtr << " in " << *PHI << "\n");
1680 UsrOI.setUnknown();
1681 Follow = true;
1682 return true;
1683 }
1684
1685 UsrOI.merge(R: PtrOI);
1686 Follow = true;
1687 return true;
1688 }
1689
1690 if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1691 // If the access is to a pointer that may or may not be the associated
1692 // value, e.g. due to a PHI, we cannot assume it will be read.
1693 AccessKind AK = AccessKind::AK_R;
1694 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1695 AK = AccessKind(AK | AccessKind::AK_MUST);
1696 else
1697 AK = AccessKind(AK | AccessKind::AK_MAY);
1698 if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1699 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1700 Ty&: *LoadI->getType()))
1701 return false;
1702
1703 auto IsAssumption = [](Instruction &I) {
1704 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1705 return II->isAssumeLikeIntrinsic();
1706 return false;
1707 };
1708
1709 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1710 // Check if the assumption and the load are executed together without
1711 // memory modification.
1712 do {
1713 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1714 return true;
1715 FromI = FromI->getNextNode();
1716 } while (FromI && FromI != ToI);
1717 return false;
1718 };
1719
1720 BasicBlock *BB = LoadI->getParent();
1721 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1722 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1723 return false;
1724 BasicBlock *IntrBB = IntrI.getParent();
1725 if (IntrI.getParent() == BB) {
1726 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1727 return false;
1728 } else {
1729 auto PredIt = pred_begin(BB: IntrBB);
1730 if (PredIt == pred_end(BB: IntrBB))
1731 return false;
1732 if ((*PredIt) != BB)
1733 return false;
1734 if (++PredIt != pred_end(BB: IntrBB))
1735 return false;
1736 for (auto *SuccBB : successors(BB)) {
1737 if (SuccBB == IntrBB)
1738 continue;
1739 if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1740 continue;
1741 return false;
1742 }
1743 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1744 return false;
1745 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1746 return false;
1747 }
1748 return true;
1749 };
1750
1751 std::pair<Value *, IntrinsicInst *> Assumption;
1752 for (const Use &LoadU : LoadI->uses()) {
1753 if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1754 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1755 continue;
1756 for (const Use &CmpU : CmpI->uses()) {
1757 if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1758 if (!IsValidAssume(*IntrI))
1759 continue;
1760 int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1761 Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1762 break;
1763 }
1764 }
1765 }
1766 if (Assumption.first)
1767 break;
1768 }
1769
1770 // Check if we found an assumption associated with this load.
1771 if (!Assumption.first || !Assumption.second)
1772 return true;
1773
1774 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1775 << *Assumption.second << ": " << *LoadI
1776 << " == " << *Assumption.first << "\n");
1777 bool UsedAssumedInformation = false;
1778 std::optional<Value *> Content = nullptr;
1779 if (Assumption.first)
1780 Content =
1781 A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1782 UsedAssumedInformation, S: AA::Interprocedural);
1783 return handleAccess(
1784 A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1785 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1786 }
1787
1788 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1789 ArrayRef<Value *> OtherOps, AccessKind AK) {
1790 for (auto *OtherOp : OtherOps) {
1791 if (OtherOp == CurPtr) {
1792 LLVM_DEBUG(
1793 dbgs()
1794 << "[AAPointerInfo] Escaping use in store like instruction " << I
1795 << "\n");
1796 return false;
1797 }
1798 }
1799
1800 // If the access is to a pointer that may or may not be the associated
1801 // value, e.g. due to a PHI, we cannot assume it will be written.
1802 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1803 AK = AccessKind(AK | AccessKind::AK_MUST);
1804 else
1805 AK = AccessKind(AK | AccessKind::AK_MAY);
1806 bool UsedAssumedInformation = false;
1807 std::optional<Value *> Content = nullptr;
1808 if (ValueOp)
1809 Content = A.getAssumedSimplified(
1810 V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1811 return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1812 Changed, Ty&: ValueTy);
1813 };
1814
1815 if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1816 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1817 *StoreI->getValueOperand()->getType(),
1818 {StoreI->getValueOperand()}, AccessKind::AK_W);
1819 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1820 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1821 {RMWI->getValOperand()}, AccessKind::AK_RW);
1822 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1823 return HandleStoreLike(
1824 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1825 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1826 AccessKind::AK_RW);
1827
1828 if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1829 if (CB->isLifetimeStartOrEnd())
1830 return true;
1831 const auto *TLI =
1832 A.getInfoCache().getTargetLibraryInfoForFunction(F: *CB->getFunction());
1833 if (getFreedOperand(CB, TLI) == U)
1834 return true;
1835 if (CB->isArgOperand(U: &U)) {
1836 unsigned ArgNo = CB->getArgOperandNo(U: &U);
1837 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1838 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1839 DepClass: DepClassTy::REQUIRED);
1840 if (!CSArgPI)
1841 return false;
1842 bool IsArgMustAcc = (getUnderlyingObject(V: CurPtr) == &AssociatedValue);
1843 Changed = translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB,
1844 IsMustAcc: IsArgMustAcc) |
1845 Changed;
1846 if (!CSArgPI->reachesReturn())
1847 return isValidState();
1848
1849 Function *Callee = CB->getCalledFunction();
1850 if (!Callee || Callee->arg_size() <= ArgNo)
1851 return false;
1852 bool UsedAssumedInformation = false;
1853 auto ReturnedValue = A.getAssumedSimplified(
1854 IRP: IRPosition::returned(F: *Callee), AA: *this, UsedAssumedInformation,
1855 S: AA::ValueScope::Intraprocedural);
1856 auto *ReturnedArg =
1857 dyn_cast_or_null<Argument>(Val: ReturnedValue.value_or(u: nullptr));
1858 auto *Arg = Callee->getArg(i: ArgNo);
1859 if (ReturnedArg && Arg != ReturnedArg)
1860 return true;
1861 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1862 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1863 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
1864 if (!CSRetPI)
1865 return false;
1866 OffsetInfo OI = OffsetInfoMap[CurPtr];
1867 CSArgPI->addReturnedOffsetsTo(OI);
1868 Changed =
1869 translateAndAddState(A, OtherAA: *CSRetPI, Offsets: OI, CB&: *CB, IsMustAcc: IsRetMustAcc) | Changed;
1870 return isValidState();
1871 }
1872 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1873 << "\n");
1874 return false;
1875 }
1876
1877 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1878 return false;
1879 };
1880 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1881 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1882 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1883 if (OffsetInfoMap.count(Val: NewU)) {
1884 LLVM_DEBUG({
1885 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1886 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1887 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1888 << "\n";
1889 }
1890 });
1891 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1892 }
1893 bool Unused;
1894 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1895 };
1896 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1897 /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1898 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1899 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1900 return indicatePessimisticFixpoint();
1901 }
1902
1903 LLVM_DEBUG({
1904 dbgs() << "Accesses by bin after update:\n";
1905 dumpState(dbgs());
1906 });
1907
1908 return Changed;
1909}
1910
1911struct AAPointerInfoReturned final : AAPointerInfoImpl {
1912 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1913 : AAPointerInfoImpl(IRP, A) {}
1914
1915 ChangeStatus updateImpl(Attributor &A) override {
1916 return indicatePessimisticFixpoint();
1917 }
1918
1919 void trackStatistics() const override {
1920 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1921 }
1922};
1923
1924struct AAPointerInfoArgument final : AAPointerInfoFloating {
1925 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1926 : AAPointerInfoFloating(IRP, A) {}
1927
1928 void trackStatistics() const override {
1929 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1930 }
1931};
1932
1933struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1934 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1935 : AAPointerInfoFloating(IRP, A) {}
1936
1937 ChangeStatus updateImpl(Attributor &A) override {
1938 using namespace AA::PointerInfo;
1939 // We handle memory intrinsics explicitly, at least the first (=
1940 // destination) and second (=source) arguments as we know how they are
1941 // accessed.
1942 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
1943 int64_t LengthVal = AA::RangeTy::Unknown;
1944 if (auto Length = MI->getLengthInBytes())
1945 LengthVal = Length->getSExtValue();
1946 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1947 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1948 if (ArgNo > 1) {
1949 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1950 << *MI << "\n");
1951 return indicatePessimisticFixpoint();
1952 } else {
1953 auto Kind =
1954 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1955 Changed =
1956 Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
1957 }
1958 LLVM_DEBUG({
1959 dbgs() << "Accesses by bin after update:\n";
1960 dumpState(dbgs());
1961 });
1962
1963 return Changed;
1964 }
1965
1966 // TODO: Once we have call site specific value information we can provide
1967 // call site specific liveness information and then it makes
1968 // sense to specialize attributes for call sites arguments instead of
1969 // redirecting requests to the callee argument.
1970 Argument *Arg = getAssociatedArgument();
1971 if (Arg) {
1972 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
1973 auto *ArgAA =
1974 A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
1975 if (ArgAA && ArgAA->getState().isValidState())
1976 return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
1977 CB&: *cast<CallBase>(Val: getCtxI()));
1978 if (!Arg->getParent()->isDeclaration())
1979 return indicatePessimisticFixpoint();
1980 }
1981
1982 bool IsKnownNoCapture;
1983 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
1984 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
1985 return indicatePessimisticFixpoint();
1986
1987 bool IsKnown = false;
1988 if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
1989 return ChangeStatus::UNCHANGED;
1990 bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
1991 auto Kind =
1992 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
1993 return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
1994 Ty: nullptr);
1995 }
1996
1997 void trackStatistics() const override {
1998 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1999 }
2000};
2001
2002struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2003 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2004 : AAPointerInfoFloating(IRP, A) {}
2005
2006 void trackStatistics() const override {
2007 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2008 }
2009};
2010} // namespace
2011
2012/// -----------------------NoUnwind Function Attribute--------------------------
2013
2014namespace {
2015struct AANoUnwindImpl : AANoUnwind {
2016 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2017
2018 void initialize(Attributor &A) override {
2019 bool IsKnown;
2020 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2021 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2022 (void)IsKnown;
2023 }
2024
2025 const std::string getAsStr(Attributor *A) const override {
2026 return getAssumed() ? "nounwind" : "may-unwind";
2027 }
2028
2029 ChangeStatus updateImpl(Attributor &A) override {
2030 auto Opcodes = {
2031 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2032 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2033 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2034
2035 auto CheckForNoUnwind = [&](Instruction &I) {
2036 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2037 return true;
2038
2039 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2040 bool IsKnownNoUnwind;
2041 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2042 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2043 IsKnown&: IsKnownNoUnwind);
2044 }
2045 return false;
2046 };
2047
2048 bool UsedAssumedInformation = false;
2049 if (!A.checkForAllInstructions(Pred: CheckForNoUnwind, QueryingAA: *this, Opcodes,
2050 UsedAssumedInformation))
2051 return indicatePessimisticFixpoint();
2052
2053 return ChangeStatus::UNCHANGED;
2054 }
2055};
2056
2057struct AANoUnwindFunction final : public AANoUnwindImpl {
2058 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2059 : AANoUnwindImpl(IRP, A) {}
2060
2061 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2062};
2063
2064/// NoUnwind attribute deduction for a call sites.
2065struct AANoUnwindCallSite final
2066 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2067 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2068 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2069
2070 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2071};
2072} // namespace
2073
2074/// ------------------------ NoSync Function Attribute -------------------------
2075
2076bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2077 switch (CB.getIntrinsicID()) {
2078 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2079 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2080 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2081 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2082 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2083 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2084 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2085 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2086 return true;
2087 case Intrinsic::amdgcn_s_barrier:
2088 if (ExecutedAligned)
2089 return true;
2090 break;
2091 default:
2092 break;
2093 }
2094 return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2095}
2096
2097bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2098 if (!I->isAtomic())
2099 return false;
2100
2101 if (auto *FI = dyn_cast<FenceInst>(Val: I))
2102 // All legal orderings for fence are stronger than monotonic.
2103 return FI->getSyncScopeID() != SyncScope::SingleThread;
2104 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2105 // Unordered is not a legal ordering for cmpxchg.
2106 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2107 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2108 }
2109
2110 AtomicOrdering Ordering;
2111 switch (I->getOpcode()) {
2112 case Instruction::AtomicRMW:
2113 Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2114 break;
2115 case Instruction::Store:
2116 Ordering = cast<StoreInst>(Val: I)->getOrdering();
2117 break;
2118 case Instruction::Load:
2119 Ordering = cast<LoadInst>(Val: I)->getOrdering();
2120 break;
2121 default:
2122 llvm_unreachable(
2123 "New atomic operations need to be known in the attributor.");
2124 }
2125
2126 return (Ordering != AtomicOrdering::Unordered &&
2127 Ordering != AtomicOrdering::Monotonic);
2128}
2129
2130namespace {
2131struct AANoSyncImpl : AANoSync {
2132 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2133
2134 void initialize(Attributor &A) override {
2135 bool IsKnown;
2136 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2137 DepClassTy::NONE, IsKnown));
2138 (void)IsKnown;
2139 }
2140
2141 const std::string getAsStr(Attributor *A) const override {
2142 return getAssumed() ? "nosync" : "may-sync";
2143 }
2144
2145 ChangeStatus updateImpl(Attributor &A) override;
2146};
2147
2148ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2149
2150 auto CheckRWInstForNoSync = [&](Instruction &I) {
2151 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2152 };
2153
2154 auto CheckForNoSync = [&](Instruction &I) {
2155 // At this point we handled all read/write effects and they are all
2156 // nosync, so they can be skipped.
2157 if (I.mayReadOrWriteMemory())
2158 return true;
2159
2160 bool IsKnown;
2161 CallBase &CB = cast<CallBase>(Val&: I);
2162 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2163 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL,
2164 IsKnown))
2165 return true;
2166
2167 // non-convergent and readnone imply nosync.
2168 return !CB.isConvergent();
2169 };
2170
2171 bool UsedAssumedInformation = false;
2172 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInstForNoSync, QueryingAA&: *this,
2173 UsedAssumedInformation) ||
2174 !A.checkForAllCallLikeInstructions(Pred: CheckForNoSync, QueryingAA: *this,
2175 UsedAssumedInformation))
2176 return indicatePessimisticFixpoint();
2177
2178 return ChangeStatus::UNCHANGED;
2179}
2180
2181struct AANoSyncFunction final : public AANoSyncImpl {
2182 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2183 : AANoSyncImpl(IRP, A) {}
2184
2185 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2186};
2187
2188/// NoSync attribute deduction for a call sites.
2189struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2190 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2191 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2192
2193 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2194};
2195} // namespace
2196
2197/// ------------------------ No-Free Attributes ----------------------------
2198
2199namespace {
2200struct AANoFreeImpl : public AANoFree {
2201 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2202
2203 void initialize(Attributor &A) override {
2204 bool IsKnown;
2205 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2206 DepClassTy::NONE, IsKnown));
2207 (void)IsKnown;
2208 }
2209
2210 ChangeStatus updateImpl(Attributor &A) override {
2211 auto CheckForNoFree = [&](Instruction &I) {
2212 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
2213 bool IsKnown;
2214 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2215 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2216 IsKnown);
2217 }
2218 // Make sure that synchronization cannot establish happens-before with a
2219 // free on another thread.
2220 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2221 };
2222
2223 bool UsedAssumedInformation = false;
2224 if (!A.checkForAllReadWriteInstructions(Pred: CheckForNoFree, QueryingAA&: *this,
2225 UsedAssumedInformation) ||
2226 !A.checkForAllCallLikeInstructions(Pred: CheckForNoFree, QueryingAA: *this,
2227 UsedAssumedInformation))
2228 return indicatePessimisticFixpoint();
2229
2230 return ChangeStatus::UNCHANGED;
2231 }
2232
2233 const std::string getAsStr(Attributor *A) const override {
2234 return getAssumed() ? "nofree" : "may-free";
2235 }
2236};
2237
2238struct AANoFreeFunction final : public AANoFreeImpl {
2239 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2240 : AANoFreeImpl(IRP, A) {}
2241
2242 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2243};
2244
2245/// NoFree attribute deduction for a call sites.
2246struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2247 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2248 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2249
2250 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2251};
2252
2253/// NoFree attribute for floating values.
2254struct AANoFreeFloating : AANoFreeImpl {
2255 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2256 : AANoFreeImpl(IRP, A) {}
2257
2258 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2259
2260 /// See Abstract Attribute::updateImpl(...).
2261 ChangeStatus updateImpl(Attributor &A) override {
2262 const IRPosition &IRP = getIRPosition();
2263
2264 bool IsKnown;
2265 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this,
2266 IRP: IRPosition::function_scope(IRP),
2267 DepClass: DepClassTy::OPTIONAL, IsKnown))
2268 return ChangeStatus::UNCHANGED;
2269
2270 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2271 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2272 Instruction *UserI = cast<Instruction>(Val: U.getUser());
2273 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2274 if (CB->isBundleOperand(U: &U))
2275 return false;
2276 if (!CB->isArgOperand(U: &U))
2277 return true;
2278 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2279
2280 // Even if the argument is nofree, we still need to check for nocapture,
2281 // as the call may capture the argument without freeing it, and the
2282 // captured argument is freed later.
2283 bool IsKnown;
2284 if (!AA::hasAssumedIRAttr<Attribute::NoFree>(
2285 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2286 DepClass: DepClassTy::REQUIRED, IsKnown))
2287 return false;
2288
2289 const AANoCapture *NoCaptureAA = nullptr;
2290 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2291 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2292 DepClass: DepClassTy::REQUIRED, IsKnown,
2293 /*IgnoreSubsumingPositions=*/false, AAPtr: &NoCaptureAA)) {
2294 if (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
2295 Follow = true;
2296 return true;
2297 }
2298 return false;
2299 }
2300
2301 return true;
2302 }
2303
2304 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
2305 if (!capturesAnyProvenance(CC: CI))
2306 return true;
2307 if (capturesAnyProvenance(CC: CI.ResultCC)) {
2308 Follow = true;
2309 return true;
2310 }
2311
2312 if (isa<ReturnInst>(Val: UserI) && getIRPosition().isArgumentPosition())
2313 return true;
2314
2315 // Capturing user.
2316 return false;
2317 };
2318 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: AssociatedValue))
2319 return indicatePessimisticFixpoint();
2320
2321 return ChangeStatus::UNCHANGED;
2322 }
2323};
2324
2325/// NoFree attribute for a call site argument.
2326struct AANoFreeArgument final : AANoFreeFloating {
2327 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2328 : AANoFreeFloating(IRP, A) {}
2329
2330 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2331};
2332
2333/// NoFree attribute for call site arguments.
2334struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2335 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2336 : AANoFreeFloating(IRP, A) {}
2337
2338 ChangeStatus updateImpl(Attributor &A) override {
2339 // TODO: Once we have call site specific value information we can provide
2340 // call site specific liveness information and then it makes
2341 // sense to specialize attributes for call sites arguments instead of
2342 // redirecting requests to the callee argument.
2343 Argument *Arg = getAssociatedArgument();
2344 if (!Arg)
2345 return indicatePessimisticFixpoint();
2346 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2347 bool IsKnown;
2348 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this, IRP: ArgPos,
2349 DepClass: DepClassTy::REQUIRED, IsKnown))
2350 return ChangeStatus::UNCHANGED;
2351 return indicatePessimisticFixpoint();
2352 }
2353
2354 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2355};
2356
2357/// NoFree attribute for function return value.
2358struct AANoFreeReturned final : AANoFreeFloating {
2359 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2360 : AANoFreeFloating(IRP, A) {
2361 llvm_unreachable("NoFree is not applicable to function returns!");
2362 }
2363
2364 void initialize(Attributor &A) override {
2365 llvm_unreachable("NoFree is not applicable to function returns!");
2366 }
2367
2368 ChangeStatus updateImpl(Attributor &A) override {
2369 llvm_unreachable("NoFree is not applicable to function returns!");
2370 }
2371
2372 void trackStatistics() const override {}
2373};
2374
2375/// NoFree attribute deduction for a call site return value.
2376struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2377 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2378 : AANoFreeFloating(IRP, A) {}
2379
2380 ChangeStatus manifest(Attributor &A) override {
2381 return ChangeStatus::UNCHANGED;
2382 }
2383 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2384};
2385} // namespace
2386
2387/// ------------------------ NonNull Argument Attribute ------------------------
2388
2389bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2390 Attribute::AttrKind ImpliedAttributeKind,
2391 bool IgnoreSubsumingPositions) {
2392 SmallVector<Attribute::AttrKind, 2> AttrKinds;
2393 AttrKinds.push_back(Elt: Attribute::NonNull);
2394 if (!NullPointerIsDefined(F: IRP.getAnchorScope(),
2395 AS: IRP.getAssociatedType()->getPointerAddressSpace()))
2396 AttrKinds.push_back(Elt: Attribute::Dereferenceable);
2397 if (A.hasAttr(IRP, AKs: AttrKinds, IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NonNull))
2398 return true;
2399
2400 DominatorTree *DT = nullptr;
2401 AssumptionCache *AC = nullptr;
2402 InformationCache &InfoCache = A.getInfoCache();
2403 if (const Function *Fn = IRP.getAnchorScope()) {
2404 if (!Fn->isDeclaration()) {
2405 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2406 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2407 }
2408 }
2409
2410 SmallVector<AA::ValueAndContext> Worklist;
2411 if (IRP.getPositionKind() != IRP_RETURNED) {
2412 Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2413 } else {
2414 bool UsedAssumedInformation = false;
2415 if (!A.checkForAllInstructions(
2416 Pred: [&](Instruction &I) {
2417 Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2418 return true;
2419 },
2420 Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2421 UsedAssumedInformation, CheckBBLivenessOnly: false, /*CheckPotentiallyDead=*/true))
2422 return false;
2423 }
2424
2425 if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2426 return !isKnownNonZero(
2427 V: VAC.getValue(),
2428 Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2429 }))
2430 return false;
2431
2432 A.manifestAttrs(IRP, DeducedAttrs: {Attribute::get(Context&: IRP.getAnchorValue().getContext(),
2433 Kind: Attribute::NonNull)});
2434 return true;
2435}
2436
2437namespace {
2438static int64_t getKnownNonNullAndDerefBytesForUse(
2439 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2440 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2441 TrackUse = false;
2442
2443 const Value *UseV = U->get();
2444 if (!UseV->getType()->isPointerTy())
2445 return 0;
2446
2447 // We need to follow common pointer manipulation uses to the accesses they
2448 // feed into. We can try to be smart to avoid looking through things we do not
2449 // like for now, e.g., non-inbounds GEPs.
2450 if (isa<CastInst>(Val: I)) {
2451 TrackUse = true;
2452 return 0;
2453 }
2454
2455 if (isa<GetElementPtrInst>(Val: I)) {
2456 TrackUse = true;
2457 return 0;
2458 }
2459
2460 Type *PtrTy = UseV->getType();
2461 const Function *F = I->getFunction();
2462 bool NullPointerIsDefined =
2463 F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2464 const DataLayout &DL = A.getInfoCache().getDL();
2465 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2466 if (CB->isBundleOperand(U)) {
2467 auto OBU = CB->getOperandBundleForOperand(OpIdx: U->getOperandNo());
2468 switch (getBundleAttrFromOBU(OBU)) {
2469 case BundleAttr::NonNull:
2470 IsNonNull = true;
2471 break;
2472 case BundleAttr::Dereferenceable: {
2473 IsNonNull |= !NullPointerIsDefined;
2474 Value *Count = getAssumeDereferenceableInfo(OBU).Count;
2475 if (auto *CI = dyn_cast<ConstantInt>(Val: Count))
2476 return CI->getZExtValue();
2477 } break;
2478 default:
2479 break;
2480 }
2481 return 0;
2482 }
2483
2484 if (CB->isCallee(U)) {
2485 IsNonNull |= !NullPointerIsDefined;
2486 return 0;
2487 }
2488
2489 unsigned ArgNo = CB->getArgOperandNo(U);
2490 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2491 // As long as we only use known information there is no need to track
2492 // dependences here.
2493 bool IsKnownNonNull;
2494 AA::hasAssumedIRAttr<Attribute::NonNull>(A, QueryingAA: &QueryingAA, IRP,
2495 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
2496 IsNonNull |= IsKnownNonNull;
2497 auto *DerefAA =
2498 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2499 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2500 }
2501
2502 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2503 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2504 Loc->Size.isScalable() || I->isVolatile())
2505 return 0;
2506
2507 int64_t Offset;
2508 const Value *Base =
2509 getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2510 if (Base && Base == &AssociatedValue) {
2511 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2512 IsNonNull |= !NullPointerIsDefined;
2513 return std::max(a: int64_t(0), b: DerefBytes);
2514 }
2515
2516 /// Corner case when an offset is 0.
2517 Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2518 /*AllowNonInbounds*/ true);
2519 if (Base && Base == &AssociatedValue && Offset == 0) {
2520 int64_t DerefBytes = Loc->Size.getValue();
2521 IsNonNull |= !NullPointerIsDefined;
2522 return std::max(a: int64_t(0), b: DerefBytes);
2523 }
2524
2525 return 0;
2526}
2527
2528struct AANonNullImpl : AANonNull {
2529 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2530
2531 void initialize(Attributor &A) override {
2532 Value &V = *getAssociatedValue().stripPointerCasts();
2533 if (isa<ConstantPointerNull>(Val: V)) {
2534 indicatePessimisticFixpoint();
2535 return;
2536 }
2537
2538 if (Instruction *CtxI = getCtxI())
2539 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
2540 }
2541
2542 /// See followUsesInMBEC
2543 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2544 AANonNull::StateType &State) {
2545 bool IsNonNull = false;
2546 bool TrackUse = false;
2547 getKnownNonNullAndDerefBytesForUse(A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I,
2548 IsNonNull, TrackUse);
2549 State.setKnown(IsNonNull);
2550 return TrackUse;
2551 }
2552
2553 const std::string getAsStr(Attributor *A) const override {
2554 return getAssumed() ? "nonnull" : "may-null";
2555 }
2556};
2557
2558/// NonNull attribute for a floating value.
2559struct AANonNullFloating : public AANonNullImpl {
2560 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2561 : AANonNullImpl(IRP, A) {}
2562
2563 ChangeStatus updateImpl(Attributor &A) override {
2564 auto CheckIRP = [&](const IRPosition &IRP) {
2565 bool IsKnownNonNull;
2566 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2567 A, QueryingAA: *this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
2568 };
2569
2570 bool Stripped;
2571 bool UsedAssumedInformation = false;
2572 Value *AssociatedValue = &getAssociatedValue();
2573 SmallVector<AA::ValueAndContext> Values;
2574 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2575 S: AA::AnyScope, UsedAssumedInformation))
2576 Stripped = false;
2577 else
2578 Stripped =
2579 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2580
2581 if (!Stripped) {
2582 bool IsKnown;
2583 if (auto *PHI = dyn_cast<PHINode>(Val: AssociatedValue))
2584 if (llvm::all_of(Range: PHI->incoming_values(), P: [&](Value *Op) {
2585 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2586 A, QueryingAA: this, IRP: IRPosition::value(V: *Op), DepClass: DepClassTy::OPTIONAL,
2587 IsKnown);
2588 }))
2589 return ChangeStatus::UNCHANGED;
2590 if (auto *Select = dyn_cast<SelectInst>(Val: AssociatedValue))
2591 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2592 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getFalseValue()),
2593 DepClass: DepClassTy::OPTIONAL, IsKnown) &&
2594 AA::hasAssumedIRAttr<Attribute::NonNull>(
2595 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getTrueValue()),
2596 DepClass: DepClassTy::OPTIONAL, IsKnown))
2597 return ChangeStatus::UNCHANGED;
2598
2599 // If we haven't stripped anything we might still be able to use a
2600 // different AA, but only if the IRP changes. Effectively when we
2601 // interpret this not as a call site value but as a floating/argument
2602 // value.
2603 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2604 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2605 return indicatePessimisticFixpoint();
2606 return ChangeStatus::UNCHANGED;
2607 }
2608
2609 for (const auto &VAC : Values)
2610 if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2611 return indicatePessimisticFixpoint();
2612
2613 return ChangeStatus::UNCHANGED;
2614 }
2615
2616 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2617};
2618
2619/// NonNull attribute for function return value.
2620struct AANonNullReturned final
2621 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2622 false, AANonNull::IRAttributeKind, false> {
2623 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2624 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2625 false, Attribute::NonNull, false>(IRP, A) {
2626 }
2627
2628 const std::string getAsStr(Attributor *A) const override {
2629 return getAssumed() ? "nonnull" : "may-null";
2630 }
2631
2632 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2633};
2634
2635/// NonNull attribute for function argument.
2636struct AANonNullArgument final
2637 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2638 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2639 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2640
2641 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2642};
2643
2644struct AANonNullCallSiteArgument final : AANonNullFloating {
2645 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2646 : AANonNullFloating(IRP, A) {}
2647
2648 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2649};
2650
2651/// NonNull attribute for a call site return position.
2652struct AANonNullCallSiteReturned final
2653 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2654 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2655 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2656
2657 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2658};
2659} // namespace
2660
2661/// ------------------------ Must-Progress Attributes --------------------------
2662namespace {
2663struct AAMustProgressImpl : public AAMustProgress {
2664 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2665 : AAMustProgress(IRP, A) {}
2666
2667 void initialize(Attributor &A) override {
2668 bool IsKnown;
2669 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2670 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2671 (void)IsKnown;
2672 }
2673
2674 const std::string getAsStr(Attributor *A) const override {
2675 return getAssumed() ? "mustprogress" : "may-not-progress";
2676 }
2677};
2678
2679struct AAMustProgressFunction final : AAMustProgressImpl {
2680 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2681 : AAMustProgressImpl(IRP, A) {}
2682
2683 ChangeStatus updateImpl(Attributor &A) override {
2684 bool IsKnown;
2685 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2686 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown)) {
2687 if (IsKnown)
2688 return indicateOptimisticFixpoint();
2689 return ChangeStatus::UNCHANGED;
2690 }
2691
2692 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2693 IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2694 bool IsKnownMustProgress;
2695 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2696 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress,
2697 /* IgnoreSubsumingPositions */ true);
2698 };
2699
2700 bool AllCallSitesKnown = true;
2701 if (!A.checkForAllCallSites(Pred: CheckForMustProgress, QueryingAA: *this,
2702 /* RequireAllCallSites */ true,
2703 UsedAssumedInformation&: AllCallSitesKnown))
2704 return indicatePessimisticFixpoint();
2705
2706 return ChangeStatus::UNCHANGED;
2707 }
2708
2709 void trackStatistics() const override {
2710 STATS_DECLTRACK_FN_ATTR(mustprogress)
2711 }
2712};
2713
2714/// MustProgress attribute deduction for a call sites.
2715struct AAMustProgressCallSite final : AAMustProgressImpl {
2716 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2717 : AAMustProgressImpl(IRP, A) {}
2718
2719 ChangeStatus updateImpl(Attributor &A) override {
2720 // TODO: Once we have call site specific value information we can provide
2721 // call site specific liveness information and then it makes
2722 // sense to specialize attributes for call sites arguments instead of
2723 // redirecting requests to the callee argument.
2724 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2725 bool IsKnownMustProgress;
2726 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2727 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress))
2728 return indicatePessimisticFixpoint();
2729 return ChangeStatus::UNCHANGED;
2730 }
2731
2732 void trackStatistics() const override {
2733 STATS_DECLTRACK_CS_ATTR(mustprogress);
2734 }
2735};
2736} // namespace
2737
2738/// ------------------------ No-Recurse Attributes ----------------------------
2739
2740namespace {
2741struct AANoRecurseImpl : public AANoRecurse {
2742 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2743
2744 void initialize(Attributor &A) override {
2745 bool IsKnown;
2746 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2747 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2748 (void)IsKnown;
2749 }
2750
2751 const std::string getAsStr(Attributor *A) const override {
2752 return getAssumed() ? "norecurse" : "may-recurse";
2753 }
2754};
2755
2756struct AANoRecurseFunction final : AANoRecurseImpl {
2757 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2758 : AANoRecurseImpl(IRP, A) {}
2759
2760 ChangeStatus updateImpl(Attributor &A) override {
2761
2762 // If all live call sites are known to be no-recurse, we are as well.
2763 auto CallSitePred = [&](AbstractCallSite ACS) {
2764 bool IsKnownNoRecurse;
2765 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2766 A, QueryingAA: this,
2767 IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
2768 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoRecurse))
2769 return false;
2770 return IsKnownNoRecurse;
2771 };
2772 bool UsedAssumedInformation = false;
2773 if (A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
2774 UsedAssumedInformation)) {
2775 // If we know all call sites and all are known no-recurse, we are done.
2776 // If all known call sites, which might not be all that exist, are known
2777 // to be no-recurse, we are not done but we can continue to assume
2778 // no-recurse. If one of the call sites we have not visited will become
2779 // live, another update is triggered.
2780 if (!UsedAssumedInformation)
2781 indicateOptimisticFixpoint();
2782 return ChangeStatus::UNCHANGED;
2783 }
2784
2785 const AAInterFnReachability *EdgeReachability =
2786 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: getIRPosition(),
2787 DepClass: DepClassTy::REQUIRED);
2788 if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2789 return indicatePessimisticFixpoint();
2790 return ChangeStatus::UNCHANGED;
2791 }
2792
2793 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2794};
2795
2796/// NoRecurse attribute deduction for a call sites.
2797struct AANoRecurseCallSite final
2798 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2799 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2800 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2801
2802 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2803};
2804} // namespace
2805
2806/// ------------------------ No-Convergent Attribute --------------------------
2807
2808namespace {
2809struct AANonConvergentImpl : public AANonConvergent {
2810 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2811 : AANonConvergent(IRP, A) {}
2812
2813 const std::string getAsStr(Attributor *A) const override {
2814 return getAssumed() ? "non-convergent" : "may-be-convergent";
2815 }
2816};
2817
2818struct AANonConvergentFunction final : AANonConvergentImpl {
2819 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2820 : AANonConvergentImpl(IRP, A) {}
2821
2822 ChangeStatus updateImpl(Attributor &A) override {
2823 // If all function calls are known to not be convergent, we are not
2824 // convergent.
2825 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2826 CallBase &CB = cast<CallBase>(Val&: Inst);
2827 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2828 if (!Callee || Callee->isIntrinsic()) {
2829 return false;
2830 }
2831 if (Callee->isDeclaration()) {
2832 return !Callee->hasFnAttribute(Kind: Attribute::Convergent);
2833 }
2834 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2835 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2836 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2837 };
2838
2839 bool UsedAssumedInformation = false;
2840 if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2841 UsedAssumedInformation)) {
2842 return indicatePessimisticFixpoint();
2843 }
2844 return ChangeStatus::UNCHANGED;
2845 }
2846
2847 ChangeStatus manifest(Attributor &A) override {
2848 if (isKnownNotConvergent() &&
2849 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::Convergent)) {
2850 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::Convergent});
2851 return ChangeStatus::CHANGED;
2852 }
2853 return ChangeStatus::UNCHANGED;
2854 }
2855
2856 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2857};
2858} // namespace
2859
2860/// -------------------- Undefined-Behavior Attributes ------------------------
2861
2862namespace {
2863struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2864 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2865 : AAUndefinedBehavior(IRP, A) {}
2866
2867 struct UBInfo {
2868 enum Kind {
2869 NullPtrAccess,
2870 UndefPtrAccess,
2871 UndefBranchCondition,
2872 UndefReturnValue,
2873 NullReturnViolatesNonNull,
2874 UndefCallArgument,
2875 NullArgViolatesNonNull,
2876 };
2877
2878 Kind K;
2879 std::optional<unsigned> ArgNo;
2880
2881 UBInfo(Kind K) : K(K), ArgNo(std::nullopt) {}
2882
2883 UBInfo(Kind K, std::optional<unsigned> ArgNo) : K(K), ArgNo(ArgNo) {}
2884 };
2885
2886 ChangeStatus updateImpl(Attributor &A) override {
2887 const size_t UBPrevSize = KnownUBInsts.size();
2888 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2889
2890 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2891 // Volatile accesses on null are not necessarily UB.
2892 if (I.isVolatile())
2893 return true;
2894
2895 // Skip instructions that are already saved.
2896 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
2897 return true;
2898
2899 // If we reach here, we know we have an instruction
2900 // that accesses memory through a pointer operand,
2901 // for which getPointerOperand() should give it to us.
2902 Value *PtrOp =
2903 const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2904 assert(PtrOp &&
2905 "Expected pointer operand of memory accessing instruction");
2906
2907 // Either we stopped and the appropriate action was taken,
2908 // or we got back a simplified value to continue.
2909 std::optional<Value *> SimplifiedPtrOp =
2910 stopOnUndefOrAssumed(A, V: PtrOp, I: &I, K: UBInfo::UndefPtrAccess);
2911 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2912 return true;
2913 const Value *PtrOpVal = *SimplifiedPtrOp;
2914
2915 // A memory access through a pointer is considered UB
2916 // only if the pointer has constant null value.
2917 // TODO: Expand it to not only check constant values.
2918 if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2919 AssumedNoUBInsts.insert(Ptr: &I);
2920 return true;
2921 }
2922 const Type *PtrTy = PtrOpVal->getType();
2923
2924 // Because we only consider instructions inside functions,
2925 // assume that a parent function exists.
2926 const Function *F = I.getFunction();
2927
2928 // A memory access using constant null pointer is only considered UB
2929 // if null pointer is _not_ defined for the target platform.
2930 if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
2931 AssumedNoUBInsts.insert(Ptr: &I);
2932 else
2933 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullPtrAccess);
2934 return true;
2935 };
2936
2937 auto InspectBrInstForUB = [&](Instruction &I) {
2938 // A conditional branch instruction is considered UB if it has `undef`
2939 // condition.
2940
2941 // Skip instructions that are already saved.
2942 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
2943 return true;
2944
2945 // We know we have a branch instruction.
2946 auto *BrInst = cast<CondBrInst>(Val: &I);
2947
2948 // Either we stopped and the appropriate action was taken,
2949 // or we got back a simplified value to continue.
2950 std::optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(
2951 A, V: BrInst->getCondition(), I: BrInst, K: UBInfo::UndefBranchCondition);
2952 if (!SimplifiedCond || !*SimplifiedCond)
2953 return true;
2954 AssumedNoUBInsts.insert(Ptr: &I);
2955 return true;
2956 };
2957
2958 auto InspectCallSiteForUB = [&](Instruction &I) {
2959 // Check whether a callsite always cause UB or not
2960
2961 // Skip instructions that are already saved.
2962 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
2963 return true;
2964
2965 // Check nonnull and noundef argument attribute violation for each
2966 // callsite.
2967 CallBase &CB = cast<CallBase>(Val&: I);
2968 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2969 if (!Callee)
2970 return true;
2971 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
2972 // If current argument is known to be simplified to null pointer and the
2973 // corresponding argument position is known to have nonnull attribute,
2974 // the argument is poison. Furthermore, if the argument is poison and
2975 // the position is known to have noundef attriubte, this callsite is
2976 // considered UB.
2977 if (idx >= Callee->arg_size())
2978 break;
2979 Value *ArgVal = CB.getArgOperand(i: idx);
2980 if (!ArgVal)
2981 continue;
2982 // Here, we handle three cases.
2983 // (1) Not having a value means it is dead. (we can replace the value
2984 // with undef)
2985 // (2) Simplified to undef. The argument violate noundef attriubte.
2986 // (3) Simplified to null pointer where known to be nonnull.
2987 // The argument is a poison value and violate noundef attribute.
2988 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
2989 bool IsKnownNoUndef;
2990 AA::hasAssumedIRAttr<Attribute::NoUndef>(
2991 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
2992 if (!IsKnownNoUndef)
2993 continue;
2994 bool UsedAssumedInformation = false;
2995 std::optional<Value *> SimplifiedVal =
2996 A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
2997 UsedAssumedInformation, S: AA::Interprocedural);
2998 if (UsedAssumedInformation)
2999 continue;
3000 if (SimplifiedVal && !*SimplifiedVal)
3001 return true;
3002 if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3003 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo(UBInfo::UndefCallArgument, idx));
3004 continue;
3005 }
3006 if (!ArgVal->getType()->isPointerTy() ||
3007 !isa<ConstantPointerNull>(Val: **SimplifiedVal))
3008 continue;
3009 bool IsKnownNonNull;
3010 AA::hasAssumedIRAttr<Attribute::NonNull>(
3011 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
3012 if (IsKnownNonNull)
3013 KnownUBInsts.try_emplace(Key: &I,
3014 Args: UBInfo(UBInfo::NullArgViolatesNonNull, idx));
3015 }
3016 return true;
3017 };
3018
3019 auto InspectReturnInstForUB = [&](Instruction &I) {
3020 auto &RI = cast<ReturnInst>(Val&: I);
3021 // Either we stopped and the appropriate action was taken,
3022 // or we got back a simplified return value to continue.
3023 std::optional<Value *> SimplifiedRetValue = stopOnUndefOrAssumed(
3024 A, V: RI.getReturnValue(), I: &I, K: UBInfo::UndefReturnValue);
3025 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3026 return true;
3027
3028 // Check if a return instruction always cause UB or not
3029 // Note: It is guaranteed that the returned position of the anchor
3030 // scope has noundef attribute when this is called.
3031 // We also ensure the return position is not "assumed dead"
3032 // because the returned value was then potentially simplified to
3033 // `undef` in AAReturnedValues without removing the `noundef`
3034 // attribute yet.
3035
3036 // When the returned position has noundef attriubte, UB occurs in the
3037 // following cases.
3038 // (1) Returned value is known to be undef.
3039 // (2) The value is known to be a null pointer and the returned
3040 // position has nonnull attribute (because the returned value is
3041 // poison).
3042 if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3043 bool IsKnownNonNull;
3044 AA::hasAssumedIRAttr<Attribute::NonNull>(
3045 A, QueryingAA: this, IRP: IRPosition::returned(F: *getAnchorScope()), DepClass: DepClassTy::NONE,
3046 IsKnown&: IsKnownNonNull);
3047 if (IsKnownNonNull)
3048 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullReturnViolatesNonNull);
3049 }
3050
3051 return true;
3052 };
3053
3054 bool UsedAssumedInformation = false;
3055 A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3056 Opcodes: {Instruction::Load, Instruction::Store,
3057 Instruction::AtomicCmpXchg,
3058 Instruction::AtomicRMW},
3059 UsedAssumedInformation,
3060 /* CheckBBLivenessOnly */ true);
3061 A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::CondBr},
3062 UsedAssumedInformation,
3063 /* CheckBBLivenessOnly */ true);
3064 A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3065 UsedAssumedInformation);
3066
3067 // If the returned position of the anchor scope has noundef attriubte, check
3068 // all returned instructions.
3069 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3070 const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3071 if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3072 bool IsKnownNoUndef;
3073 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3074 A, QueryingAA: this, IRP: ReturnIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3075 if (IsKnownNoUndef)
3076 A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3077 Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3078 /* CheckBBLivenessOnly */ true);
3079 }
3080 }
3081
3082 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3083 UBPrevSize != KnownUBInsts.size())
3084 return ChangeStatus::CHANGED;
3085 return ChangeStatus::UNCHANGED;
3086 }
3087
3088 bool isKnownToCauseUB(Instruction *I) const override {
3089 return KnownUBInsts.count(Key: I);
3090 }
3091
3092 bool isAssumedToCauseUB(Instruction *I) const override {
3093 // In simple words, if an instruction is not in the assumed to _not_
3094 // cause UB, then it is assumed UB (that includes those
3095 // in the KnownUBInsts set). The rest is boilerplate
3096 // is to ensure that it is one of the instructions we test
3097 // for UB.
3098
3099 switch (I->getOpcode()) {
3100 case Instruction::Load:
3101 case Instruction::Store:
3102 case Instruction::AtomicCmpXchg:
3103 case Instruction::AtomicRMW:
3104 case Instruction::CondBr:
3105 return !AssumedNoUBInsts.count(Ptr: I);
3106 default:
3107 return false;
3108 }
3109 return false;
3110 }
3111
3112 /// Emit an optimization remark explaining why \p I is known to cause UB,
3113 /// per \p Info, right before it is replaced with 'unreachable'.
3114 static void emitUBRemark(Attributor &A, Instruction *I, const UBInfo &Info) {
3115 auto Remark = [&](OptimizationRemark OR) {
3116 switch (Info.K) {
3117 case UBInfo::NullPtrAccess:
3118 case UBInfo::UndefPtrAccess: {
3119 return OR << "Memory access through a pointer known to be "
3120 << ore::NV("Pointer",
3121 getPointerOperand(I, /*AllowVolatile*/ true))
3122 << " is undefined behavior; replacing with 'unreachable'.";
3123 }
3124 case UBInfo::UndefBranchCondition:
3125 return OR << "Branch condition known to be "
3126 << ore::NV("Condition", cast<CondBrInst>(Val: I)->getCondition())
3127 << " is undefined behavior; replacing with 'unreachable'.";
3128 case UBInfo::UndefReturnValue:
3129 case UBInfo::NullReturnViolatesNonNull:
3130 return OR << "Value returned known to be "
3131 << ore::NV("ReturnValue",
3132 cast<ReturnInst>(Val: I)->getReturnValue())
3133 << " is undefined behavior; replacing with 'unreachable'.";
3134 case UBInfo::UndefCallArgument:
3135 case UBInfo::NullArgViolatesNonNull: {
3136 bool IsUndef = Info.K == UBInfo::UndefCallArgument;
3137 CallBase &CB = *cast<CallBase>(Val: I);
3138 OR << "Argument " << ore::NV("ArgNo", *Info.ArgNo)
3139 << " passed to parameter of ";
3140 if (auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand()))
3141 OR << ore::NV("Callee", Callee);
3142 else
3143 OR << "the callee";
3144 return OR << " known to be "
3145 << ore::NV("Argument", IsUndef ? "undef" : "null")
3146 << " is undefined behavior; replacing with 'unreachable'.";
3147 }
3148 }
3149 llvm_unreachable("Unknown UBInfo::Kind");
3150 };
3151 A.emitRemark<OptimizationRemark>(I, RemarkName: "UndefinedBehavior", RemarkCB&: Remark);
3152 }
3153
3154 ChangeStatus manifest(Attributor &A) override {
3155 if (KnownUBInsts.empty())
3156 return ChangeStatus::UNCHANGED;
3157 for (const auto &[I, Info] : KnownUBInsts) {
3158 emitUBRemark(A, I, Info);
3159 A.changeToUnreachableAfterManifest(I);
3160 }
3161 return ChangeStatus::CHANGED;
3162 }
3163
3164 const std::string getAsStr(Attributor *A) const override {
3165 return getAssumed() ? "undefined-behavior" : "no-ub";
3166 }
3167
3168 /// Note: The correctness of this analysis depends on the fact that the
3169 /// following 2 sets will stop changing after some point.
3170 /// "Change" here means that their size changes.
3171 /// The size of each set is monotonically increasing
3172 /// (we only add items to them) and it is upper bounded by the number of
3173 /// instructions in the processed function (we can never save more
3174 /// elements in either set than this number). Hence, at some point,
3175 /// they will stop increasing.
3176 /// Consequently, at some point, both sets will have stopped
3177 /// changing, effectively making the analysis reach a fixpoint.
3178
3179 /// Note: These 2 sets are disjoint and an instruction can be considered
3180 /// one of 3 things:
3181 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3182 /// the KnownUBInsts set.
3183 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3184 /// has a reason to assume it).
3185 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3186 /// could not find a reason to assume or prove that it can cause UB,
3187 /// hence it assumes it doesn't. We have a set for these instructions
3188 /// so that we don't reprocess them in every update.
3189 /// Note however that instructions in this set may cause UB.
3190
3191protected:
3192 /// A map from all live instructions _known_ to cause UB to the reason why,
3193 /// used to build actionable optimization remarks in manifest().
3194 MapVector<Instruction *, UBInfo> KnownUBInsts;
3195
3196private:
3197 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3198 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3199
3200 // Should be called on updates in which if we're processing an instruction
3201 // \p I that depends on a value \p V, one of the following has to happen:
3202 // - If the value is assumed, then stop.
3203 // - If the value is known but undef, then consider it UB for \p K.
3204 // - Otherwise, do specific processing with the simplified value.
3205 // We return std::nullopt in the first 2 cases to signify that an appropriate
3206 // action was taken and the caller should stop.
3207 // Otherwise, we return the simplified value that the caller should
3208 // use for specific processing.
3209 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3210 Instruction *I, UBInfo::Kind K) {
3211 bool UsedAssumedInformation = false;
3212 std::optional<Value *> SimplifiedV =
3213 A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3214 UsedAssumedInformation, S: AA::Interprocedural);
3215 if (!UsedAssumedInformation) {
3216 // Don't depend on assumed values.
3217 if (!SimplifiedV) {
3218 // If it is known (which we tested above) but it doesn't have a value,
3219 // then we can assume `undef` and hence the instruction is UB.
3220 KnownUBInsts.try_emplace(Key: I, Args&: K);
3221 return std::nullopt;
3222 }
3223 if (!*SimplifiedV)
3224 return nullptr;
3225 V = *SimplifiedV;
3226 }
3227 if (isa<UndefValue>(Val: V)) {
3228 KnownUBInsts.try_emplace(Key: I, Args&: K);
3229 return std::nullopt;
3230 }
3231 return V;
3232 }
3233};
3234
3235struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3236 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3237 : AAUndefinedBehaviorImpl(IRP, A) {}
3238
3239 void trackStatistics() const override {
3240 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3241 "Number of instructions known to have UB");
3242 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3243 KnownUBInsts.size();
3244 }
3245};
3246} // namespace
3247
3248/// ------------------------ Will-Return Attributes ----------------------------
3249
3250namespace {
3251// Helper function that checks whether a function has any cycle which we don't
3252// know if it is bounded or not.
3253// Loops with maximum trip count are considered bounded, any other cycle not.
3254static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3255 ScalarEvolution *SE =
3256 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3257 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3258 // If either SCEV or LoopInfo is not available for the function then we assume
3259 // any cycle to be unbounded cycle.
3260 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3261 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3262 if (!SE || !LI) {
3263 for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3264 if (SCCI.hasCycle())
3265 return true;
3266 return false;
3267 }
3268
3269 // If there's irreducible control, the function may contain non-loop cycles.
3270 if (mayContainIrreducibleControl(F, LI))
3271 return true;
3272
3273 // Any loop that does not have a max trip count is considered unbounded cycle.
3274 for (auto *L : LI->getLoopsInPreorder()) {
3275 if (!SE->getSmallConstantMaxTripCount(L))
3276 return true;
3277 }
3278 return false;
3279}
3280
3281struct AAWillReturnImpl : public AAWillReturn {
3282 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3283 : AAWillReturn(IRP, A) {}
3284
3285 void initialize(Attributor &A) override {
3286 bool IsKnown;
3287 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3288 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3289 (void)IsKnown;
3290 }
3291
3292 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3293 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3294 if (!A.hasAttr(IRP: getIRPosition(), AKs: {Attribute::MustProgress}))
3295 return false;
3296
3297 bool IsKnown;
3298 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3299 return IsKnown || !KnownOnly;
3300 return false;
3301 }
3302
3303 ChangeStatus updateImpl(Attributor &A) override {
3304 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3305 return ChangeStatus::UNCHANGED;
3306
3307 auto CheckForWillReturn = [&](Instruction &I) {
3308 IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3309 bool IsKnown;
3310 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3311 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown)) {
3312 if (IsKnown)
3313 return true;
3314 } else {
3315 return false;
3316 }
3317 bool IsKnownNoRecurse;
3318 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3319 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoRecurse);
3320 };
3321
3322 bool UsedAssumedInformation = false;
3323 if (!A.checkForAllCallLikeInstructions(Pred: CheckForWillReturn, QueryingAA: *this,
3324 UsedAssumedInformation))
3325 return indicatePessimisticFixpoint();
3326
3327 auto CheckForVolatile = [&](Instruction &I) {
3328 // Volatile operations are not willreturn.
3329 return !I.isVolatile();
3330 };
3331 if (!A.checkForAllInstructions(Pred: CheckForVolatile, QueryingAA: *this,
3332 Opcodes: {Instruction::Load, Instruction::Store,
3333 Instruction::AtomicCmpXchg,
3334 Instruction::AtomicRMW},
3335 UsedAssumedInformation))
3336 return indicatePessimisticFixpoint();
3337
3338 return ChangeStatus::UNCHANGED;
3339 }
3340
3341 const std::string getAsStr(Attributor *A) const override {
3342 return getAssumed() ? "willreturn" : "may-noreturn";
3343 }
3344};
3345
3346struct AAWillReturnFunction final : AAWillReturnImpl {
3347 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3348 : AAWillReturnImpl(IRP, A) {}
3349
3350 void initialize(Attributor &A) override {
3351 AAWillReturnImpl::initialize(A);
3352
3353 Function *F = getAnchorScope();
3354 assert(F && "Did expect an anchor function");
3355 if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3356 indicatePessimisticFixpoint();
3357 }
3358
3359 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3360};
3361
3362/// WillReturn attribute deduction for a call sites.
3363struct AAWillReturnCallSite final
3364 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3365 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3366 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3367
3368 ChangeStatus updateImpl(Attributor &A) override {
3369 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3370 return ChangeStatus::UNCHANGED;
3371
3372 return AACalleeToCallSite::updateImpl(A);
3373 }
3374
3375 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3376};
3377} // namespace
3378
3379/// -------------------AAIntraFnReachability Attribute--------------------------
3380
3381/// All information associated with a reachability query. This boilerplate code
3382/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3383/// different \p ToTy values.
3384template <typename ToTy> struct ReachabilityQueryInfo {
3385 enum class Reachable {
3386 No,
3387 Yes,
3388 };
3389
3390 /// Start here,
3391 const Instruction *From = nullptr;
3392 /// reach this place,
3393 const ToTy *To = nullptr;
3394 /// without going through any of these instructions,
3395 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3396 /// and remember if it worked:
3397 Reachable Result = Reachable::No;
3398
3399 /// Precomputed hash for this RQI.
3400 unsigned Hash = 0;
3401
3402 unsigned computeHashValue() const {
3403 assert(Hash == 0 && "Computed hash twice!");
3404 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3405 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3406 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3407 detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3408 b: InstSetDMI::getHashValue(BES: ExclusionSet));
3409 }
3410
3411 ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3412 : From(From), To(To) {}
3413
3414 /// Constructor replacement to ensure unique and stable sets are used for the
3415 /// cache.
3416 ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3417 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3418 : From(&From), To(&To), ExclusionSet(ES) {
3419
3420 if (!ES || ES->empty()) {
3421 ExclusionSet = nullptr;
3422 } else if (MakeUnique) {
3423 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3424 }
3425 }
3426
3427 ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3428 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3429};
3430
3431namespace llvm {
3432template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3433 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3434 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3435
3436 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3437 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3438 }
3439 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3440 const ReachabilityQueryInfo<ToTy> *RHS) {
3441 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3442 return false;
3443 return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3444 }
3445};
3446
3447} // namespace llvm
3448
3449namespace {
3450
3451template <typename BaseTy, typename ToTy>
3452struct CachedReachabilityAA : public BaseTy {
3453 using RQITy = ReachabilityQueryInfo<ToTy>;
3454
3455 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3456
3457 bool isQueryAA() const override { return true; }
3458
3459 ChangeStatus updateImpl(Attributor &A) override {
3460 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3461 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3462 RQITy *RQI = QueryVector[u];
3463 if (RQI->Result == RQITy::Reachable::No &&
3464 isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3465 Changed = ChangeStatus::CHANGED;
3466 }
3467 return Changed;
3468 }
3469
3470 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3471 bool IsTemporaryRQI) = 0;
3472
3473 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3474 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3475 RQI.Result = Result;
3476
3477 // Remove the temporary RQI from the cache.
3478 if (IsTemporaryRQI)
3479 QueryCache.erase(&RQI);
3480
3481 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3482 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3483 // this query. 2) We did not use the exclusion set, potentially because
3484 // there is none.
3485 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3486 RQITy PlainRQI(RQI.From, RQI.To);
3487 if (!QueryCache.count(&PlainRQI)) {
3488 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3489 RQIPtr->Result = Result;
3490 QueryVector.push_back(RQIPtr);
3491 QueryCache.insert(RQIPtr);
3492 }
3493 }
3494
3495 // Check if we need to insert a new permanent RQI with the exclusion set.
3496 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3497 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3498 "Did not expect empty set!");
3499 RQITy *RQIPtr = new (A.Allocator)
3500 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3501 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3502 RQIPtr->Result = Result;
3503 assert(!QueryCache.count(RQIPtr));
3504 QueryVector.push_back(RQIPtr);
3505 QueryCache.insert(RQIPtr);
3506 }
3507
3508 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3509 A.registerForUpdate(AA&: *this);
3510 return Result == RQITy::Reachable::Yes;
3511 }
3512
3513 const std::string getAsStr(Attributor *A) const override {
3514 // TODO: Return the number of reachable queries.
3515 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3516 }
3517
3518 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3519 typename RQITy::Reachable &Result) {
3520 if (!this->getState().isValidState()) {
3521 Result = RQITy::Reachable::Yes;
3522 return true;
3523 }
3524
3525 // If we have an exclusion set we might be able to find our answer by
3526 // ignoring it first.
3527 if (StackRQI.ExclusionSet) {
3528 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3529 auto It = QueryCache.find(&PlainRQI);
3530 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3531 Result = RQITy::Reachable::No;
3532 return true;
3533 }
3534 }
3535
3536 auto It = QueryCache.find(&StackRQI);
3537 if (It != QueryCache.end()) {
3538 Result = (*It)->Result;
3539 return true;
3540 }
3541
3542 // Insert a temporary for recursive queries. We will replace it with a
3543 // permanent entry later.
3544 QueryCache.insert(&StackRQI);
3545 return false;
3546 }
3547
3548private:
3549 SmallVector<RQITy *> QueryVector;
3550 DenseSet<RQITy *> QueryCache;
3551};
3552
3553struct AAIntraFnReachabilityFunction final
3554 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3555 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3556 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3557 : Base(IRP, A) {
3558 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3559 F: *IRP.getAssociatedFunction());
3560 }
3561
3562 bool isAssumedReachable(
3563 Attributor &A, const Instruction &From, const Instruction &To,
3564 const AA::InstExclusionSetTy *ExclusionSet) const override {
3565 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3566 if (&From == &To)
3567 return true;
3568
3569 RQITy StackRQI(A, From, To, ExclusionSet, false);
3570 RQITy::Reachable Result;
3571 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3572 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3573 /*IsTemporaryRQI=*/true);
3574 return Result == RQITy::Reachable::Yes;
3575 }
3576
3577 ChangeStatus updateImpl(Attributor &A) override {
3578 // We only depend on liveness. DeadEdges is all we care about, check if any
3579 // of them changed.
3580 auto *LivenessAA =
3581 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3582 if (LivenessAA &&
3583 llvm::all_of(Range&: DeadEdges,
3584 P: [&](const auto &DeadEdge) {
3585 return LivenessAA->isEdgeDead(From: DeadEdge.first,
3586 To: DeadEdge.second);
3587 }) &&
3588 llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3589 return LivenessAA->isAssumedDead(BB);
3590 })) {
3591 return ChangeStatus::UNCHANGED;
3592 }
3593 DeadEdges.clear();
3594 DeadBlocks.clear();
3595 return Base::updateImpl(A);
3596 }
3597
3598 bool isReachableImpl(Attributor &A, RQITy &RQI,
3599 bool IsTemporaryRQI) override {
3600 const Instruction *Origin = RQI.From;
3601 bool UsedExclusionSet = false;
3602
3603 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3604 const AA::InstExclusionSetTy *ExclusionSet) {
3605 const Instruction *IP = &From;
3606 while (IP && IP != &To) {
3607 if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3608 UsedExclusionSet = true;
3609 break;
3610 }
3611 IP = IP->getNextNode();
3612 }
3613 return IP == &To;
3614 };
3615
3616 const BasicBlock *FromBB = RQI.From->getParent();
3617 const BasicBlock *ToBB = RQI.To->getParent();
3618 assert(FromBB->getParent() == ToBB->getParent() &&
3619 "Not an intra-procedural query!");
3620
3621 // Check intra-block reachability, however, other reaching paths are still
3622 // possible.
3623 if (FromBB == ToBB &&
3624 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3625 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3626 IsTemporaryRQI);
3627
3628 // Check if reaching the ToBB block is sufficient or if even that would not
3629 // ensure reaching the target. In the latter case we are done.
3630 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3631 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3632 IsTemporaryRQI);
3633
3634 const Function *Fn = FromBB->getParent();
3635 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3636 if (RQI.ExclusionSet)
3637 for (auto *I : *RQI.ExclusionSet)
3638 if (I->getFunction() == Fn)
3639 ExclusionBlocks.insert(Ptr: I->getParent());
3640
3641 // Check if we make it out of the FromBB block at all.
3642 if (ExclusionBlocks.count(Ptr: FromBB) &&
3643 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3644 RQI.ExclusionSet))
3645 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3646
3647 auto *LivenessAA =
3648 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3649 if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3650 DeadBlocks.insert(V: ToBB);
3651 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3652 IsTemporaryRQI);
3653 }
3654
3655 SmallPtrSet<const BasicBlock *, 16> Visited;
3656 SmallVector<const BasicBlock *, 16> Worklist;
3657 Worklist.push_back(Elt: FromBB);
3658
3659 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3660 while (!Worklist.empty()) {
3661 const BasicBlock *BB = Worklist.pop_back_val();
3662 if (!Visited.insert(Ptr: BB).second)
3663 continue;
3664 for (const BasicBlock *SuccBB : successors(BB)) {
3665 if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3666 LocalDeadEdges.insert(V: {BB, SuccBB});
3667 continue;
3668 }
3669 // We checked before if we just need to reach the ToBB block.
3670 if (SuccBB == ToBB)
3671 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3672 IsTemporaryRQI);
3673 if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3674 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3675 IsTemporaryRQI);
3676
3677 if (ExclusionBlocks.count(Ptr: SuccBB)) {
3678 UsedExclusionSet = true;
3679 continue;
3680 }
3681 Worklist.push_back(Elt: SuccBB);
3682 }
3683 }
3684
3685 DeadEdges.insert_range(R&: LocalDeadEdges);
3686 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3687 IsTemporaryRQI);
3688 }
3689
3690 void trackStatistics() const override {}
3691
3692private:
3693 // Set of assumed dead blocks we used in the last query. If any changes we
3694 // update the state.
3695 DenseSet<const BasicBlock *> DeadBlocks;
3696
3697 // Set of assumed dead edges we used in the last query. If any changes we
3698 // update the state.
3699 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3700
3701 /// The dominator tree of the function to short-circuit reasoning.
3702 const DominatorTree *DT = nullptr;
3703};
3704} // namespace
3705
3706/// ------------------------ NoAlias Argument Attribute ------------------------
3707
3708bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3709 Attribute::AttrKind ImpliedAttributeKind,
3710 bool IgnoreSubsumingPositions) {
3711 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3712 "Unexpected attribute kind");
3713 Value *Val = &IRP.getAssociatedValue();
3714 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3715 if (isa<AllocaInst>(Val))
3716 return true;
3717 } else {
3718 IgnoreSubsumingPositions = true;
3719 }
3720
3721 if (isa<UndefValue>(Val))
3722 return true;
3723
3724 if (isa<ConstantPointerNull>(Val) &&
3725 !NullPointerIsDefined(F: IRP.getAnchorScope(),
3726 AS: Val->getType()->getPointerAddressSpace()))
3727 return true;
3728
3729 if (A.hasAttr(IRP, AKs: {Attribute::ByVal, Attribute::NoAlias},
3730 IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NoAlias))
3731 return true;
3732
3733 return false;
3734}
3735
3736namespace {
3737struct AANoAliasImpl : AANoAlias {
3738 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3739 assert(getAssociatedType()->isPointerTy() &&
3740 "Noalias is a pointer attribute");
3741 }
3742
3743 const std::string getAsStr(Attributor *A) const override {
3744 return getAssumed() ? "noalias" : "may-alias";
3745 }
3746};
3747
3748/// NoAlias attribute for a floating value.
3749struct AANoAliasFloating final : AANoAliasImpl {
3750 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3751 : AANoAliasImpl(IRP, A) {}
3752
3753 ChangeStatus updateImpl(Attributor &A) override {
3754 // TODO: Implement this.
3755 return indicatePessimisticFixpoint();
3756 }
3757
3758 void trackStatistics() const override {
3759 STATS_DECLTRACK_FLOATING_ATTR(noalias)
3760 }
3761};
3762
3763/// NoAlias attribute for an argument.
3764struct AANoAliasArgument final
3765 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3766 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3767 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3768
3769 ChangeStatus updateImpl(Attributor &A) override {
3770 // We have to make sure no-alias on the argument does not break
3771 // synchronization when this is a callback argument, see also [1] below.
3772 // If synchronization cannot be affected, we delegate to the base updateImpl
3773 // function, otherwise we give up for now.
3774
3775 // If the function is no-sync, no-alias cannot break synchronization.
3776 bool IsKnownNoSycn;
3777 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3778 A, QueryingAA: this, IRP: IRPosition::function_scope(IRP: getIRPosition()),
3779 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn))
3780 return Base::updateImpl(A);
3781
3782 // If the argument is read-only, no-alias cannot break synchronization.
3783 bool IsKnown;
3784 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3785 return Base::updateImpl(A);
3786
3787 // If the argument is never passed through callbacks, no-alias cannot break
3788 // synchronization.
3789 bool UsedAssumedInformation = false;
3790 if (A.checkForAllCallSites(
3791 Pred: [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, QueryingAA: *this,
3792 RequireAllCallSites: true, UsedAssumedInformation))
3793 return Base::updateImpl(A);
3794
3795 // TODO: add no-alias but make sure it doesn't break synchronization by
3796 // introducing fake uses. See:
3797 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3798 // International Workshop on OpenMP 2018,
3799 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3800
3801 return indicatePessimisticFixpoint();
3802 }
3803
3804 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3805};
3806
3807struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3808 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3809 : AANoAliasImpl(IRP, A) {}
3810
3811 /// Determine if the underlying value may alias with the call site argument
3812 /// \p OtherArgNo of \p ICS (= the underlying call site).
3813 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3814 const AAMemoryBehavior &MemBehaviorAA,
3815 const CallBase &CB, unsigned OtherArgNo) {
3816 // We do not need to worry about aliasing with the underlying IRP.
3817 if (this->getCallSiteArgNo() == (int)OtherArgNo)
3818 return false;
3819
3820 // If it is not a pointer or pointer vector we do not alias.
3821 const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3822 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3823 return false;
3824
3825 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3826 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClass: DepClassTy::NONE);
3827
3828 // If the argument is readnone, there is no read-write aliasing.
3829 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3830 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3831 return false;
3832 }
3833
3834 // If the argument is readonly and the underlying value is readonly, there
3835 // is no read-write aliasing.
3836 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3837 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3838 IsReadOnly) {
3839 A.recordDependence(FromAA: MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3840 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3841 return false;
3842 }
3843
3844 // We have to utilize actual alias analysis queries so we need the object.
3845 if (!AAR)
3846 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3847 F: *getAnchorScope());
3848
3849 // Try to rule it out at the call site.
3850 bool IsAliasing = !AAR || !AAR->isNoAlias(V1: &getAssociatedValue(), V2: ArgOp);
3851 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3852 "callsite arguments: "
3853 << getAssociatedValue() << " " << *ArgOp << " => "
3854 << (IsAliasing ? "" : "no-") << "alias \n");
3855
3856 return IsAliasing;
3857 }
3858
3859 bool isKnownNoAliasDueToNoAliasPreservation(
3860 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3861 // We can deduce "noalias" if the following conditions hold.
3862 // (i) Associated value is assumed to be noalias in the definition.
3863 // (ii) Associated value is assumed to be no-capture in all the uses
3864 // possibly executed before this callsite.
3865 // (iii) There is no other pointer argument which could alias with the
3866 // value.
3867
3868 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3869 const Function *ScopeFn = VIRP.getAnchorScope();
3870 // Check whether the value is captured in the scope using AANoCapture.
3871 // Look at CFG and check only uses possibly executed before this
3872 // callsite.
3873 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3874 Instruction *UserI = cast<Instruction>(Val: U.getUser());
3875
3876 // If UserI is the curr instruction and there is a single potential use of
3877 // the value in UserI we allow the use.
3878 // TODO: We should inspect the operands and allow those that cannot alias
3879 // with the value.
3880 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3881 return true;
3882
3883 if (ScopeFn) {
3884 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3885 if (CB->isArgOperand(U: &U)) {
3886
3887 unsigned ArgNo = CB->getArgOperandNo(U: &U);
3888
3889 bool IsKnownNoCapture;
3890 if (AA::hasAssumedIRAttr<Attribute::Captures>(
3891 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
3892 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
3893 return true;
3894 }
3895 }
3896
3897 if (!AA::isPotentiallyReachable(
3898 A, FromI: *UserI, ToI: *getCtxI(), QueryingAA: *this, /* ExclusionSet */ nullptr,
3899 GoBackwardsCB: [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3900 return true;
3901 }
3902
3903 // TODO: We should track the capturing uses in AANoCapture but the problem
3904 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3905 // a value in the module slice.
3906 // TODO(captures): Make this more precise.
3907 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
3908 if (capturesNothing(CC: CI))
3909 return true;
3910 if (CI.isPassthrough()) {
3911 Follow = true;
3912 return true;
3913 }
3914 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
3915 return false;
3916 };
3917
3918 bool IsKnownNoCapture;
3919 const AANoCapture *NoCaptureAA = nullptr;
3920 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
3921 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false, AAPtr: &NoCaptureAA);
3922 if (!IsAssumedNoCapture &&
3923 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3924 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
3925 LLVM_DEBUG(
3926 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3927 << " cannot be noalias as it is potentially captured\n");
3928 return false;
3929 }
3930 }
3931 if (NoCaptureAA)
3932 A.recordDependence(FromAA: *NoCaptureAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3933
3934 // Check there is no other pointer argument which could alias with the
3935 // value passed at this call site.
3936 // TODO: AbstractCallSite
3937 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
3938 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3939 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
3940 return false;
3941
3942 return true;
3943 }
3944
3945 ChangeStatus updateImpl(Attributor &A) override {
3946 // If the argument is readnone we are done as there are no accesses via the
3947 // argument.
3948 auto *MemBehaviorAA =
3949 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
3950 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
3951 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3952 return ChangeStatus::UNCHANGED;
3953 }
3954
3955 bool IsKnownNoAlias;
3956 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3957 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
3958 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
3959 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
3960 << " is not no-alias at the definition\n");
3961 return indicatePessimisticFixpoint();
3962 }
3963
3964 AAResults *AAR = nullptr;
3965 if (MemBehaviorAA &&
3966 isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
3967 LLVM_DEBUG(
3968 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
3969 return ChangeStatus::UNCHANGED;
3970 }
3971
3972 return indicatePessimisticFixpoint();
3973 }
3974
3975 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
3976};
3977
3978/// NoAlias attribute for function return value.
3979struct AANoAliasReturned final : AANoAliasImpl {
3980 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
3981 : AANoAliasImpl(IRP, A) {}
3982
3983 ChangeStatus updateImpl(Attributor &A) override {
3984
3985 auto CheckReturnValue = [&](Value &RV) -> bool {
3986 if (Constant *C = dyn_cast<Constant>(Val: &RV))
3987 if (C->isNullValue() || isa<UndefValue>(Val: C))
3988 return true;
3989
3990 /// For now, we can only deduce noalias if we have call sites.
3991 /// FIXME: add more support.
3992 if (!isa<CallBase>(Val: &RV))
3993 return false;
3994
3995 const IRPosition &RVPos = IRPosition::value(V: RV);
3996 bool IsKnownNoAlias;
3997 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
3998 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias))
3999 return false;
4000
4001 bool IsKnownNoCapture;
4002 const AANoCapture *NoCaptureAA = nullptr;
4003 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4004 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
4005 AAPtr: &NoCaptureAA);
4006 return IsAssumedNoCapture ||
4007 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4008 };
4009
4010 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA: *this))
4011 return indicatePessimisticFixpoint();
4012
4013 return ChangeStatus::UNCHANGED;
4014 }
4015
4016 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4017};
4018
4019/// NoAlias attribute deduction for a call site return value.
4020struct AANoAliasCallSiteReturned final
4021 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4022 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4023 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4024
4025 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4026};
4027} // namespace
4028
4029/// -------------------AAIsDead Function Attribute-----------------------
4030
4031namespace {
4032struct AAIsDeadValueImpl : public AAIsDead {
4033 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4034
4035 bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4036
4037 bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4038
4039 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4040
4041 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4042
4043 bool isAssumedDead(const Instruction *I) const override {
4044 return I == getCtxI() && isAssumedDead();
4045 }
4046
4047 bool isKnownDead(const Instruction *I) const override {
4048 return isAssumedDead(I) && isKnownDead();
4049 }
4050
4051 const std::string getAsStr(Attributor *A) const override {
4052 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4053 }
4054
4055 /// Check if all uses are assumed dead.
4056 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4057 // Callers might not check the type, void has no uses.
4058 if (V.getType()->isVoidTy() || V.use_empty())
4059 return true;
4060
4061 // If we replace a value with a constant there are no uses left afterwards.
4062 if (!isa<Constant>(Val: V)) {
4063 if (auto *I = dyn_cast<Instruction>(Val: &V))
4064 if (!A.isRunOn(Fn&: *I->getFunction()))
4065 return false;
4066 bool UsedAssumedInformation = false;
4067 std::optional<Constant *> C =
4068 A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4069 if (!C || *C)
4070 return true;
4071 }
4072
4073 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4074 // Explicitly set the dependence class to required because we want a long
4075 // chain of N dependent instructions to be considered live as soon as one is
4076 // without going through N update cycles. This is not required for
4077 // correctness.
4078 return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4079 LivenessDepClass: DepClassTy::REQUIRED,
4080 /* IgnoreDroppableUses */ false);
4081 }
4082
4083 /// Determine if \p I is assumed to be side-effect free.
4084 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4085 if (!I || wouldInstructionBeTriviallyDead(I))
4086 return true;
4087
4088 if (!I->isTerminator() && !I->mayHaveSideEffects())
4089 return true;
4090
4091 auto *CB = dyn_cast<CallBase>(Val: I);
4092 if (!CB || isa<IntrinsicInst>(Val: CB))
4093 return false;
4094
4095 const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4096
4097 bool IsKnownNoUnwind;
4098 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4099 A, QueryingAA: this, IRP: CallIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind))
4100 return false;
4101
4102 bool IsKnown;
4103 return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4104 }
4105};
4106
4107struct AAIsDeadFloating : public AAIsDeadValueImpl {
4108 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4109 : AAIsDeadValueImpl(IRP, A) {}
4110
4111 void initialize(Attributor &A) override {
4112 AAIsDeadValueImpl::initialize(A);
4113
4114 if (isa<UndefValue>(Val: getAssociatedValue())) {
4115 indicatePessimisticFixpoint();
4116 return;
4117 }
4118
4119 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4120 if (!isAssumedSideEffectFree(A, I)) {
4121 if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4122 indicatePessimisticFixpoint();
4123 else
4124 removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4125 }
4126 }
4127
4128 bool isDeadFence(Attributor &A, FenceInst &FI) {
4129 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4130 IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4131 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4132 return false;
4133 A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4134 return true;
4135 }
4136
4137 bool isDeadStore(Attributor &A, StoreInst &SI,
4138 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4139 // Lang ref now states volatile store is not UB/dead, let's skip them.
4140 if (SI.isVolatile())
4141 return false;
4142
4143 // If we are collecting assumes to be deleted we are in the manifest stage.
4144 // It's problematic to collect the potential copies again now so we use the
4145 // cached ones.
4146 bool UsedAssumedInformation = false;
4147 if (!AssumeOnlyInst) {
4148 PotentialCopies.clear();
4149 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4150 UsedAssumedInformation)) {
4151 LLVM_DEBUG(
4152 dbgs()
4153 << "[AAIsDead] Could not determine potential copies of store!\n");
4154 return false;
4155 }
4156 }
4157 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4158 << " potential copies.\n");
4159
4160 InformationCache &InfoCache = A.getInfoCache();
4161 return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4162 if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4163 UsedAssumedInformation))
4164 return true;
4165 if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4166 if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4167 auto &UserI = cast<Instruction>(Val&: *U.getUser());
4168 if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4169 if (AssumeOnlyInst)
4170 AssumeOnlyInst->insert(X: &UserI);
4171 return true;
4172 }
4173 return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4174 })) {
4175 return true;
4176 }
4177 }
4178 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4179 << " is assumed live!\n");
4180 return false;
4181 });
4182 }
4183
4184 const std::string getAsStr(Attributor *A) const override {
4185 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4186 if (isa_and_nonnull<StoreInst>(Val: I))
4187 if (isValidState())
4188 return "assumed-dead-store";
4189 if (isa_and_nonnull<FenceInst>(Val: I))
4190 if (isValidState())
4191 return "assumed-dead-fence";
4192 return AAIsDeadValueImpl::getAsStr(A);
4193 }
4194
4195 ChangeStatus updateImpl(Attributor &A) override {
4196 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4197 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4198 if (!isDeadStore(A, SI&: *SI))
4199 return indicatePessimisticFixpoint();
4200 } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4201 if (!isDeadFence(A, FI&: *FI))
4202 return indicatePessimisticFixpoint();
4203 } else {
4204 if (!isAssumedSideEffectFree(A, I))
4205 return indicatePessimisticFixpoint();
4206 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4207 return indicatePessimisticFixpoint();
4208 }
4209 return ChangeStatus::UNCHANGED;
4210 }
4211
4212 bool isRemovableStore() const override {
4213 return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4214 }
4215
4216 ChangeStatus manifest(Attributor &A) override {
4217 Value &V = getAssociatedValue();
4218 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4219 // If we get here we basically know the users are all dead. We check if
4220 // isAssumedSideEffectFree returns true here again because it might not be
4221 // the case and only the users are dead but the instruction (=call) is
4222 // still needed.
4223 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4224 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4225 bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4226 (void)IsDead;
4227 assert(IsDead && "Store was assumed to be dead!");
4228 A.deleteAfterManifest(I&: *I);
4229 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4230 Instruction *AOI = AssumeOnlyInst[i];
4231 for (auto *Usr : AOI->users())
4232 AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4233 A.deleteAfterManifest(I&: *AOI);
4234 }
4235 return ChangeStatus::CHANGED;
4236 }
4237 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4238 assert(isDeadFence(A, *FI));
4239 A.deleteAfterManifest(I&: *FI);
4240 return ChangeStatus::CHANGED;
4241 }
4242 if (isAssumedSideEffectFree(A, I) && !I->isTerminator()) {
4243 A.deleteAfterManifest(I&: *I);
4244 return ChangeStatus::CHANGED;
4245 }
4246 }
4247 return ChangeStatus::UNCHANGED;
4248 }
4249
4250 void trackStatistics() const override {
4251 STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4252 }
4253
4254private:
4255 // The potential copies of a dead store, used for deletion during manifest.
4256 SmallSetVector<Value *, 4> PotentialCopies;
4257};
4258
4259struct AAIsDeadArgument : public AAIsDeadFloating {
4260 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4261 : AAIsDeadFloating(IRP, A) {}
4262
4263 ChangeStatus manifest(Attributor &A) override {
4264 Argument &Arg = *getAssociatedArgument();
4265 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4266 if (A.registerFunctionSignatureRewrite(
4267 Arg, /* ReplacementTypes */ {},
4268 CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4269 ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4270 return ChangeStatus::CHANGED;
4271 }
4272 return ChangeStatus::UNCHANGED;
4273 }
4274
4275 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4276};
4277
4278struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4279 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4280 : AAIsDeadValueImpl(IRP, A) {}
4281
4282 void initialize(Attributor &A) override {
4283 AAIsDeadValueImpl::initialize(A);
4284 if (isa<UndefValue>(Val: getAssociatedValue()))
4285 indicatePessimisticFixpoint();
4286 }
4287
4288 ChangeStatus updateImpl(Attributor &A) override {
4289 // TODO: Once we have call site specific value information we can provide
4290 // call site specific liveness information and then it makes
4291 // sense to specialize attributes for call sites arguments instead of
4292 // redirecting requests to the callee argument.
4293 Argument *Arg = getAssociatedArgument();
4294 if (!Arg)
4295 return indicatePessimisticFixpoint();
4296 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4297 auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4298 if (!ArgAA)
4299 return indicatePessimisticFixpoint();
4300 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4301 }
4302
4303 ChangeStatus manifest(Attributor &A) override {
4304 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4305 Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4306 assert(!isa<UndefValue>(U.get()) &&
4307 "Expected undef values to be filtered out!");
4308 UndefValue &UV = *UndefValue::get(T: U->getType());
4309 if (A.changeUseAfterManifest(U, NV&: UV))
4310 return ChangeStatus::CHANGED;
4311 return ChangeStatus::UNCHANGED;
4312 }
4313
4314 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4315};
4316
4317struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4318 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4319 : AAIsDeadFloating(IRP, A) {}
4320
4321 bool isAssumedDead() const override {
4322 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4323 }
4324
4325 void initialize(Attributor &A) override {
4326 AAIsDeadFloating::initialize(A);
4327 if (isa<UndefValue>(Val: getAssociatedValue())) {
4328 indicatePessimisticFixpoint();
4329 return;
4330 }
4331
4332 // We track this separately as a secondary state.
4333 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4334 }
4335
4336 ChangeStatus updateImpl(Attributor &A) override {
4337 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4338 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4339 IsAssumedSideEffectFree = false;
4340 Changed = ChangeStatus::CHANGED;
4341 }
4342 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4343 return indicatePessimisticFixpoint();
4344 return Changed;
4345 }
4346
4347 void trackStatistics() const override {
4348 if (IsAssumedSideEffectFree)
4349 STATS_DECLTRACK_CSRET_ATTR(IsDead)
4350 else
4351 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4352 }
4353
4354 const std::string getAsStr(Attributor *A) const override {
4355 return isAssumedDead()
4356 ? "assumed-dead"
4357 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4358 }
4359
4360private:
4361 bool IsAssumedSideEffectFree = true;
4362};
4363
4364struct AAIsDeadReturned : public AAIsDeadValueImpl {
4365 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4366 : AAIsDeadValueImpl(IRP, A) {}
4367
4368 ChangeStatus updateImpl(Attributor &A) override {
4369
4370 bool UsedAssumedInformation = false;
4371 A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4372 Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4373
4374 auto PredForCallSite = [&](AbstractCallSite ACS) {
4375 if (ACS.isCallbackCall() || !ACS.getInstruction())
4376 return false;
4377 return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4378 };
4379
4380 if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4381 UsedAssumedInformation))
4382 return indicatePessimisticFixpoint();
4383
4384 return ChangeStatus::UNCHANGED;
4385 }
4386
4387 ChangeStatus manifest(Attributor &A) override {
4388 // TODO: Rewrite the signature to return void?
4389 bool AnyChange = false;
4390 UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4391 auto RetInstPred = [&](Instruction &I) {
4392 ReturnInst &RI = cast<ReturnInst>(Val&: I);
4393 if (!isa<UndefValue>(Val: RI.getReturnValue()))
4394 AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4395 return true;
4396 };
4397 bool UsedAssumedInformation = false;
4398 A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4399 UsedAssumedInformation);
4400 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4401 }
4402
4403 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4404};
4405
4406struct AAIsDeadFunction : public AAIsDead {
4407 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4408
4409 void initialize(Attributor &A) override {
4410 Function *F = getAnchorScope();
4411 assert(F && "Did expect an anchor function");
4412 if (!isAssumedDeadInternalFunction(A)) {
4413 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4414 assumeLive(A, BB: F->getEntryBlock());
4415 }
4416 }
4417
4418 bool isAssumedDeadInternalFunction(Attributor &A) {
4419 if (!getAnchorScope()->hasLocalLinkage())
4420 return false;
4421 // An AA that will not be updated is fixed pessimistically right after
4422 // initialization, so an assumed-dead result would be discarded. Checking
4423 // the call sites recursively initializes the liveness of every internal
4424 // caller, which in CGSCC cleanup repeats for each callee SCC.
4425 if (!A.shouldUpdateAA<AAIsDead>(IRP: getIRPosition()))
4426 return false;
4427 bool UsedAssumedInformation = false;
4428 return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4429 RequireAllCallSites: true, UsedAssumedInformation);
4430 }
4431
4432 const std::string getAsStr(Attributor *A) const override {
4433 return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4434 std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4435 std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4436 std::to_string(val: KnownDeadEnds.size()) + "]";
4437 }
4438
4439 ChangeStatus manifest(Attributor &A) override {
4440 assert(getState().isValidState() &&
4441 "Attempted to manifest an invalid state!");
4442
4443 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4444 Function &F = *getAnchorScope();
4445
4446 if (AssumedLiveBlocks.empty()) {
4447 A.deleteAfterManifest(F);
4448 return ChangeStatus::CHANGED;
4449 }
4450
4451 // Flag to determine if we can change an invoke to a call assuming the
4452 // callee is nounwind. This is not possible if the personality of the
4453 // function allows to catch asynchronous exceptions.
4454 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4455
4456 KnownDeadEnds.set_union(ToBeExploredFrom);
4457 for (const Instruction *DeadEndI : KnownDeadEnds) {
4458 auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4459 if (!CB)
4460 continue;
4461 bool IsKnownNoReturn;
4462 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4463 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL,
4464 IsKnown&: IsKnownNoReturn);
4465 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4466 continue;
4467
4468 if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4469 A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4470 else
4471 A.changeToUnreachableAfterManifest(
4472 I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4473 HasChanged = ChangeStatus::CHANGED;
4474 }
4475
4476 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4477 for (BasicBlock &BB : F)
4478 if (!AssumedLiveBlocks.count(V: &BB)) {
4479 A.deleteAfterManifest(BB);
4480 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4481 HasChanged = ChangeStatus::CHANGED;
4482 }
4483
4484 return HasChanged;
4485 }
4486
4487 ChangeStatus updateImpl(Attributor &A) override;
4488
4489 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4490 assert(From->getParent() == getAnchorScope() &&
4491 To->getParent() == getAnchorScope() &&
4492 "Used AAIsDead of the wrong function");
4493 return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4494 }
4495
4496 void trackStatistics() const override {}
4497
4498 /// Returns true if the function is assumed dead.
4499 bool isAssumedDead() const override { return false; }
4500
4501 bool isKnownDead() const override { return false; }
4502
4503 bool isAssumedDead(const BasicBlock *BB) const override {
4504 assert(BB->getParent() == getAnchorScope() &&
4505 "BB must be in the same anchor scope function.");
4506
4507 if (!getAssumed())
4508 return false;
4509 return !AssumedLiveBlocks.count(V: BB);
4510 }
4511
4512 bool isKnownDead(const BasicBlock *BB) const override {
4513 return getKnown() && isAssumedDead(BB);
4514 }
4515
4516 bool isAssumedDead(const Instruction *I) const override {
4517 assert(I->getParent()->getParent() == getAnchorScope() &&
4518 "Instruction must be in the same anchor scope function.");
4519
4520 if (!getAssumed())
4521 return false;
4522
4523 // If it is not in AssumedLiveBlocks then it for sure dead.
4524 // Otherwise, it can still be after noreturn call in a live block.
4525 if (!AssumedLiveBlocks.count(V: I->getParent()))
4526 return true;
4527
4528 // If it is not after a liveness barrier it is live.
4529 const Instruction *PrevI = I->getPrevNode();
4530 while (PrevI) {
4531 if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4532 return true;
4533 PrevI = PrevI->getPrevNode();
4534 }
4535 return false;
4536 }
4537
4538 bool isKnownDead(const Instruction *I) const override {
4539 return getKnown() && isAssumedDead(I);
4540 }
4541
4542 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4543 /// that internal function called from \p BB should now be looked at.
4544 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4545 if (!AssumedLiveBlocks.insert(V: &BB).second)
4546 return false;
4547
4548 if (!A.isDuringDeduction())
4549 return true;
4550
4551 // We assume that all of BB is (probably) live now and if there are calls to
4552 // internal functions we will assume that those are now live as well. This
4553 // is a performance optimization for blocks with calls to a lot of internal
4554 // functions. It can however cause dead functions to be treated as live.
4555 for (const Instruction &I : BB)
4556 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4557 if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4558 if (F->hasLocalLinkage()) {
4559 LLVM_DEBUG({
4560 dbgs() << "[AAIsDead] Seeding live internal callee ";
4561 F->printAsOperand(dbgs(), /*PrintType=*/false);
4562 dbgs() << " from ";
4563 BB.getParent()->printAsOperand(dbgs(), /*PrintType=*/false);
4564 dbgs() << "\n";
4565 });
4566 A.markLiveInternalFunction(F: *F);
4567 }
4568 return true;
4569 }
4570
4571 /// Collection of instructions that need to be explored again, e.g., we
4572 /// did assume they do not transfer control to (one of their) successors.
4573 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4574
4575 /// Collection of instructions that are known to not transfer control.
4576 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4577
4578 /// Collection of all assumed live edges
4579 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4580
4581 /// Collection of all assumed live BasicBlocks.
4582 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4583};
4584
4585static bool
4586identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4587 AbstractAttribute &AA,
4588 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4589 const IRPosition &IPos = IRPosition::callsite_function(CB);
4590
4591 bool IsKnownNoReturn;
4592 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4593 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoReturn))
4594 return !IsKnownNoReturn;
4595 if (CB.isTerminator())
4596 AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4597 else
4598 AliveSuccessors.push_back(Elt: CB.getNextNode());
4599 return false;
4600}
4601
4602static bool
4603identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4604 AbstractAttribute &AA,
4605 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4606 bool UsedAssumedInformation =
4607 identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4608
4609 // First, determine if we can change an invoke to a call assuming the
4610 // callee is nounwind. This is not possible if the personality of the
4611 // function allows to catch asynchronous exceptions.
4612 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4613 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4614 } else {
4615 const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4616
4617 bool IsKnownNoUnwind;
4618 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4619 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
4620 UsedAssumedInformation |= !IsKnownNoUnwind;
4621 } else {
4622 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4623 }
4624 }
4625 return UsedAssumedInformation;
4626}
4627
4628static bool
4629identifyAliveSuccessors(Attributor &, const UncondBrInst &BI,
4630 AbstractAttribute &,
4631 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4632 AliveSuccessors.push_back(Elt: &BI.getSuccessor()->front());
4633 return false;
4634}
4635
4636static bool
4637identifyAliveSuccessors(Attributor &A, const CondBrInst &BI,
4638 AbstractAttribute &AA,
4639 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4640 bool UsedAssumedInformation = false;
4641 std::optional<Constant *> C =
4642 A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4643 if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4644 // No value yet, assume both edges are dead.
4645 } else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4646 const BasicBlock *SuccBB =
4647 BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4648 AliveSuccessors.push_back(Elt: &SuccBB->front());
4649 } else {
4650 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4651 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4652 UsedAssumedInformation = false;
4653 }
4654 return UsedAssumedInformation;
4655}
4656
4657static bool
4658identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4659 AbstractAttribute &AA,
4660 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4661 bool UsedAssumedInformation = false;
4662 SmallVector<AA::ValueAndContext> Values;
4663 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4664 Values, S: AA::AnyScope,
4665 UsedAssumedInformation)) {
4666 // Something went wrong, assume all successors are live.
4667 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4668 AliveSuccessors.push_back(Elt: &SuccBB->front());
4669 return false;
4670 }
4671
4672 if (Values.empty() ||
4673 (Values.size() == 1 &&
4674 isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4675 // No valid value yet, assume all edges are dead.
4676 return UsedAssumedInformation;
4677 }
4678
4679 Type &Ty = *SI.getCondition()->getType();
4680 SmallPtrSet<ConstantInt *, 8> Constants;
4681 auto CheckForConstantInt = [&](Value *V) {
4682 if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4683 Constants.insert(Ptr: CI);
4684 return true;
4685 }
4686 return false;
4687 };
4688
4689 if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4690 return CheckForConstantInt(VAC.getValue());
4691 })) {
4692 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4693 AliveSuccessors.push_back(Elt: &SuccBB->front());
4694 return UsedAssumedInformation;
4695 }
4696
4697 unsigned MatchedCases = 0;
4698 for (const auto &CaseIt : SI.cases()) {
4699 if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4700 ++MatchedCases;
4701 AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4702 }
4703 }
4704
4705 // If all potential values have been matched, we will not visit the default
4706 // case.
4707 if (MatchedCases < Constants.size())
4708 AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4709 return UsedAssumedInformation;
4710}
4711
4712ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4713 ChangeStatus Change = ChangeStatus::UNCHANGED;
4714
4715 if (AssumedLiveBlocks.empty()) {
4716 if (isAssumedDeadInternalFunction(A))
4717 return ChangeStatus::UNCHANGED;
4718
4719 Function *F = getAnchorScope();
4720 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4721 assumeLive(A, BB: F->getEntryBlock());
4722 Change = ChangeStatus::CHANGED;
4723 }
4724
4725 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4726 << getAnchorScope()->size() << "] BBs and "
4727 << ToBeExploredFrom.size() << " exploration points and "
4728 << KnownDeadEnds.size() << " known dead ends\n");
4729
4730 // Copy and clear the list of instructions we need to explore from. It is
4731 // refilled with instructions the next update has to look at.
4732 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4733 ToBeExploredFrom.end());
4734 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4735
4736 SmallVector<const Instruction *, 8> AliveSuccessors;
4737 while (!Worklist.empty()) {
4738 const Instruction *I = Worklist.pop_back_val();
4739 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4740
4741 // Fast forward for uninteresting instructions. We could look for UB here
4742 // though.
4743 while (!I->isTerminator() && !isa<CallBase>(Val: I))
4744 I = I->getNextNode();
4745
4746 AliveSuccessors.clear();
4747
4748 bool UsedAssumedInformation = false;
4749 switch (I->getOpcode()) {
4750 // TODO: look for (assumed) UB to backwards propagate "deadness".
4751 default:
4752 assert(I->isTerminator() &&
4753 "Expected non-terminators to be handled already!");
4754 for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4755 AliveSuccessors.push_back(Elt: &SuccBB->front());
4756 break;
4757 case Instruction::Call:
4758 UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4759 AA&: *this, AliveSuccessors);
4760 break;
4761 case Instruction::Invoke:
4762 UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4763 AA&: *this, AliveSuccessors);
4764 break;
4765 case Instruction::UncondBr:
4766 UsedAssumedInformation = identifyAliveSuccessors(
4767 A, BI: cast<UncondBrInst>(Val: *I), *this, AliveSuccessors);
4768 break;
4769 case Instruction::CondBr:
4770 UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<CondBrInst>(Val: *I),
4771 AA&: *this, AliveSuccessors);
4772 break;
4773 case Instruction::Switch:
4774 UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4775 AA&: *this, AliveSuccessors);
4776 break;
4777 }
4778
4779 if (UsedAssumedInformation) {
4780 NewToBeExploredFrom.insert(X: I);
4781 } else if (AliveSuccessors.empty() ||
4782 (I->isTerminator() &&
4783 AliveSuccessors.size() < I->getNumSuccessors())) {
4784 if (KnownDeadEnds.insert(X: I))
4785 Change = ChangeStatus::CHANGED;
4786 }
4787
4788 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4789 << AliveSuccessors.size() << " UsedAssumedInformation: "
4790 << UsedAssumedInformation << "\n");
4791
4792 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4793 if (!I->isTerminator()) {
4794 assert(AliveSuccessors.size() == 1 &&
4795 "Non-terminator expected to have a single successor!");
4796 Worklist.push_back(Elt: AliveSuccessor);
4797 } else {
4798 // record the assumed live edge
4799 auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4800 if (AssumedLiveEdges.insert(V: Edge).second)
4801 Change = ChangeStatus::CHANGED;
4802 if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4803 Worklist.push_back(Elt: AliveSuccessor);
4804 }
4805 }
4806 }
4807
4808 // Check if the content of ToBeExploredFrom changed, ignore the order.
4809 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4810 llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4811 return !ToBeExploredFrom.count(key: I);
4812 })) {
4813 Change = ChangeStatus::CHANGED;
4814 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4815 }
4816
4817 // If we know everything is live there is no need to query for liveness.
4818 // Instead, indicating a pessimistic fixpoint will cause the state to be
4819 // "invalid" and all queries to be answered conservatively without lookups.
4820 // To be in this state we have to (1) finished the exploration and (3) not
4821 // discovered any non-trivial dead end and (2) not ruled unreachable code
4822 // dead.
4823 if (ToBeExploredFrom.empty() &&
4824 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4825 llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4826 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4827 }))
4828 return indicatePessimisticFixpoint();
4829 return Change;
4830}
4831
4832/// Liveness information for a call sites.
4833struct AAIsDeadCallSite final : AAIsDeadFunction {
4834 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4835 : AAIsDeadFunction(IRP, A) {}
4836
4837 void initialize(Attributor &A) override {
4838 // TODO: Once we have call site specific value information we can provide
4839 // call site specific liveness information and then it makes
4840 // sense to specialize attributes for call sites instead of
4841 // redirecting requests to the callee.
4842 llvm_unreachable("Abstract attributes for liveness are not "
4843 "supported for call sites yet!");
4844 }
4845
4846 ChangeStatus updateImpl(Attributor &A) override {
4847 return indicatePessimisticFixpoint();
4848 }
4849
4850 void trackStatistics() const override {}
4851};
4852} // namespace
4853
4854/// -------------------- Dereferenceable Argument Attribute --------------------
4855
4856namespace {
4857struct AADereferenceableImpl : AADereferenceable {
4858 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4859 : AADereferenceable(IRP, A) {}
4860 using StateType = DerefState;
4861
4862 void initialize(Attributor &A) override {
4863 Value &V = *getAssociatedValue().stripPointerCasts();
4864 SmallVector<Attribute, 4> Attrs;
4865 A.getAttrs(IRP: getIRPosition(),
4866 AKs: {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4867 Attrs, /* IgnoreSubsumingPositions */ false);
4868 for (const Attribute &Attr : Attrs)
4869 takeKnownDerefBytesMaximum(Bytes: Attr.getValueAsInt());
4870
4871 // Ensure we initialize the non-null AA (if necessary).
4872 bool IsKnownNonNull;
4873 AA::hasAssumedIRAttr<Attribute::NonNull>(
4874 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
4875
4876 bool CanBeNull;
4877 takeKnownDerefBytesMaximum(Bytes: V.getPointerDereferenceableBytes(
4878 DL: A.getDataLayout(), CanBeNull, /*CanBeFreed=*/nullptr));
4879
4880 if (Instruction *CtxI = getCtxI())
4881 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
4882 }
4883
4884 StateType &getState() override { return *this; }
4885 const StateType &getState() const override { return *this; }
4886
4887 /// Helper function for collecting accessed bytes in must-be-executed-context
4888 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4889 DerefState &State) {
4890 const Value *UseV = U->get();
4891 if (!UseV->getType()->isPointerTy())
4892 return;
4893
4894 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
4895 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4896 return;
4897
4898 int64_t Offset;
4899 const Value *Base = GetPointerBaseWithConstantOffset(
4900 Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
4901 if (Base && Base == &getAssociatedValue())
4902 State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
4903 }
4904
4905 /// See followUsesInMBEC
4906 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4907 AADereferenceable::StateType &State) {
4908 bool IsNonNull = false;
4909 bool TrackUse = false;
4910 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4911 A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I, IsNonNull, TrackUse);
4912 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4913 << " for instruction " << *I << "\n");
4914
4915 addAccessedBytesForUse(A, U, I, State);
4916 State.takeKnownDerefBytesMaximum(Bytes: DerefBytes);
4917 return TrackUse;
4918 }
4919
4920 ChangeStatus manifest(Attributor &A) override {
4921 ChangeStatus Change = AADereferenceable::manifest(A);
4922 bool IsKnownNonNull;
4923 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4924 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
4925 if (IsAssumedNonNull &&
4926 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::DereferenceableOrNull)) {
4927 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::DereferenceableOrNull});
4928 return ChangeStatus::CHANGED;
4929 }
4930 return Change;
4931 }
4932
4933 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
4934 SmallVectorImpl<Attribute> &Attrs) const override {
4935 // TODO: Add *_globally support
4936 bool IsKnownNonNull;
4937 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4938 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
4939 if (IsAssumedNonNull)
4940 Attrs.emplace_back(Args: Attribute::getWithDereferenceableBytes(
4941 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
4942 else
4943 Attrs.emplace_back(Args: Attribute::getWithDereferenceableOrNullBytes(
4944 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
4945 }
4946
4947 const std::string getAsStr(Attributor *A) const override {
4948 if (!getAssumedDereferenceableBytes())
4949 return "unknown-dereferenceable";
4950 bool IsKnownNonNull;
4951 bool IsAssumedNonNull = false;
4952 if (A)
4953 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
4954 A&: *A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
4955 return std::string("dereferenceable") +
4956 (IsAssumedNonNull ? "" : "_or_null") +
4957 (isAssumedGlobal() ? "_globally" : "") + "<" +
4958 std::to_string(val: getKnownDereferenceableBytes()) + "-" +
4959 std::to_string(val: getAssumedDereferenceableBytes()) + ">" +
4960 (!A ? " [non-null is unknown]" : "");
4961 }
4962};
4963
4964/// Dereferenceable attribute for a floating value.
4965struct AADereferenceableFloating : AADereferenceableImpl {
4966 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
4967 : AADereferenceableImpl(IRP, A) {}
4968
4969 ChangeStatus updateImpl(Attributor &A) override {
4970 bool Stripped;
4971 bool UsedAssumedInformation = false;
4972 SmallVector<AA::ValueAndContext> Values;
4973 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
4974 S: AA::AnyScope, UsedAssumedInformation)) {
4975 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
4976 Stripped = false;
4977 } else {
4978 Stripped = Values.size() != 1 ||
4979 Values.front().getValue() != &getAssociatedValue();
4980 }
4981
4982 const DataLayout &DL = A.getDataLayout();
4983 DerefState T;
4984
4985 auto VisitValueCB = [&](const Value &V) -> bool {
4986 unsigned IdxWidth =
4987 DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
4988 APInt Offset(IdxWidth, 0);
4989 const Value *Base = stripAndAccumulateOffsets(
4990 A, QueryingAA: *this, Val: &V, DL, Offset, /* GetMinOffset */ false,
4991 /* AllowNonInbounds */ true);
4992
4993 const auto *AA = A.getAAFor<AADereferenceable>(
4994 QueryingAA: *this, IRP: IRPosition::value(V: *Base), DepClass: DepClassTy::REQUIRED);
4995 int64_t DerefBytes = 0;
4996 if (!AA || (!Stripped && this == AA)) {
4997 // Use IR information if we did not strip anything.
4998 // TODO: track globally.
4999 bool CanBeNull;
5000 DerefBytes = Base->getPointerDereferenceableBytes(
5001 DL, CanBeNull, /*CanBeFreed=*/nullptr);
5002 T.GlobalState.indicatePessimisticFixpoint();
5003 } else {
5004 const DerefState &DS = AA->getState();
5005 DerefBytes = DS.DerefBytesState.getAssumed();
5006 T.GlobalState &= DS.GlobalState;
5007 }
5008
5009 // For now we do not try to "increase" dereferenceability due to negative
5010 // indices as we first have to come up with code to deal with loops and
5011 // for overflows of the dereferenceable bytes.
5012 int64_t OffsetSExt = Offset.getSExtValue();
5013 if (OffsetSExt < 0)
5014 OffsetSExt = 0;
5015
5016 T.takeAssumedDerefBytesMinimum(
5017 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5018
5019 if (this == AA) {
5020 if (!Stripped) {
5021 // If nothing was stripped IR information is all we got.
5022 T.takeKnownDerefBytesMaximum(
5023 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5024 T.indicatePessimisticFixpoint();
5025 } else if (OffsetSExt > 0) {
5026 // If something was stripped but there is circular reasoning we look
5027 // for the offset. If it is positive we basically decrease the
5028 // dereferenceable bytes in a circular loop now, which will simply
5029 // drive them down to the known value in a very slow way which we
5030 // can accelerate.
5031 T.indicatePessimisticFixpoint();
5032 }
5033 }
5034
5035 return T.isValidState();
5036 };
5037
5038 for (const auto &VAC : Values)
5039 if (!VisitValueCB(*VAC.getValue()))
5040 return indicatePessimisticFixpoint();
5041
5042 return clampStateAndIndicateChange(S&: getState(), R: T);
5043 }
5044
5045 void trackStatistics() const override {
5046 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5047 }
5048};
5049
5050/// Dereferenceable attribute for a return value.
5051struct AADereferenceableReturned final
5052 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5053 using Base =
5054 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5055 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5056 : Base(IRP, A) {}
5057
5058 void trackStatistics() const override {
5059 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5060 }
5061};
5062
5063/// Dereferenceable attribute for an argument
5064struct AADereferenceableArgument final
5065 : AAArgumentFromCallSiteArguments<AADereferenceable,
5066 AADereferenceableImpl> {
5067 using Base =
5068 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5069 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5070 : Base(IRP, A) {}
5071
5072 void trackStatistics() const override {
5073 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5074 }
5075};
5076
5077/// Dereferenceable attribute for a call site argument.
5078struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5079 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5080 : AADereferenceableFloating(IRP, A) {}
5081
5082 void trackStatistics() const override {
5083 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5084 }
5085};
5086
5087/// Dereferenceable attribute deduction for a call site return value.
5088struct AADereferenceableCallSiteReturned final
5089 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5090 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5091 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5092 : Base(IRP, A) {}
5093
5094 void trackStatistics() const override {
5095 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5096 }
5097};
5098} // namespace
5099
5100// ------------------------ Align Argument Attribute ------------------------
5101
5102namespace {
5103
5104static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5105 Value &AssociatedValue, const Use *U,
5106 const Instruction *I, bool &TrackUse) {
5107 // We need to follow common pointer manipulation uses to the accesses they
5108 // feed into.
5109 if (isa<CastInst>(Val: I)) {
5110 // Follow all but ptr2int casts.
5111 TrackUse = !isa<PtrToIntInst>(Val: I);
5112 return 0;
5113 }
5114 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5115 if (GEP->hasAllConstantIndices())
5116 TrackUse = true;
5117 return 0;
5118 }
5119 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I))
5120 switch (II->getIntrinsicID()) {
5121 case Intrinsic::ptrmask: {
5122 // Is it appropriate to pull attribute in initialization?
5123 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5124 QueryingAA, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::NONE);
5125 const auto *AlignAA = A.getAAFor<AAAlign>(
5126 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5127 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5128 unsigned ShiftValue = std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5129 b: Value::MaxAlignmentExponent);
5130 Align ConstAlign = Align::fromLog2(Log2Value: ShiftValue);
5131 if (ConstAlign >= AlignAA->getKnownAlign())
5132 return Align(1).value();
5133 }
5134 if (AlignAA)
5135 return AlignAA->getKnownAlign().value();
5136 break;
5137 }
5138 case Intrinsic::amdgcn_make_buffer_rsrc: {
5139 const auto *AlignAA = A.getAAFor<AAAlign>(
5140 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5141 if (AlignAA)
5142 return AlignAA->getKnownAlign().value();
5143 break;
5144 }
5145 default:
5146 break;
5147 }
5148
5149 MaybeAlign MA;
5150 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5151 if (CB->isBundleOperand(U) || CB->isCallee(U))
5152 return 0;
5153
5154 unsigned ArgNo = CB->getArgOperandNo(U);
5155 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5156 // As long as we only use known information there is no need to track
5157 // dependences here.
5158 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
5159 if (AlignAA)
5160 MA = MaybeAlign(AlignAA->getKnownAlign());
5161 }
5162
5163 const DataLayout &DL = A.getDataLayout();
5164 const Value *UseV = U->get();
5165 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5166 if (SI->getPointerOperand() == UseV)
5167 MA = SI->getAlign();
5168 } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5169 if (LI->getPointerOperand() == UseV)
5170 MA = LI->getAlign();
5171 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5172 if (AI->getPointerOperand() == UseV)
5173 MA = AI->getAlign();
5174 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5175 if (AI->getPointerOperand() == UseV)
5176 MA = AI->getAlign();
5177 }
5178
5179 if (!MA || *MA <= QueryingAA.getKnownAlign())
5180 return 0;
5181
5182 unsigned Alignment = MA->value();
5183 int64_t Offset;
5184
5185 if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5186 if (Base == &AssociatedValue) {
5187 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5188 // So we can say that the maximum power of two which is a divisor of
5189 // gcd(Offset, Alignment) is an alignment.
5190
5191 uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5192 Alignment = llvm::bit_floor(Value: gcd);
5193 }
5194 }
5195
5196 return Alignment;
5197}
5198
5199struct AAAlignImpl : AAAlign {
5200 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5201
5202 void initialize(Attributor &A) override {
5203 SmallVector<Attribute, 4> Attrs;
5204 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::Alignment}, Attrs);
5205 for (const Attribute &Attr : Attrs)
5206 takeKnownMaximum(Value: Attr.getValueAsInt());
5207
5208 Value &V = *getAssociatedValue().stripPointerCasts();
5209 takeKnownMaximum(Value: V.getPointerAlignment(DL: A.getDataLayout()).value());
5210
5211 if (Instruction *CtxI = getCtxI())
5212 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5213 }
5214
5215 ChangeStatus manifest(Attributor &A) override {
5216 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5217
5218 // Check for users that allow alignment annotations.
5219 Value &AssociatedValue = getAssociatedValue();
5220 if (isa<ConstantData>(Val: AssociatedValue))
5221 return ChangeStatus::UNCHANGED;
5222
5223 for (const Use &U : AssociatedValue.uses()) {
5224 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
5225 if (SI->getPointerOperand() == &AssociatedValue)
5226 if (SI->getAlign() < getAssumedAlign()) {
5227 STATS_DECLTRACK(AAAlign, Store,
5228 "Number of times alignment added to a store");
5229 SI->setAlignment(getAssumedAlign());
5230 InstrChanged = ChangeStatus::CHANGED;
5231 }
5232 } else if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
5233 if (LI->getPointerOperand() == &AssociatedValue)
5234 if (LI->getAlign() < getAssumedAlign()) {
5235 LI->setAlignment(getAssumedAlign());
5236 STATS_DECLTRACK(AAAlign, Load,
5237 "Number of times alignment added to a load");
5238 InstrChanged = ChangeStatus::CHANGED;
5239 }
5240 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: U.getUser())) {
5241 if (RMW->getPointerOperand() == &AssociatedValue) {
5242 if (RMW->getAlign() < getAssumedAlign()) {
5243 STATS_DECLTRACK(AAAlign, AtomicRMW,
5244 "Number of times alignment added to atomicrmw");
5245
5246 RMW->setAlignment(getAssumedAlign());
5247 InstrChanged = ChangeStatus::CHANGED;
5248 }
5249 }
5250 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: U.getUser())) {
5251 if (CAS->getPointerOperand() == &AssociatedValue) {
5252 if (CAS->getAlign() < getAssumedAlign()) {
5253 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5254 "Number of times alignment added to cmpxchg");
5255 CAS->setAlignment(getAssumedAlign());
5256 InstrChanged = ChangeStatus::CHANGED;
5257 }
5258 }
5259 }
5260 }
5261
5262 ChangeStatus Changed = AAAlign::manifest(A);
5263
5264 Align InheritAlign =
5265 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5266 if (InheritAlign >= getAssumedAlign())
5267 return InstrChanged;
5268 return Changed | InstrChanged;
5269 }
5270
5271 // TODO: Provide a helper to determine the implied ABI alignment and check in
5272 // the existing manifest method and a new one for AAAlignImpl that value
5273 // to avoid making the alignment explicit if it did not improve.
5274
5275 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5276 SmallVectorImpl<Attribute> &Attrs) const override {
5277 if (getAssumedAlign() > 1)
5278 Attrs.emplace_back(
5279 Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5280 }
5281
5282 /// See followUsesInMBEC
5283 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5284 AAAlign::StateType &State) {
5285 bool TrackUse = false;
5286
5287 unsigned int KnownAlign =
5288 getKnownAlignForUse(A, QueryingAA&: *this, AssociatedValue&: getAssociatedValue(), U, I, TrackUse);
5289 State.takeKnownMaximum(Value: KnownAlign);
5290
5291 return TrackUse;
5292 }
5293
5294 const std::string getAsStr(Attributor *A) const override {
5295 return "align<" + std::to_string(val: getKnownAlign().value()) + "-" +
5296 std::to_string(val: getAssumedAlign().value()) + ">";
5297 }
5298};
5299
5300/// Align attribute for a floating value.
5301struct AAAlignFloating : AAAlignImpl {
5302 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5303
5304 ChangeStatus updateImpl(Attributor &A) override {
5305 const DataLayout &DL = A.getDataLayout();
5306
5307 bool Stripped;
5308 bool UsedAssumedInformation = false;
5309 SmallVector<AA::ValueAndContext> Values;
5310 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5311 S: AA::AnyScope, UsedAssumedInformation)) {
5312 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5313 Stripped = false;
5314 } else {
5315 Stripped = Values.size() != 1 ||
5316 Values.front().getValue() != &getAssociatedValue();
5317 }
5318
5319 StateType T;
5320 auto VisitValueCB = [&](Value &V) -> bool {
5321 if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5322 return true;
5323 const auto *AA = A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V),
5324 DepClass: DepClassTy::REQUIRED);
5325 if (!AA || (!Stripped && this == AA)) {
5326 int64_t Offset;
5327 unsigned Alignment = 1;
5328 if (const Value *Base =
5329 GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5330 // TODO: Use AAAlign for the base too.
5331 Align PA = Base->getPointerAlignment(DL);
5332 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5333 // So we can say that the maximum power of two which is a divisor of
5334 // gcd(Offset, Alignment) is an alignment.
5335
5336 uint32_t gcd =
5337 std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5338 Alignment = llvm::bit_floor(Value: gcd);
5339 } else {
5340 Alignment = V.getPointerAlignment(DL).value();
5341 }
5342 // Use only IR information if we did not strip anything.
5343 T.takeKnownMaximum(Value: Alignment);
5344 T.indicatePessimisticFixpoint();
5345 } else {
5346 // Use abstract attribute information.
5347 const AAAlign::StateType &DS = AA->getState();
5348 T ^= DS;
5349 }
5350 return T.isValidState();
5351 };
5352
5353 for (const auto &VAC : Values) {
5354 if (!VisitValueCB(*VAC.getValue()))
5355 return indicatePessimisticFixpoint();
5356 }
5357
5358 // TODO: If we know we visited all incoming values, thus no are assumed
5359 // dead, we can take the known information from the state T.
5360 return clampStateAndIndicateChange(S&: getState(), R: T);
5361 }
5362
5363 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5364};
5365
5366/// Align attribute for function return value.
5367struct AAAlignReturned final
5368 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5369 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5370 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5371
5372 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5373};
5374
5375/// Align attribute for function argument.
5376struct AAAlignArgument final
5377 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5378 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5379 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5380
5381 ChangeStatus manifest(Attributor &A) override {
5382 // If the associated argument is involved in a must-tail call we give up
5383 // because we would need to keep the argument alignments of caller and
5384 // callee in-sync. Just does not seem worth the trouble right now.
5385 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5386 return ChangeStatus::UNCHANGED;
5387 return Base::manifest(A);
5388 }
5389
5390 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5391};
5392
5393struct AAAlignCallSiteArgument final : AAAlignFloating {
5394 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5395 : AAAlignFloating(IRP, A) {}
5396
5397 ChangeStatus manifest(Attributor &A) override {
5398 // If the associated argument is involved in a must-tail call we give up
5399 // because we would need to keep the argument alignments of caller and
5400 // callee in-sync. Just does not seem worth the trouble right now.
5401 if (Argument *Arg = getAssociatedArgument())
5402 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5403 return ChangeStatus::UNCHANGED;
5404 ChangeStatus Changed = AAAlignImpl::manifest(A);
5405 Align InheritAlign =
5406 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5407 if (InheritAlign >= getAssumedAlign())
5408 Changed = ChangeStatus::UNCHANGED;
5409 return Changed;
5410 }
5411
5412 ChangeStatus updateImpl(Attributor &A) override {
5413 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5414 if (Argument *Arg = getAssociatedArgument()) {
5415 // We only take known information from the argument
5416 // so we do not need to track a dependence.
5417 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5418 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::NONE);
5419 if (ArgAlignAA)
5420 takeKnownMaximum(Value: ArgAlignAA->getKnownAlign().value());
5421 }
5422 return Changed;
5423 }
5424
5425 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5426};
5427
5428/// Align attribute deduction for a call site return value.
5429struct AAAlignCallSiteReturned final
5430 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5431 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5432 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5433 : Base(IRP, A) {}
5434
5435 ChangeStatus updateImpl(Attributor &A) override {
5436 Instruction *I = getIRPosition().getCtxI();
5437 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
5438 switch (II->getIntrinsicID()) {
5439 case Intrinsic::ptrmask: {
5440 Align Alignment;
5441 bool Valid = false;
5442
5443 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5444 QueryingAA: *this, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::REQUIRED);
5445 if (ConstVals && ConstVals->isValidState()) {
5446 unsigned ShiftValue =
5447 std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5448 b: Value::MaxAlignmentExponent);
5449 Alignment = Align::fromLog2(Log2Value: ShiftValue);
5450 Valid = true;
5451 }
5452
5453 const auto *AlignAA =
5454 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5455 DepClass: DepClassTy::REQUIRED);
5456 if (AlignAA) {
5457 Alignment = std::max(a: AlignAA->getAssumedAlign(), b: Alignment);
5458 Valid = true;
5459 }
5460
5461 if (Valid)
5462 return clampStateAndIndicateChange<StateType>(
5463 S&: this->getState(),
5464 R: std::min(a: this->getAssumedAlign(), b: Alignment).value());
5465 break;
5466 }
5467 // FIXME: Should introduce target specific sub-attributes and letting
5468 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5469 // specific intrinsics.
5470 case Intrinsic::amdgcn_make_buffer_rsrc: {
5471 const auto *AlignAA =
5472 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5473 DepClass: DepClassTy::REQUIRED);
5474 if (AlignAA)
5475 return clampStateAndIndicateChange<StateType>(
5476 S&: this->getState(), R: AlignAA->getAssumedAlign().value());
5477 break;
5478 }
5479 default:
5480 break;
5481 }
5482 }
5483 return Base::updateImpl(A);
5484 };
5485 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5486};
5487} // namespace
5488
5489/// ------------------ Function No-Return Attribute ----------------------------
5490namespace {
5491struct AANoReturnImpl : public AANoReturn {
5492 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5493
5494 void initialize(Attributor &A) override {
5495 bool IsKnown;
5496 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5497 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5498 (void)IsKnown;
5499 }
5500
5501 const std::string getAsStr(Attributor *A) const override {
5502 return getAssumed() ? "noreturn" : "may-return";
5503 }
5504
5505 ChangeStatus updateImpl(Attributor &A) override {
5506 auto CheckForNoReturn = [](Instruction &) { return false; };
5507 bool UsedAssumedInformation = false;
5508 if (!A.checkForAllInstructions(Pred: CheckForNoReturn, QueryingAA: *this,
5509 Opcodes: {(unsigned)Instruction::Ret},
5510 UsedAssumedInformation))
5511 return indicatePessimisticFixpoint();
5512 return ChangeStatus::UNCHANGED;
5513 }
5514};
5515
5516struct AANoReturnFunction final : AANoReturnImpl {
5517 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5518 : AANoReturnImpl(IRP, A) {}
5519
5520 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5521};
5522
5523/// NoReturn attribute deduction for a call sites.
5524struct AANoReturnCallSite final
5525 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5526 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5527 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5528
5529 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5530};
5531} // namespace
5532
5533/// ----------------------- Instance Info ---------------------------------
5534
5535namespace {
5536/// A class to hold the state of for no-capture attributes.
5537struct AAInstanceInfoImpl : public AAInstanceInfo {
5538 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5539 : AAInstanceInfo(IRP, A) {}
5540
5541 void initialize(Attributor &A) override {
5542 Value &V = getAssociatedValue();
5543 if (auto *C = dyn_cast<Constant>(Val: &V)) {
5544 if (C->isThreadDependent())
5545 indicatePessimisticFixpoint();
5546 else
5547 indicateOptimisticFixpoint();
5548 return;
5549 }
5550 if (auto *CB = dyn_cast<CallBase>(Val: &V))
5551 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5552 !CB->mayReadFromMemory()) {
5553 indicateOptimisticFixpoint();
5554 return;
5555 }
5556 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5557 const auto *CI =
5558 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5559 F: *I->getFunction());
5560 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5561 indicatePessimisticFixpoint();
5562 return;
5563 }
5564 }
5565 }
5566
5567 ChangeStatus updateImpl(Attributor &A) override {
5568 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5569
5570 Value &V = getAssociatedValue();
5571 const Function *Scope = nullptr;
5572 if (auto *I = dyn_cast<Instruction>(Val: &V))
5573 Scope = I->getFunction();
5574 if (auto *A = dyn_cast<Argument>(Val: &V)) {
5575 Scope = A->getParent();
5576 if (!Scope->hasLocalLinkage())
5577 return Changed;
5578 }
5579 if (!Scope)
5580 return indicateOptimisticFixpoint();
5581
5582 bool IsKnownNoRecurse;
5583 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5584 A, QueryingAA: this, IRP: IRPosition::function(F: *Scope), DepClass: DepClassTy::OPTIONAL,
5585 IsKnown&: IsKnownNoRecurse))
5586 return Changed;
5587
5588 auto UsePred = [&](const Use &U, bool &Follow) {
5589 const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5590 if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5591 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5592 Follow = true;
5593 return true;
5594 }
5595 if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5596 (isa<StoreInst>(Val: UserI) &&
5597 cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5598 return true;
5599 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5600 // This check is not guaranteeing uniqueness but for now that we cannot
5601 // end up with two versions of \p U thinking it was one.
5602 auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5603 if (!Callee || !Callee->hasLocalLinkage())
5604 return true;
5605 if (!CB->isArgOperand(U: &U))
5606 return false;
5607 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5608 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5609 DepClass: DepClassTy::OPTIONAL);
5610 if (!ArgInstanceInfoAA ||
5611 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5612 return false;
5613 // If this call base might reach the scope again we might forward the
5614 // argument back here. This is very conservative.
5615 if (AA::isPotentiallyReachable(
5616 A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5617 GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5618 return false;
5619 return true;
5620 }
5621 return false;
5622 };
5623
5624 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5625 if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5626 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5627 if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5628 AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5629 return true;
5630 }
5631 return false;
5632 };
5633
5634 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5635 LivenessDepClass: DepClassTy::OPTIONAL,
5636 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5637 return indicatePessimisticFixpoint();
5638
5639 return Changed;
5640 }
5641
5642 const std::string getAsStr(Attributor *A) const override {
5643 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5644 }
5645
5646 void trackStatistics() const override {}
5647};
5648
5649/// InstanceInfo attribute for floating values.
5650struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5651 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5652 : AAInstanceInfoImpl(IRP, A) {}
5653};
5654
5655/// NoCapture attribute for function arguments.
5656struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5657 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5658 : AAInstanceInfoFloating(IRP, A) {}
5659};
5660
5661/// InstanceInfo attribute for call site arguments.
5662struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5663 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5664 : AAInstanceInfoImpl(IRP, A) {}
5665
5666 ChangeStatus updateImpl(Attributor &A) override {
5667 // TODO: Once we have call site specific value information we can provide
5668 // call site specific liveness information and then it makes
5669 // sense to specialize attributes for call sites arguments instead of
5670 // redirecting requests to the callee argument.
5671 Argument *Arg = getAssociatedArgument();
5672 if (!Arg)
5673 return indicatePessimisticFixpoint();
5674 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5675 auto *ArgAA =
5676 A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5677 if (!ArgAA)
5678 return indicatePessimisticFixpoint();
5679 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5680 }
5681};
5682
5683/// InstanceInfo attribute for function return value.
5684struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5685 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5686 : AAInstanceInfoImpl(IRP, A) {
5687 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5688 }
5689
5690 void initialize(Attributor &A) override {
5691 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5692 }
5693
5694 ChangeStatus updateImpl(Attributor &A) override {
5695 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5696 }
5697};
5698
5699/// InstanceInfo attribute deduction for a call site return value.
5700struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5701 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5702 : AAInstanceInfoFloating(IRP, A) {}
5703};
5704} // namespace
5705
5706/// ----------------------- Variable Capturing ---------------------------------
5707bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5708 Attribute::AttrKind ImpliedAttributeKind,
5709 bool IgnoreSubsumingPositions) {
5710 assert(ImpliedAttributeKind == Attribute::Captures &&
5711 "Unexpected attribute kind");
5712 Value &V = IRP.getAssociatedValue();
5713 if (!isa<Constant>(Val: V) && !IRP.isArgumentPosition())
5714 return V.use_empty();
5715
5716 // You cannot "capture" null if it is not a valid pointer.
5717 if (isa<UndefValue>(Val: V) ||
5718 (isa<ConstantPointerNull>(Val: V) &&
5719 !NullPointerIsDefined(F: IRP.getAnchorScope(),
5720 AS: V.getType()->getPointerAddressSpace()))) {
5721 return true;
5722 }
5723
5724 SmallVector<Attribute, 1> Attrs;
5725 A.getAttrs(IRP, AKs: {Attribute::Captures}, Attrs,
5726 /* IgnoreSubsumingPositions */ true);
5727 for (const Attribute &Attr : Attrs)
5728 if (capturesNothing(CC: Attr.getCaptureInfo()))
5729 return true;
5730
5731 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5732 if (Argument *Arg = IRP.getAssociatedArgument()) {
5733 SmallVector<Attribute, 1> Attrs;
5734 A.getAttrs(IRP: IRPosition::argument(Arg: *Arg),
5735 AKs: {Attribute::Captures, Attribute::ByVal}, Attrs,
5736 /* IgnoreSubsumingPositions */ true);
5737 bool ArgNoCapture = any_of(Range&: Attrs, P: [](Attribute Attr) {
5738 return Attr.getKindAsEnum() == Attribute::ByVal ||
5739 capturesNothing(CC: Attr.getCaptureInfo());
5740 });
5741 if (ArgNoCapture) {
5742 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(
5743 Context&: V.getContext(), CI: CaptureInfo::none()));
5744 return true;
5745 }
5746 }
5747
5748 if (const Function *F = IRP.getAssociatedFunction()) {
5749 // Check what state the associated function can actually capture.
5750 AANoCapture::StateType State;
5751 determineFunctionCaptureCapabilities(IRP, F: *F, State);
5752 if (State.isKnown(BitsEncoding: NO_CAPTURE)) {
5753 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(Context&: V.getContext(),
5754 CI: CaptureInfo::none()));
5755 return true;
5756 }
5757 }
5758
5759 return false;
5760}
5761
5762/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5763/// depending on the ability of the function associated with \p IRP to capture
5764/// state in memory and through "returning/throwing", respectively.
5765void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5766 const Function &F,
5767 BitIntegerState &State) {
5768 // TODO: Once we have memory behavior attributes we should use them here.
5769
5770 // If we know we cannot communicate or write to memory, we do not care about
5771 // ptr2int anymore. The value can still be captured by whether the function
5772 // returns.
5773 bool ReadOnly = F.onlyReadsMemory();
5774 bool NoThrow = F.doesNotThrow();
5775 bool WillReturn = F.willReturn();
5776 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5777 if (ReadOnly && NoThrow && WillReturn && IsVoidReturn) {
5778 State.addKnownBits(Bits: NO_CAPTURE);
5779 return;
5780 }
5781
5782 // A function cannot capture state in memory if it only reads memory, it can
5783 // however return/throw state and the state might be influenced by the
5784 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5785 if (ReadOnly)
5786 State.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5787
5788 // A function cannot communicate state back if it does not through
5789 // exceptions and doesn not return values.
5790 if (NoThrow && IsVoidReturn)
5791 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5792
5793 // Check existing "returned" attributes.
5794 int ArgNo = IRP.getCalleeArgNo();
5795 if (!NoThrow || ArgNo < 0 ||
5796 !F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned))
5797 return;
5798
5799 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5800 if (F.hasParamAttribute(ArgNo: U, Kind: Attribute::Returned)) {
5801 if (U == unsigned(ArgNo))
5802 State.removeAssumedBits(BitsEncoding: NOT_CAPTURED_IN_RET);
5803 else if (ReadOnly && WillReturn)
5804 State.addKnownBits(Bits: NO_CAPTURE);
5805 else
5806 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5807 break;
5808 }
5809}
5810
5811namespace {
5812/// A class to hold the state of for no-capture attributes.
5813struct AANoCaptureImpl : public AANoCapture {
5814 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5815
5816 void initialize(Attributor &A) override {
5817 bool IsKnown;
5818 assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
5819 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5820 (void)IsKnown;
5821 }
5822
5823 ChangeStatus updateImpl(Attributor &A) override;
5824
5825 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5826 SmallVectorImpl<Attribute> &Attrs) const override {
5827 if (!isAssumedNoCaptureMaybeReturned())
5828 return;
5829
5830 if (isArgumentPosition()) {
5831 if (isAssumedNoCapture())
5832 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: Attribute::Captures));
5833 else if (A.Opts.attributor_manifest_internal)
5834 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
5835 }
5836 }
5837
5838 const std::string getAsStr(Attributor *A) const override {
5839 if (isKnownNoCapture())
5840 return "known not-captured";
5841 if (isAssumedNoCapture())
5842 return "assumed not-captured";
5843 if (isKnownNoCaptureMaybeReturned())
5844 return "known not-captured-maybe-returned";
5845 if (isAssumedNoCaptureMaybeReturned())
5846 return "assumed not-captured-maybe-returned";
5847 return "assumed-captured";
5848 }
5849
5850 /// Check the use \p U and update \p State accordingly. Return true if we
5851 /// should continue to update the state.
5852 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5853 bool &Follow) {
5854 Instruction *UInst = cast<Instruction>(Val: U.getUser());
5855 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5856 << *UInst << "\n");
5857
5858 // Deal with ptr2int by following uses.
5859 if (isa<PtrToIntInst>(Val: UInst)) {
5860 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5861 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5862 /* Return */ CapturedInRet: true);
5863 }
5864
5865 // For stores we already checked if we can follow them, if they make it
5866 // here we give up.
5867 if (isa<StoreInst>(Val: UInst))
5868 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5869 /* Return */ CapturedInRet: true);
5870
5871 // Explicitly catch return instructions.
5872 if (isa<ReturnInst>(Val: UInst)) {
5873 if (UInst->getFunction() == getAnchorScope())
5874 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5875 /* Return */ CapturedInRet: true);
5876 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5877 /* Return */ CapturedInRet: true);
5878 }
5879
5880 // For now we only use special logic for call sites. However, the tracker
5881 // itself knows about a lot of other non-capturing cases already.
5882 auto *CB = dyn_cast<CallBase>(Val: UInst);
5883 if (!CB || !CB->isArgOperand(U: &U))
5884 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5885 /* Return */ CapturedInRet: true);
5886
5887 unsigned ArgNo = CB->getArgOperandNo(U: &U);
5888 const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
5889 // If we have a abstract no-capture attribute for the argument we can use
5890 // it to justify a non-capture attribute here. This allows recursion!
5891 bool IsKnownNoCapture;
5892 const AANoCapture *ArgNoCaptureAA = nullptr;
5893 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
5894 A, QueryingAA: this, IRP: CSArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
5895 AAPtr: &ArgNoCaptureAA);
5896 if (IsAssumedNoCapture)
5897 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5898 /* Return */ CapturedInRet: false);
5899 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
5900 Follow = true;
5901 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5902 /* Return */ CapturedInRet: false);
5903 }
5904
5905 // Lastly, we could not find a reason no-capture can be assumed so we don't.
5906 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5907 /* Return */ CapturedInRet: true);
5908 }
5909
5910 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
5911 /// \p CapturedInRet, then return true if we should continue updating the
5912 /// state.
5913 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
5914 bool CapturedInInt, bool CapturedInRet) {
5915 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
5916 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
5917 if (CapturedInMem)
5918 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_MEM);
5919 if (CapturedInInt)
5920 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_INT);
5921 if (CapturedInRet)
5922 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_RET);
5923 return State.isAssumed(BitsEncoding: AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
5924 }
5925};
5926
5927ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
5928 const IRPosition &IRP = getIRPosition();
5929 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
5930 : &IRP.getAssociatedValue();
5931 if (!V)
5932 return indicatePessimisticFixpoint();
5933
5934 const Function *F =
5935 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
5936
5937 // TODO: Is the checkForAllUses below useful for constants?
5938 if (!F)
5939 return indicatePessimisticFixpoint();
5940
5941 AANoCapture::StateType T;
5942 const IRPosition &FnPos = IRPosition::function(F: *F);
5943
5944 // Readonly means we cannot capture through memory.
5945 bool IsKnown;
5946 if (AA::isAssumedReadOnly(A, IRP: FnPos, QueryingAA: *this, IsKnown)) {
5947 T.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5948 if (IsKnown)
5949 addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5950 }
5951
5952 // Make sure all returned values are different than the underlying value.
5953 // TODO: we could do this in a more sophisticated way inside
5954 // AAReturnedValues, e.g., track all values that escape through returns
5955 // directly somehow.
5956 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
5957 SmallVector<AA::ValueAndContext> Values;
5958 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *F), AA: this, Values,
5959 S: AA::ValueScope::Intraprocedural,
5960 UsedAssumedInformation))
5961 return false;
5962 bool SeenConstant = false;
5963 for (const AA::ValueAndContext &VAC : Values) {
5964 if (isa<Constant>(Val: VAC.getValue())) {
5965 if (SeenConstant)
5966 return false;
5967 SeenConstant = true;
5968 } else if (!isa<Argument>(Val: VAC.getValue()) ||
5969 VAC.getValue() == getAssociatedArgument())
5970 return false;
5971 }
5972 return true;
5973 };
5974
5975 bool IsKnownNoUnwind;
5976 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
5977 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
5978 bool IsVoidTy = F->getReturnType()->isVoidTy();
5979 bool UsedAssumedInformation = false;
5980 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
5981 T.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5982 // If the function only reads memory and does not return the value, the
5983 // value can still be captured by whether the function returns.
5984 bool IsKnownWillReturn;
5985 bool IsAssumedWillReturn = AA::hasAssumedIRAttr<Attribute::WillReturn>(
5986 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownWillReturn);
5987 if (IsAssumedWillReturn && T.isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
5988 return ChangeStatus::UNCHANGED;
5989 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
5990 addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5991 if (IsKnownWillReturn && isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
5992 return indicateOptimisticFixpoint();
5993 }
5994 }
5995 }
5996
5997 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
5998 // TODO(captures): Make this more precise.
5999 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6000 if (capturesNothing(CC: CI))
6001 return true;
6002 if (CI.isPassthrough()) {
6003 Follow = true;
6004 return true;
6005 }
6006 return checkUse(A, State&: T, U, Follow);
6007 };
6008
6009 if (!A.checkForAllUses(Pred: UseCheck, QueryingAA: *this, V: *V))
6010 return indicatePessimisticFixpoint();
6011
6012 AANoCapture::StateType &S = getState();
6013 auto Assumed = S.getAssumed();
6014 S.intersectAssumedBits(BitsEncoding: T.getAssumed());
6015 if (!isAssumedNoCaptureMaybeReturned())
6016 return indicatePessimisticFixpoint();
6017 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6018 : ChangeStatus::CHANGED;
6019}
6020
6021/// NoCapture attribute for function arguments.
6022struct AANoCaptureArgument final : AANoCaptureImpl {
6023 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6024 : AANoCaptureImpl(IRP, A) {}
6025
6026 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6027};
6028
6029/// NoCapture attribute for call site arguments.
6030struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6031 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6032 : AANoCaptureImpl(IRP, A) {}
6033
6034 ChangeStatus updateImpl(Attributor &A) override {
6035 // TODO: Once we have call site specific value information we can provide
6036 // call site specific liveness information and then it makes
6037 // sense to specialize attributes for call sites arguments instead of
6038 // redirecting requests to the callee argument.
6039 Argument *Arg = getAssociatedArgument();
6040 if (!Arg)
6041 return indicatePessimisticFixpoint();
6042 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6043 bool IsKnownNoCapture;
6044 const AANoCapture *ArgAA = nullptr;
6045 if (AA::hasAssumedIRAttr<Attribute::Captures>(
6046 A, QueryingAA: this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6047 AAPtr: &ArgAA))
6048 return ChangeStatus::UNCHANGED;
6049 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6050 return indicatePessimisticFixpoint();
6051 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
6052 }
6053
6054 void trackStatistics() const override {
6055 STATS_DECLTRACK_CSARG_ATTR(nocapture)
6056 };
6057};
6058
6059/// NoCapture attribute for floating values.
6060struct AANoCaptureFloating final : AANoCaptureImpl {
6061 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6062 : AANoCaptureImpl(IRP, A) {}
6063
6064 void trackStatistics() const override {
6065 STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6066 }
6067};
6068
6069/// NoCapture attribute for function return value.
6070struct AANoCaptureReturned final : AANoCaptureImpl {
6071 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6072 : AANoCaptureImpl(IRP, A) {
6073 llvm_unreachable("NoCapture is not applicable to function returns!");
6074 }
6075
6076 void initialize(Attributor &A) override {
6077 llvm_unreachable("NoCapture is not applicable to function returns!");
6078 }
6079
6080 ChangeStatus updateImpl(Attributor &A) override {
6081 llvm_unreachable("NoCapture is not applicable to function returns!");
6082 }
6083
6084 void trackStatistics() const override {}
6085};
6086
6087/// NoCapture attribute deduction for a call site return value.
6088struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6089 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6090 : AANoCaptureImpl(IRP, A) {}
6091
6092 void initialize(Attributor &A) override {
6093 const Function *F = getAnchorScope();
6094 // Check what state the associated function can actually capture.
6095 determineFunctionCaptureCapabilities(IRP: getIRPosition(), F: *F, State&: *this);
6096 }
6097
6098 void trackStatistics() const override {
6099 STATS_DECLTRACK_CSRET_ATTR(nocapture)
6100 }
6101};
6102} // namespace
6103
6104/// ------------------ Value Simplify Attribute ----------------------------
6105
6106bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6107 // FIXME: Add a typecast support.
6108 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6109 A: SimplifiedAssociatedValue, B: Other, Ty);
6110 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6111 return false;
6112
6113 LLVM_DEBUG({
6114 if (SimplifiedAssociatedValue)
6115 dbgs() << "[ValueSimplify] is assumed to be "
6116 << **SimplifiedAssociatedValue << "\n";
6117 else
6118 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6119 });
6120 return true;
6121}
6122
6123namespace {
6124struct AAValueSimplifyImpl : AAValueSimplify {
6125 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6126 : AAValueSimplify(IRP, A) {}
6127
6128 void initialize(Attributor &A) override {
6129 if (getAssociatedValue().getType()->isVoidTy())
6130 indicatePessimisticFixpoint();
6131 if (A.hasSimplificationCallback(IRP: getIRPosition()))
6132 indicatePessimisticFixpoint();
6133 }
6134
6135 const std::string getAsStr(Attributor *A) const override {
6136 LLVM_DEBUG({
6137 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6138 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6139 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6140 });
6141 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6142 : "not-simple";
6143 }
6144
6145 void trackStatistics() const override {}
6146
6147 std::optional<Value *>
6148 getAssumedSimplifiedValue(Attributor &A) const override {
6149 return SimplifiedAssociatedValue;
6150 }
6151
6152 /// Ensure the return value is \p V with type \p Ty, if not possible return
6153 /// nullptr. If \p Check is true we will only verify such an operation would
6154 /// suceed and return a non-nullptr value if that is the case. No IR is
6155 /// generated or modified.
6156 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6157 bool Check) {
6158 if (auto *TypedV = AA::getWithType(V, Ty))
6159 return TypedV;
6160 if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6161 return Check ? &V
6162 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6163 S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6164 return nullptr;
6165 }
6166
6167 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6168 /// If \p Check is true we will only verify such an operation would suceed and
6169 /// return a non-nullptr value if that is the case. No IR is generated or
6170 /// modified.
6171 static Value *reproduceInst(Attributor &A,
6172 const AbstractAttribute &QueryingAA,
6173 Instruction &I, Type &Ty, Instruction *CtxI,
6174 bool Check, ValueToValueMapTy &VMap) {
6175 assert(CtxI && "Cannot reproduce an instruction without context!");
6176 if (Check && (I.mayReadFromMemory() ||
6177 !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6178 /* TLI */ DT: nullptr)))
6179 return nullptr;
6180 for (Value *Op : I.operands()) {
6181 Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6182 if (!NewOp) {
6183 assert(Check && "Manifest of new value unexpectedly failed!");
6184 return nullptr;
6185 }
6186 if (!Check)
6187 VMap[Op] = NewOp;
6188 }
6189 if (Check)
6190 return &I;
6191
6192 Instruction *CloneI = I.clone();
6193 // TODO: Try to salvage debug information here.
6194 CloneI->setDebugLoc(DebugLoc());
6195 VMap[&I] = CloneI;
6196 CloneI->insertBefore(InsertPos: CtxI->getIterator());
6197 RemapInstruction(I: CloneI, VM&: VMap);
6198 return CloneI;
6199 }
6200
6201 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6202 /// If \p Check is true we will only verify such an operation would suceed and
6203 /// return a non-nullptr value if that is the case. No IR is generated or
6204 /// modified.
6205 static Value *reproduceValue(Attributor &A,
6206 const AbstractAttribute &QueryingAA, Value &V,
6207 Type &Ty, Instruction *CtxI, bool Check,
6208 ValueToValueMapTy &VMap) {
6209 if (const auto &NewV = VMap.lookup(Val: &V))
6210 return NewV;
6211 bool UsedAssumedInformation = false;
6212 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6213 V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6214 if (!SimpleV.has_value())
6215 return PoisonValue::get(T: &Ty);
6216 Value *EffectiveV = &V;
6217 if (*SimpleV)
6218 EffectiveV = *SimpleV;
6219 if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6220 return C;
6221 if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6222 InfoCache&: A.getInfoCache()))
6223 return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6224 if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6225 if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6226 return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6227 return nullptr;
6228 }
6229
6230 /// Return a value we can use as replacement for the associated one, or
6231 /// nullptr if we don't have one that makes sense.
6232 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6233 Value *NewV = SimplifiedAssociatedValue
6234 ? *SimplifiedAssociatedValue
6235 : UndefValue::get(T: getAssociatedType());
6236 if (NewV && NewV != &getAssociatedValue()) {
6237 ValueToValueMapTy VMap;
6238 // First verify we can reprduce the value with the required type at the
6239 // context location before we actually start modifying the IR.
6240 if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6241 /* CheckOnly */ Check: true, VMap))
6242 return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6243 /* CheckOnly */ Check: false, VMap);
6244 }
6245 return nullptr;
6246 }
6247
6248 /// Helper function for querying AAValueSimplify and updating candidate.
6249 /// \param IRP The value position we are trying to unify with SimplifiedValue
6250 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6251 const IRPosition &IRP, bool Simplify = true) {
6252 bool UsedAssumedInformation = false;
6253 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6254 if (Simplify)
6255 QueryingValueSimplified = A.getAssumedSimplified(
6256 IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6257 return unionAssumed(Other: QueryingValueSimplified);
6258 }
6259
6260 /// Returns a candidate is found or not
6261 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6262 if (!getAssociatedValue().getType()->isIntegerTy())
6263 return false;
6264
6265 // This will also pass the call base context.
6266 const auto *AA =
6267 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6268 if (!AA)
6269 return false;
6270
6271 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6272
6273 if (!COpt) {
6274 SimplifiedAssociatedValue = std::nullopt;
6275 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6276 return true;
6277 }
6278 if (auto *C = *COpt) {
6279 SimplifiedAssociatedValue = C;
6280 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6281 return true;
6282 }
6283 return false;
6284 }
6285
6286 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6287 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6288 return true;
6289 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6290 return true;
6291 return false;
6292 }
6293
6294 ChangeStatus manifest(Attributor &A) override {
6295 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6296 for (auto &U : getAssociatedValue().uses()) {
6297 // Check if we need to adjust the insertion point to make sure the IR is
6298 // valid.
6299 Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6300 if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6301 IP = PHI->getIncomingBlock(U)->getTerminator();
6302 if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6303 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6304 << " -> " << *NewV << " :: " << *this << "\n");
6305 if (A.changeUseAfterManifest(U, NV&: *NewV))
6306 Changed = ChangeStatus::CHANGED;
6307 }
6308 }
6309
6310 return Changed | AAValueSimplify::manifest(A);
6311 }
6312
6313 ChangeStatus indicatePessimisticFixpoint() override {
6314 SimplifiedAssociatedValue = &getAssociatedValue();
6315 return AAValueSimplify::indicatePessimisticFixpoint();
6316 }
6317};
6318
6319struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6320 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6321 : AAValueSimplifyImpl(IRP, A) {}
6322
6323 void initialize(Attributor &A) override {
6324 AAValueSimplifyImpl::initialize(A);
6325 if (A.hasAttr(IRP: getIRPosition(),
6326 AKs: {Attribute::InAlloca, Attribute::Preallocated,
6327 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6328 /* IgnoreSubsumingPositions */ true))
6329 indicatePessimisticFixpoint();
6330 }
6331
6332 ChangeStatus updateImpl(Attributor &A) override {
6333 // Byval is only replacable if it is readonly otherwise we would write into
6334 // the replaced value and not the copy that byval creates implicitly.
6335 Argument *Arg = getAssociatedArgument();
6336 if (Arg->hasByValAttr()) {
6337 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6338 // there is no race by not copying a constant byval.
6339 bool IsKnown;
6340 if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6341 return indicatePessimisticFixpoint();
6342 }
6343
6344 auto Before = SimplifiedAssociatedValue;
6345
6346 auto PredForCallSite = [&](AbstractCallSite ACS) {
6347 const IRPosition &ACSArgPos =
6348 IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6349 // Check if a coresponding argument was found or if it is on not
6350 // associated (which can happen for callback calls).
6351 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6352 return false;
6353
6354 // Simplify the argument operand explicitly and check if the result is
6355 // valid in the current scope. This avoids refering to simplified values
6356 // in other functions, e.g., we don't want to say a an argument in a
6357 // static function is actually an argument in a different function.
6358 bool UsedAssumedInformation = false;
6359 std::optional<Constant *> SimpleArgOp =
6360 A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6361 if (!SimpleArgOp)
6362 return true;
6363 if (!*SimpleArgOp)
6364 return false;
6365 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6366 return false;
6367 return unionAssumed(Other: *SimpleArgOp);
6368 };
6369
6370 // Generate a answer specific to a call site context.
6371 bool Success;
6372 bool UsedAssumedInformation = false;
6373 if (hasCallBaseContext() &&
6374 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6375 Success = PredForCallSite(
6376 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6377 else
6378 Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6379 UsedAssumedInformation);
6380
6381 if (!Success)
6382 if (!askSimplifiedValueForOtherAAs(A))
6383 return indicatePessimisticFixpoint();
6384
6385 // If a candidate was found in this update, return CHANGED.
6386 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6387 : ChangeStatus ::CHANGED;
6388 }
6389
6390 void trackStatistics() const override {
6391 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6392 }
6393};
6394
6395struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6396 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6397 : AAValueSimplifyImpl(IRP, A) {}
6398
6399 std::optional<Value *>
6400 getAssumedSimplifiedValue(Attributor &A) const override {
6401 if (!isValidState())
6402 return nullptr;
6403 return SimplifiedAssociatedValue;
6404 }
6405
6406 ChangeStatus updateImpl(Attributor &A) override {
6407 auto Before = SimplifiedAssociatedValue;
6408
6409 auto ReturnInstCB = [&](Instruction &I) {
6410 auto &RI = cast<ReturnInst>(Val&: I);
6411 return checkAndUpdate(
6412 A, QueryingAA: *this,
6413 IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6414 };
6415
6416 bool UsedAssumedInformation = false;
6417 if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6418 UsedAssumedInformation))
6419 if (!askSimplifiedValueForOtherAAs(A))
6420 return indicatePessimisticFixpoint();
6421
6422 // If a candidate was found in this update, return CHANGED.
6423 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6424 : ChangeStatus ::CHANGED;
6425 }
6426
6427 ChangeStatus manifest(Attributor &A) override {
6428 // We queried AAValueSimplify for the returned values so they will be
6429 // replaced if a simplified form was found. Nothing to do here.
6430 return ChangeStatus::UNCHANGED;
6431 }
6432
6433 void trackStatistics() const override {
6434 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6435 }
6436};
6437
6438struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6439 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6440 : AAValueSimplifyImpl(IRP, A) {}
6441
6442 void initialize(Attributor &A) override {
6443 AAValueSimplifyImpl::initialize(A);
6444 Value &V = getAnchorValue();
6445
6446 // TODO: add other stuffs
6447 if (isa<Constant>(Val: V))
6448 indicatePessimisticFixpoint();
6449 }
6450
6451 ChangeStatus updateImpl(Attributor &A) override {
6452 auto Before = SimplifiedAssociatedValue;
6453 if (!askSimplifiedValueForOtherAAs(A))
6454 return indicatePessimisticFixpoint();
6455
6456 // If a candidate was found in this update, return CHANGED.
6457 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6458 : ChangeStatus ::CHANGED;
6459 }
6460
6461 void trackStatistics() const override {
6462 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6463 }
6464};
6465
6466struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6467 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6468 : AAValueSimplifyImpl(IRP, A) {}
6469
6470 void initialize(Attributor &A) override {
6471 SimplifiedAssociatedValue = nullptr;
6472 indicateOptimisticFixpoint();
6473 }
6474 ChangeStatus updateImpl(Attributor &A) override {
6475 llvm_unreachable(
6476 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6477 }
6478 void trackStatistics() const override {
6479 STATS_DECLTRACK_FN_ATTR(value_simplify)
6480 }
6481};
6482
6483struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6484 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6485 : AAValueSimplifyFunction(IRP, A) {}
6486 void trackStatistics() const override {
6487 STATS_DECLTRACK_CS_ATTR(value_simplify)
6488 }
6489};
6490
6491struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6492 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6493 : AAValueSimplifyImpl(IRP, A) {}
6494
6495 void initialize(Attributor &A) override {
6496 AAValueSimplifyImpl::initialize(A);
6497 Function *Fn = getAssociatedFunction();
6498 assert(Fn && "Did expect an associted function");
6499 for (Argument &Arg : Fn->args()) {
6500 if (Arg.hasReturnedAttr()) {
6501 auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6502 ArgNo: Arg.getArgNo());
6503 if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6504 checkAndUpdate(A, QueryingAA: *this, IRP))
6505 indicateOptimisticFixpoint();
6506 else
6507 indicatePessimisticFixpoint();
6508 return;
6509 }
6510 }
6511 }
6512
6513 ChangeStatus updateImpl(Attributor &A) override {
6514 return indicatePessimisticFixpoint();
6515 }
6516
6517 void trackStatistics() const override {
6518 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6519 }
6520};
6521
6522struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6523 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6524 : AAValueSimplifyFloating(IRP, A) {}
6525
6526 ChangeStatus manifest(Attributor &A) override {
6527 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6528 // TODO: We should avoid simplification duplication to begin with.
6529 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6530 IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6531 if (FloatAA && FloatAA->getState().isValidState())
6532 return Changed;
6533
6534 if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6535 Use &U = cast<CallBase>(Val: &getAnchorValue())
6536 ->getArgOperandUse(i: getCallSiteArgNo());
6537 if (A.changeUseAfterManifest(U, NV&: *NewV))
6538 Changed = ChangeStatus::CHANGED;
6539 }
6540
6541 return Changed | AAValueSimplify::manifest(A);
6542 }
6543
6544 void trackStatistics() const override {
6545 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6546 }
6547};
6548} // namespace
6549
6550/// Collect the non-musttail tail calls in the function of \p QueryingAA. Tail
6551/// calls cannot access allocas of the caller.
6552/// TODO: Be smarter about new allocas escaping into tail calls.
6553static bool collectTailCalls(Attributor &A, const AbstractAttribute &QueryingAA,
6554 SmallVectorImpl<CallInst *> &TailCalls) {
6555 bool UsedAssumedInformation = false;
6556 return A.checkForAllInstructions(
6557 Pred: [&](Instruction &I) {
6558 auto &CI = cast<CallInst>(Val&: I);
6559 if (CI.isTailCall() && !CI.isMustTailCall())
6560 TailCalls.push_back(Elt: &CI);
6561 return true;
6562 },
6563 QueryingAA, Opcodes: {Instruction::Call}, UsedAssumedInformation);
6564}
6565
6566/// ----------------------- Heap-To-Stack Conversion ---------------------------
6567namespace {
6568struct AAHeapToStackFunction final : public AAHeapToStack {
6569
6570 static bool isGlobalizedLocal(const CallBase &CB) {
6571 Attribute A = CB.getFnAttr(Kind: "alloc-family");
6572 return A.isValid() && A.getValueAsString() == "__kmpc_alloc_shared";
6573 }
6574
6575 struct AllocationInfo {
6576 /// The call that allocates the memory.
6577 CallBase *const CB;
6578
6579 /// Whether this allocation is an OpenMP globalized local variable.
6580 bool IsGlobalizedLocal = false;
6581
6582 /// The status wrt. a rewrite.
6583 enum {
6584 STACK_DUE_TO_USE,
6585 STACK_DUE_TO_FREE,
6586 INVALID,
6587 } Status = STACK_DUE_TO_USE;
6588
6589 /// Flag to indicate if we encountered a use that might free this allocation
6590 /// but which is not in the deallocation infos.
6591 bool HasPotentiallyFreeingUnknownUses = false;
6592
6593 /// Flag to indicate that we should place the new alloca in the function
6594 /// entry block rather than where the call site (CB) is.
6595 bool MoveAllocaIntoEntry = true;
6596
6597 /// The set of free calls that use this allocation.
6598 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6599 };
6600
6601 struct DeallocationInfo {
6602 /// The call that deallocates the memory.
6603 CallBase *const CB;
6604 /// The value freed by the call.
6605 Value *FreedOp;
6606
6607 /// Flag to indicate if we don't know all objects this deallocation might
6608 /// free.
6609 bool MightFreeUnknownObjects = false;
6610
6611 /// The set of allocation calls that are potentially freed.
6612 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6613 };
6614
6615 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6616 : AAHeapToStack(IRP, A) {}
6617
6618 ~AAHeapToStackFunction() override {
6619 // Ensure we call the destructor so we release any memory allocated in the
6620 // sets.
6621 for (auto &It : AllocationInfos)
6622 It.second->~AllocationInfo();
6623 for (auto &It : DeallocationInfos)
6624 It.second->~DeallocationInfo();
6625 }
6626
6627 void initialize(Attributor &A) override {
6628 AAHeapToStack::initialize(A);
6629
6630 const Function *F = getAnchorScope();
6631 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6632
6633 auto AllocationIdentifierCB = [&](Instruction &I) {
6634 CallBase *CB = dyn_cast<CallBase>(Val: &I);
6635 if (!CB)
6636 return true;
6637 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6638 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6639 return true;
6640 }
6641 // To do heap to stack, we need to know that the allocation itself is
6642 // removable once uses are rewritten, and that we can initialize the
6643 // alloca to the same pattern as the original allocation result.
6644 if (isRemovableAlloc(V: CB, TLI)) {
6645 auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6646 if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6647 AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6648 AllocationInfos[CB] = AI;
6649 AI->IsGlobalizedLocal = isGlobalizedLocal(CB: *CB);
6650 }
6651 }
6652 return true;
6653 };
6654
6655 bool UsedAssumedInformation = false;
6656 bool Success = A.checkForAllCallLikeInstructions(
6657 Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6658 /* CheckBBLivenessOnly */ false,
6659 /* CheckPotentiallyDead */ true);
6660 (void)Success;
6661 assert(Success && "Did not expect the call base visit callback to fail!");
6662
6663 Attributor::SimplifictionCallbackTy SCB =
6664 [](const IRPosition &, const AbstractAttribute *,
6665 bool &) -> std::optional<Value *> { return nullptr; };
6666 for (const auto &It : AllocationInfos)
6667 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6668 CB: SCB);
6669 for (const auto &It : DeallocationInfos)
6670 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6671 CB: SCB);
6672 }
6673
6674 const std::string getAsStr(Attributor *A) const override {
6675 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6676 for (const auto &It : AllocationInfos) {
6677 if (It.second->Status == AllocationInfo::INVALID)
6678 ++NumInvalidMallocs;
6679 else
6680 ++NumH2SMallocs;
6681 }
6682 return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6683 std::to_string(val: NumInvalidMallocs);
6684 }
6685
6686 void trackStatistics() const override {
6687 STATS_DECL(
6688 MallocCalls, Function,
6689 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6690 for (const auto &It : AllocationInfos)
6691 if (It.second->Status != AllocationInfo::INVALID)
6692 ++BUILD_STAT_NAME(MallocCalls, Function);
6693 }
6694
6695 bool isAssumedHeapToStack(const CallBase &CB) const override {
6696 if (isValidState())
6697 if (AllocationInfo *AI =
6698 AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6699 return AI->Status != AllocationInfo::INVALID;
6700 return false;
6701 }
6702
6703 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6704 if (!isValidState())
6705 return false;
6706
6707 for (const auto &It : AllocationInfos) {
6708 AllocationInfo &AI = *It.second;
6709 if (AI.Status == AllocationInfo::INVALID)
6710 continue;
6711
6712 if (AI.PotentialFreeCalls.count(key: &CB))
6713 return true;
6714 }
6715
6716 return false;
6717 }
6718
6719 ChangeStatus manifest(Attributor &A) override {
6720 assert(getState().isValidState() &&
6721 "Attempted to manifest an invalid state!");
6722
6723 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6724 Function *F = getAnchorScope();
6725 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6726
6727 // The new allocas may escape into tail calls.
6728 SmallVector<CallInst *, 16> TailCalls;
6729 if (any_of(Range&: AllocationInfos,
6730 P: [](const std::pair<CallBase *, AllocationInfo *> &It) {
6731 return It.second->Status != AllocationInfo::INVALID;
6732 }) &&
6733 !collectTailCalls(A, QueryingAA: *this, TailCalls))
6734 return ChangeStatus::UNCHANGED;
6735
6736 for (auto &It : AllocationInfos) {
6737 AllocationInfo &AI = *It.second;
6738 if (AI.Status == AllocationInfo::INVALID)
6739 continue;
6740
6741 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6742 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6743 A.deleteAfterManifest(I&: *FreeCall);
6744 HasChanged = ChangeStatus::CHANGED;
6745 }
6746
6747 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6748 << "\n");
6749
6750 auto Remark = [&](OptimizationRemark OR) {
6751 if (AI.IsGlobalizedLocal)
6752 return OR << "Moving globalized variable to the stack.";
6753 return OR << "Moving memory allocation from the heap to the stack.";
6754 };
6755 if (AI.IsGlobalizedLocal)
6756 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6757 else
6758 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6759
6760 const DataLayout &DL = A.getInfoCache().getDL();
6761 Value *Size;
6762 std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6763 if (SizeAPI) {
6764 Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6765 } else {
6766 ObjectSizeOpts Opts;
6767 ObjectSizeOffsetEvaluator Eval(*AI.CB->getModule(), TLI, Opts);
6768 SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6769 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6770 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6771 Size = SizeOffsetPair.Size;
6772 }
6773
6774 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6775 ? F->getEntryBlock().begin()
6776 : AI.CB->getIterator();
6777
6778 Align Alignment(1);
6779 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6780 Alignment = std::max(a: Alignment, b: *RetAlign);
6781 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6782 std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6783 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6784 "Expected an alignment during manifest!");
6785 Alignment =
6786 std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6787 }
6788
6789 // TODO: Hoist the alloca towards the function entry.
6790 unsigned AS = DL.getAllocaAddrSpace();
6791 Instruction *Alloca =
6792 new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6793 AI.CB->getName() + ".h2s", IP);
6794
6795 if (Alloca->getType() != AI.CB->getType())
6796 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6797 S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6798
6799 auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6800 auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6801 assert(InitVal &&
6802 "Must be able to materialize initial memory state of allocation");
6803
6804 A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6805
6806 if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6807 auto *NBB = II->getNormalDest();
6808 UncondBrInst::Create(Target: NBB, InsertBefore: AI.CB->getParent());
6809 A.deleteAfterManifest(I&: *AI.CB);
6810 } else {
6811 A.deleteAfterManifest(I&: *AI.CB);
6812 }
6813
6814 // Initialize the alloca with the same value as used by the allocation
6815 // function. We can skip undef as the initial value of an alloc is
6816 // undef, and the memset would simply end up being DSEd.
6817 if (!isa<UndefValue>(Val: InitVal)) {
6818 IRBuilder<> Builder(Alloca->getNextNode());
6819 // TODO: Use alignment above if align!=1
6820 Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6821 }
6822 HasChanged = ChangeStatus::CHANGED;
6823 }
6824
6825 for (CallInst *CI : TailCalls)
6826 CI->setTailCall(false);
6827
6828 return HasChanged;
6829 }
6830
6831 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6832 Value &V) {
6833 bool UsedAssumedInformation = false;
6834 std::optional<Constant *> SimpleV =
6835 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6836 if (!SimpleV)
6837 return APInt(64, 0);
6838 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
6839 return CI->getValue();
6840 return std::nullopt;
6841 }
6842
6843 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6844 AllocationInfo &AI) {
6845 auto Mapper = [&](const Value *V) -> const Value * {
6846 bool UsedAssumedInformation = false;
6847 if (std::optional<Constant *> SimpleV =
6848 A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
6849 if (*SimpleV)
6850 return *SimpleV;
6851 return V;
6852 };
6853
6854 const Function *F = getAnchorScope();
6855 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6856 return getAllocSize(CB: AI.CB, TLI, Mapper);
6857 }
6858
6859 /// Collection of all malloc-like calls in a function with associated
6860 /// information.
6861 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
6862
6863 /// Collection of all free-like calls in a function with associated
6864 /// information.
6865 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
6866
6867 ChangeStatus updateImpl(Attributor &A) override;
6868};
6869
6870ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6871 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6872 const Function *F = getAnchorScope();
6873 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6874
6875 const auto *LivenessAA =
6876 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
6877
6878 MustBeExecutedContextExplorer *Explorer =
6879 A.getInfoCache().getMustBeExecutedContextExplorer();
6880
6881 bool StackIsAccessibleByOtherThreads =
6882 A.getInfoCache().stackIsAccessibleByOtherThreads();
6883
6884 LoopInfo *LI =
6885 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
6886 std::optional<bool> MayContainIrreducibleControl;
6887 auto IsInLoop = [&](BasicBlock &BB) {
6888 if (&F->getEntryBlock() == &BB)
6889 return false;
6890 if (!MayContainIrreducibleControl.has_value())
6891 MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
6892 if (*MayContainIrreducibleControl)
6893 return true;
6894 if (!LI)
6895 return true;
6896 return LI->getLoopFor(BB: &BB) != nullptr;
6897 };
6898
6899 // Flag to ensure we update our deallocation information at most once per
6900 // updateImpl call and only if we use the free check reasoning.
6901 bool HasUpdatedFrees = false;
6902
6903 auto UpdateFrees = [&]() {
6904 HasUpdatedFrees = true;
6905
6906 for (auto &It : DeallocationInfos) {
6907 DeallocationInfo &DI = *It.second;
6908 // For now we cannot use deallocations that have unknown inputs, skip
6909 // them.
6910 if (DI.MightFreeUnknownObjects)
6911 continue;
6912
6913 // No need to analyze dead calls, ignore them instead.
6914 bool UsedAssumedInformation = false;
6915 if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
6916 /* CheckBBLivenessOnly */ true))
6917 continue;
6918
6919 // Use the non-optimistic version to get the freed object.
6920 Value *Obj = getUnderlyingObject(V: DI.FreedOp);
6921 if (!Obj) {
6922 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
6923 DI.MightFreeUnknownObjects = true;
6924 continue;
6925 }
6926
6927 // Free of null and undef can be ignored as no-ops (or UB in the latter
6928 // case).
6929 if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
6930 continue;
6931
6932 CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
6933 if (!ObjCB) {
6934 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
6935 << "\n");
6936 DI.MightFreeUnknownObjects = true;
6937 continue;
6938 }
6939
6940 AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
6941 if (!AI) {
6942 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
6943 << "\n");
6944 DI.MightFreeUnknownObjects = true;
6945 continue;
6946 }
6947
6948 DI.PotentialAllocationCalls.insert(X: ObjCB);
6949 }
6950 };
6951
6952 auto FreeCheck = [&](AllocationInfo &AI) {
6953 // If the stack is not accessible by other threads, the "must-free" logic
6954 // doesn't apply as the pointer could be shared and needs to be places in
6955 // "shareable" memory.
6956 if (!StackIsAccessibleByOtherThreads) {
6957 bool IsKnownNoSycn;
6958 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
6959 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn)) {
6960 LLVM_DEBUG(
6961 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
6962 "other threads and function is not nosync:\n");
6963 return false;
6964 }
6965 }
6966 if (!HasUpdatedFrees)
6967 UpdateFrees();
6968
6969 // TODO: Allow multi exit functions that have different free calls.
6970 if (AI.PotentialFreeCalls.size() != 1) {
6971 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
6972 << AI.PotentialFreeCalls.size() << "\n");
6973 return false;
6974 }
6975 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
6976 DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
6977 if (!DI) {
6978 LLVM_DEBUG(
6979 dbgs() << "[H2S] unique free call was not known as deallocation call "
6980 << *UniqueFree << "\n");
6981 return false;
6982 }
6983 if (DI->MightFreeUnknownObjects) {
6984 LLVM_DEBUG(
6985 dbgs() << "[H2S] unique free call might free unknown allocations\n");
6986 return false;
6987 }
6988 if (DI->PotentialAllocationCalls.empty())
6989 return true;
6990 if (DI->PotentialAllocationCalls.size() > 1) {
6991 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
6992 << DI->PotentialAllocationCalls.size()
6993 << " different allocations\n");
6994 return false;
6995 }
6996 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
6997 LLVM_DEBUG(
6998 dbgs()
6999 << "[H2S] unique free call not known to free this allocation but "
7000 << **DI->PotentialAllocationCalls.begin() << "\n");
7001 return false;
7002 }
7003
7004 // __kmpc_alloc_shared and __kmpc_free_shared are by construction matched.
7005 if (!AI.IsGlobalizedLocal) {
7006 Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
7007 if (!Explorer || !Explorer->isInContextOf(I: UniqueFree, PP: CtxI)) {
7008 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7009 "with the allocation "
7010 << *UniqueFree << "\n");
7011 return false;
7012 }
7013 }
7014 return true;
7015 };
7016
7017 auto UsesCheck = [&](AllocationInfo &AI) {
7018 bool ValidUsesOnly = true;
7019
7020 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7021 Instruction *UserI = cast<Instruction>(Val: U.getUser());
7022 if (isa<LoadInst>(Val: UserI))
7023 return true;
7024 if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
7025 if (SI->getValueOperand() == U.get()) {
7026 LLVM_DEBUG(dbgs()
7027 << "[H2S] escaping store to memory: " << *UserI << "\n");
7028 ValidUsesOnly = false;
7029 } else {
7030 // A store into the malloc'ed memory is fine.
7031 }
7032 return true;
7033 }
7034 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7035 // A musttail call cannot access allocas of the caller, and its marker
7036 // cannot be removed.
7037 if (CB->isMustTailCall()) {
7038 LLVM_DEBUG(dbgs() << "[H2S] musttail user: " << *UserI << "\n");
7039 ValidUsesOnly = false;
7040 return true;
7041 }
7042
7043 if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7044 return true;
7045 if (DeallocationInfos.count(Key: CB)) {
7046 AI.PotentialFreeCalls.insert(X: CB);
7047 return true;
7048 }
7049
7050 unsigned ArgNo = CB->getArgOperandNo(U: &U);
7051 auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7052
7053 bool IsKnownNoCapture;
7054 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7055 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
7056
7057 // If a call site argument use is nofree, we are fine.
7058 bool IsKnownNoFree;
7059 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7060 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoFree);
7061
7062 if (!IsAssumedNoCapture ||
7063 (!AI.IsGlobalizedLocal && !IsAssumedNoFree)) {
7064 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7065
7066 // Emit a missed remark if this is missed OpenMP globalization.
7067 auto Remark = [&](OptimizationRemarkMissed ORM) {
7068 return ORM
7069 << "Could not move globalized variable to the stack. "
7070 "Variable is potentially captured in call. Mark "
7071 "parameter as `__attribute__((noescape))` to override.";
7072 };
7073
7074 if (ValidUsesOnly && AI.IsGlobalizedLocal)
7075 A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7076
7077 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7078 ValidUsesOnly = false;
7079 }
7080 return true;
7081 }
7082
7083 if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7084 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7085 Follow = true;
7086 return true;
7087 }
7088 // Unknown user for which we can not track uses further (in a way that
7089 // makes sense).
7090 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7091 ValidUsesOnly = false;
7092 return true;
7093 };
7094 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7095 LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7096 EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7097 auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7098 return !SI || StackIsAccessibleByOtherThreads ||
7099 AA::isAssumedThreadLocalObject(
7100 A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7101 }))
7102 return false;
7103 return ValidUsesOnly;
7104 };
7105
7106 // The actual update starts here. We look at all allocations and depending on
7107 // their status perform the appropriate check(s).
7108 for (auto &It : AllocationInfos) {
7109 AllocationInfo &AI = *It.second;
7110 if (AI.Status == AllocationInfo::INVALID)
7111 continue;
7112
7113 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7114 std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7115 if (!APAlign) {
7116 // Can't generate an alloca which respects the required alignment
7117 // on the allocation.
7118 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7119 << "\n");
7120 AI.Status = AllocationInfo::INVALID;
7121 Changed = ChangeStatus::CHANGED;
7122 continue;
7123 }
7124 if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7125 !APAlign->isPowerOf2()) {
7126 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7127 << "\n");
7128 AI.Status = AllocationInfo::INVALID;
7129 Changed = ChangeStatus::CHANGED;
7130 continue;
7131 }
7132 }
7133
7134 std::optional<APInt> Size = getSize(A, AA: *this, AI);
7135 if (!AI.IsGlobalizedLocal && A.Opts.max_heap_to_stack_size != -1) {
7136 if (!Size || Size->ugt(RHS: A.Opts.max_heap_to_stack_size)) {
7137 LLVM_DEBUG({
7138 if (!Size)
7139 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7140 else
7141 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7142 << A.Opts.max_heap_to_stack_size << "\n";
7143 });
7144
7145 AI.Status = AllocationInfo::INVALID;
7146 Changed = ChangeStatus::CHANGED;
7147 continue;
7148 }
7149 }
7150
7151 switch (AI.Status) {
7152 case AllocationInfo::STACK_DUE_TO_USE:
7153 if (UsesCheck(AI))
7154 break;
7155 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7156 [[fallthrough]];
7157 case AllocationInfo::STACK_DUE_TO_FREE:
7158 if (FreeCheck(AI))
7159 break;
7160 AI.Status = AllocationInfo::INVALID;
7161 Changed = ChangeStatus::CHANGED;
7162 break;
7163 case AllocationInfo::INVALID:
7164 llvm_unreachable("Invalid allocations should never reach this point!");
7165 };
7166
7167 // Check if we still think we can move it into the entry block. If the
7168 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7169 // ignore the potential complications associated with loops.
7170 bool IsGlobalizedLocal = AI.IsGlobalizedLocal;
7171 if (AI.MoveAllocaIntoEntry &&
7172 (!Size.has_value() ||
7173 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7174 AI.MoveAllocaIntoEntry = false;
7175 }
7176
7177 return Changed;
7178}
7179} // namespace
7180
7181/// ----------------------- Privatizable Pointers ------------------------------
7182namespace {
7183struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7184 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7185 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7186
7187 ChangeStatus indicatePessimisticFixpoint() override {
7188 AAPrivatizablePtr::indicatePessimisticFixpoint();
7189 PrivatizableType = nullptr;
7190 return ChangeStatus::CHANGED;
7191 }
7192
7193 /// Identify the type we can chose for a private copy of the underlying
7194 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7195 /// none.
7196 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7197
7198 /// Return a privatizable type that encloses both T0 and T1.
7199 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7200 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7201 std::optional<Type *> T1) {
7202 if (!T0)
7203 return T1;
7204 if (!T1)
7205 return T0;
7206 if (T0 == T1)
7207 return T0;
7208 return nullptr;
7209 }
7210
7211 std::optional<Type *> getPrivatizableType() const override {
7212 return PrivatizableType;
7213 }
7214
7215 const std::string getAsStr(Attributor *A) const override {
7216 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7217 }
7218
7219protected:
7220 std::optional<Type *> PrivatizableType;
7221};
7222
7223// TODO: Do this for call site arguments (probably also other values) as well.
7224
7225struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7226 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7227 : AAPrivatizablePtrImpl(IRP, A) {}
7228
7229 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7230 // If this is a byval argument and we know all the call sites (so we can
7231 // rewrite them), there is no need to check them explicitly.
7232 bool UsedAssumedInformation = false;
7233 SmallVector<Attribute, 1> Attrs;
7234 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::ByVal}, Attrs,
7235 /* IgnoreSubsumingPositions */ true);
7236 if (!Attrs.empty() &&
7237 A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7238 RequireAllCallSites: true, UsedAssumedInformation))
7239 return Attrs[0].getValueAsType();
7240
7241 std::optional<Type *> Ty;
7242 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7243
7244 // Make sure the associated call site argument has the same type at all call
7245 // sites and it is an allocation we know is safe to privatize, for now that
7246 // means we only allow alloca instructions.
7247 // TODO: We can additionally analyze the accesses in the callee to create
7248 // the type from that information instead. That is a little more
7249 // involved and will be done in a follow up patch.
7250 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7251 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7252 // Check if a coresponding argument was found or if it is one not
7253 // associated (which can happen for callback calls).
7254 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7255 return false;
7256
7257 // Check that all call sites agree on a type.
7258 auto *PrivCSArgAA =
7259 A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7260 if (!PrivCSArgAA)
7261 return false;
7262 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7263
7264 LLVM_DEBUG({
7265 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7266 if (CSTy && *CSTy)
7267 (*CSTy)->print(dbgs());
7268 else if (CSTy)
7269 dbgs() << "<nullptr>";
7270 else
7271 dbgs() << "<none>";
7272 });
7273
7274 Ty = combineTypes(T0: Ty, T1: CSTy);
7275
7276 LLVM_DEBUG({
7277 dbgs() << " : New Type: ";
7278 if (Ty && *Ty)
7279 (*Ty)->print(dbgs());
7280 else if (Ty)
7281 dbgs() << "<nullptr>";
7282 else
7283 dbgs() << "<none>";
7284 dbgs() << "\n";
7285 });
7286
7287 return !Ty || *Ty;
7288 };
7289
7290 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7291 UsedAssumedInformation))
7292 return nullptr;
7293 return Ty;
7294 }
7295
7296 ChangeStatus updateImpl(Attributor &A) override {
7297 PrivatizableType = identifyPrivatizableType(A);
7298 if (!PrivatizableType)
7299 return ChangeStatus::UNCHANGED;
7300 if (!*PrivatizableType)
7301 return indicatePessimisticFixpoint();
7302
7303 // The dependence is optional so we don't give up once we give up on the
7304 // alignment.
7305 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7306 DepClass: DepClassTy::OPTIONAL);
7307
7308 // Avoid arguments with padding for now.
7309 if (!A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal) &&
7310 !isDenselyPacked(Ty: *PrivatizableType, DL: A.getInfoCache().getDL())) {
7311 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7312 return indicatePessimisticFixpoint();
7313 }
7314
7315 // Collect the types that will replace the privatizable type in the function
7316 // signature.
7317 SmallVector<Type *, 16> ReplacementTypes;
7318 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7319
7320 // Verify callee and caller agree on how the promoted argument would be
7321 // passed.
7322 Function &Fn = *getIRPosition().getAnchorScope();
7323 const auto *TTI =
7324 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7325 if (!TTI) {
7326 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7327 << Fn.getName() << "\n");
7328 return indicatePessimisticFixpoint();
7329 }
7330
7331 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7332 CallBase *CB = ACS.getInstruction();
7333 return TTI->areTypesABICompatible(
7334 Caller: CB->getCaller(),
7335 Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7336 Types: ReplacementTypes);
7337 };
7338 bool UsedAssumedInformation = false;
7339 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7340 UsedAssumedInformation)) {
7341 LLVM_DEBUG(
7342 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7343 << Fn.getName() << "\n");
7344 return indicatePessimisticFixpoint();
7345 }
7346
7347 // Register a rewrite of the argument.
7348 Argument *Arg = getAssociatedArgument();
7349 if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7350 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7351 return indicatePessimisticFixpoint();
7352 }
7353
7354 unsigned ArgNo = Arg->getArgNo();
7355
7356 // Helper to check if for the given call site the associated argument is
7357 // passed to a callback where the privatization would be different.
7358 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7359 SmallVector<const Use *, 4> CallbackUses;
7360 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7361 for (const Use *U : CallbackUses) {
7362 AbstractCallSite CBACS(U);
7363 assert(CBACS && CBACS.isCallbackCall());
7364 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7365 int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7366
7367 LLVM_DEBUG({
7368 dbgs()
7369 << "[AAPrivatizablePtr] Argument " << *Arg
7370 << "check if can be privatized in the context of its parent ("
7371 << Arg->getParent()->getName()
7372 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7373 "callback ("
7374 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7375 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7376 << CBACS.getCallArgOperand(CBArg) << " vs "
7377 << CB.getArgOperand(ArgNo) << "\n"
7378 << "[AAPrivatizablePtr] " << CBArg << " : "
7379 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7380 });
7381
7382 if (CBArgNo != int(ArgNo))
7383 continue;
7384 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7385 QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7386 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7387 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7388 if (!CBArgPrivTy)
7389 continue;
7390 if (*CBArgPrivTy == PrivatizableType)
7391 continue;
7392 }
7393
7394 LLVM_DEBUG({
7395 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7396 << " cannot be privatized in the context of its parent ("
7397 << Arg->getParent()->getName()
7398 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7399 "callback ("
7400 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7401 << ").\n[AAPrivatizablePtr] for which the argument "
7402 "privatization is not compatible.\n";
7403 });
7404 return false;
7405 }
7406 }
7407 return true;
7408 };
7409
7410 // Helper to check if for the given call site the associated argument is
7411 // passed to a direct call where the privatization would be different.
7412 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7413 CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7414 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7415 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7416 "Expected a direct call operand for callback call operand");
7417
7418 Function *DCCallee =
7419 dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7420 LLVM_DEBUG({
7421 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7422 << " check if be privatized in the context of its parent ("
7423 << Arg->getParent()->getName()
7424 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7425 "direct call of ("
7426 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7427 });
7428
7429 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7430 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7431 QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7432 DepClass: DepClassTy::REQUIRED);
7433 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7434 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7435 if (!DCArgPrivTy)
7436 return true;
7437 if (*DCArgPrivTy == PrivatizableType)
7438 return true;
7439 }
7440 }
7441
7442 LLVM_DEBUG({
7443 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7444 << " cannot be privatized in the context of its parent ("
7445 << Arg->getParent()->getName()
7446 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7447 "direct call of ("
7448 << ACS.getInstruction()->getCalledOperand()->getName()
7449 << ").\n[AAPrivatizablePtr] for which the argument "
7450 "privatization is not compatible.\n";
7451 });
7452 return false;
7453 };
7454
7455 // Helper to check if the associated argument is used at the given abstract
7456 // call site in a way that is incompatible with the privatization assumed
7457 // here.
7458 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7459 if (ACS.isDirectCall())
7460 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7461 if (ACS.isCallbackCall())
7462 return IsCompatiblePrivArgOfDirectCS(ACS);
7463 return false;
7464 };
7465
7466 if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7467 UsedAssumedInformation))
7468 return indicatePessimisticFixpoint();
7469
7470 return ChangeStatus::UNCHANGED;
7471 }
7472
7473 /// Given a type to private \p PrivType, collect the constituates (which are
7474 /// used) in \p ReplacementTypes.
7475 static void
7476 identifyReplacementTypes(Type *PrivType,
7477 SmallVectorImpl<Type *> &ReplacementTypes) {
7478 // TODO: For now we expand the privatization type to the fullest which can
7479 // lead to dead arguments that need to be removed later.
7480 assert(PrivType && "Expected privatizable type!");
7481
7482 // Traverse the type, extract constituate types on the outermost level.
7483 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7484 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7485 ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7486 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7487 ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7488 Elt: PrivArrayType->getElementType());
7489 } else {
7490 ReplacementTypes.push_back(Elt: PrivType);
7491 }
7492 }
7493
7494 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7495 /// The values needed are taken from the arguments of \p F starting at
7496 /// position \p ArgNo.
7497 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7498 unsigned ArgNo, BasicBlock::iterator IP) {
7499 assert(PrivType && "Expected privatizable type!");
7500
7501 IRBuilder<NoFolder> IRB(IP);
7502 const DataLayout &DL = F.getDataLayout();
7503
7504 // Traverse the type, build GEPs and stores.
7505 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7506 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7507 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7508 Value *Ptr =
7509 constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7510 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7511 }
7512 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7513 Type *PointeeTy = PrivArrayType->getElementType();
7514 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7515 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7516 Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7517 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7518 }
7519 } else {
7520 new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7521 }
7522 }
7523
7524 /// Extract values from \p Base according to the type \p PrivType at the
7525 /// call position \p ACS. The values are appended to \p ReplacementValues.
7526 void createReplacementValues(Align Alignment, Type *PrivType,
7527 AbstractCallSite ACS, Value *Base,
7528 SmallVectorImpl<Value *> &ReplacementValues) {
7529 assert(Base && "Expected base value!");
7530 assert(PrivType && "Expected privatizable type!");
7531 Instruction *IP = ACS.getInstruction();
7532
7533 IRBuilder<NoFolder> IRB(IP);
7534 const DataLayout &DL = IP->getDataLayout();
7535
7536 // Traverse the type, build GEPs and loads.
7537 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7538 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7539 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7540 Type *PointeeTy = PrivStructType->getElementType(N: u);
7541 Value *Ptr =
7542 constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7543 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7544 L->setAlignment(Alignment);
7545 ReplacementValues.push_back(Elt: L);
7546 }
7547 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7548 Type *PointeeTy = PrivArrayType->getElementType();
7549 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7550 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7551 Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7552 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7553 L->setAlignment(Alignment);
7554 ReplacementValues.push_back(Elt: L);
7555 }
7556 } else {
7557 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7558 L->setAlignment(Alignment);
7559 ReplacementValues.push_back(Elt: L);
7560 }
7561 }
7562
7563 ChangeStatus manifest(Attributor &A) override {
7564 if (!PrivatizableType)
7565 return ChangeStatus::UNCHANGED;
7566 assert(*PrivatizableType && "Expected privatizable type!");
7567
7568 Argument *Arg = getAssociatedArgument();
7569 // Query AAAlign attribute for alignment of associated argument to
7570 // determine the best alignment of loads.
7571 const auto *AlignAA =
7572 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7573
7574 // Callback to repair the associated function. A new alloca is placed at the
7575 // beginning and initialized with the values passed through arguments. The
7576 // new alloca replaces the use of the old pointer argument.
7577 Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7578 [=](const Attributor::ArgumentReplacementInfo &ARI,
7579 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7580 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7581 BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7582 const DataLayout &DL = IP->getDataLayout();
7583 unsigned AS = DL.getAllocaAddrSpace();
7584 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7585 Arg->getName() + ".priv", IP);
7586 createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7587 ArgNo: ArgIt->getArgNo(), IP);
7588
7589 if (AI->getType() != Arg->getType())
7590 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7591 S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7592 Arg->replaceAllUsesWith(V: AI);
7593
7594 // The new alloca may escape into tail calls. Find them here, as calls
7595 // may have been replaced or deleted since manifest.
7596 for (Instruction &I : instructions(F&: ReplacementFn)) {
7597 auto *CI = dyn_cast<CallInst>(Val: &I);
7598 if (CI && CI->isTailCall() && !CI->isMustTailCall())
7599 CI->setTailCall(false);
7600 }
7601 };
7602
7603 // Callback to repair a call site of the associated function. The elements
7604 // of the privatizable type are loaded prior to the call and passed to the
7605 // new function version.
7606 Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7607 [=](const Attributor::ArgumentReplacementInfo &ARI,
7608 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7609 // When no alignment is specified for the load instruction,
7610 // natural alignment is assumed.
7611 createReplacementValues(
7612 Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7613 PrivType: *PrivatizableType, ACS,
7614 Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7615 ReplacementValues&: NewArgOperands);
7616 };
7617
7618 // Collect the types that will replace the privatizable type in the function
7619 // signature.
7620 SmallVector<Type *, 16> ReplacementTypes;
7621 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7622
7623 // Register a rewrite of the argument.
7624 if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7625 CalleeRepairCB: std::move(FnRepairCB),
7626 ACSRepairCB: std::move(ACSRepairCB)))
7627 return ChangeStatus::CHANGED;
7628 return ChangeStatus::UNCHANGED;
7629 }
7630
7631 void trackStatistics() const override {
7632 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7633 }
7634};
7635
7636struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7637 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7638 : AAPrivatizablePtrImpl(IRP, A) {}
7639
7640 void initialize(Attributor &A) override {
7641 // TODO: We can privatize more than arguments.
7642 indicatePessimisticFixpoint();
7643 }
7644
7645 ChangeStatus updateImpl(Attributor &A) override {
7646 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7647 "updateImpl will not be called");
7648 }
7649
7650 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7651 Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7652 if (!Obj) {
7653 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7654 return nullptr;
7655 }
7656
7657 if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7658 if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7659 if (CI->isOne())
7660 return AI->getAllocatedType();
7661 if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7662 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7663 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7664 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7665 return PrivArgAA->getPrivatizableType();
7666 }
7667
7668 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7669 "alloca nor privatizable argument: "
7670 << *Obj << "!\n");
7671 return nullptr;
7672 }
7673
7674 void trackStatistics() const override {
7675 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7676 }
7677};
7678
7679struct AAPrivatizablePtrCallSiteArgument final
7680 : public AAPrivatizablePtrFloating {
7681 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7682 : AAPrivatizablePtrFloating(IRP, A) {}
7683
7684 void initialize(Attributor &A) override {
7685 if (A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal))
7686 indicateOptimisticFixpoint();
7687 }
7688
7689 ChangeStatus updateImpl(Attributor &A) override {
7690 PrivatizableType = identifyPrivatizableType(A);
7691 if (!PrivatizableType)
7692 return ChangeStatus::UNCHANGED;
7693 if (!*PrivatizableType)
7694 return indicatePessimisticFixpoint();
7695
7696 const IRPosition &IRP = getIRPosition();
7697 bool IsKnownNoCapture;
7698 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7699 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture);
7700 if (!IsAssumedNoCapture) {
7701 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7702 return indicatePessimisticFixpoint();
7703 }
7704
7705 bool IsKnownNoAlias;
7706 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7707 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
7708 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7709 return indicatePessimisticFixpoint();
7710 }
7711
7712 bool IsKnown;
7713 if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7714 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7715 return indicatePessimisticFixpoint();
7716 }
7717
7718 return ChangeStatus::UNCHANGED;
7719 }
7720
7721 void trackStatistics() const override {
7722 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7723 }
7724};
7725
7726struct AAPrivatizablePtrCallSiteReturned final
7727 : public AAPrivatizablePtrFloating {
7728 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7729 : AAPrivatizablePtrFloating(IRP, A) {}
7730
7731 void initialize(Attributor &A) override {
7732 // TODO: We can privatize more than arguments.
7733 indicatePessimisticFixpoint();
7734 }
7735
7736 void trackStatistics() const override {
7737 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7738 }
7739};
7740
7741struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7742 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7743 : AAPrivatizablePtrFloating(IRP, A) {}
7744
7745 void initialize(Attributor &A) override {
7746 // TODO: We can privatize more than arguments.
7747 indicatePessimisticFixpoint();
7748 }
7749
7750 void trackStatistics() const override {
7751 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7752 }
7753};
7754} // namespace
7755
7756/// -------------------- Memory Behavior Attributes ----------------------------
7757/// Includes read-none, read-only, and write-only.
7758/// ----------------------------------------------------------------------------
7759namespace {
7760struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7761 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7762 : AAMemoryBehavior(IRP, A) {}
7763
7764 void initialize(Attributor &A) override {
7765 intersectAssumedBits(BitsEncoding: BEST_STATE);
7766 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7767 AAMemoryBehavior::initialize(A);
7768 }
7769
7770 /// Return the memory behavior information encoded in the IR for \p IRP.
7771 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7772 BitIntegerState &State,
7773 bool IgnoreSubsumingPositions = false) {
7774 SmallVector<Attribute, 2> Attrs;
7775 A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7776 for (const Attribute &Attr : Attrs) {
7777 switch (Attr.getKindAsEnum()) {
7778 case Attribute::ReadNone:
7779 State.addKnownBits(Bits: NO_ACCESSES);
7780 break;
7781 case Attribute::ReadOnly:
7782 State.addKnownBits(Bits: NO_WRITES);
7783 break;
7784 case Attribute::WriteOnly:
7785 State.addKnownBits(Bits: NO_READS);
7786 break;
7787 default:
7788 llvm_unreachable("Unexpected attribute!");
7789 }
7790 }
7791
7792 // Only a call site anchor performs the accesses of the position. A floating
7793 // value is only defined by its anchor, e.g., a loaded pointer may be
7794 // written through even though the load does not write.
7795 IRPosition::Kind Kind = IRP.getPositionKind();
7796 if (Kind == IRPosition::IRP_CALL_SITE ||
7797 Kind == IRPosition::IRP_CALL_SITE_ARGUMENT) {
7798 if (auto *CB = dyn_cast<CallBase>(Val: &IRP.getAnchorValue())) {
7799 ModRefInfo MR = CB->getMemoryEffects().getModRef();
7800 if (!isRefSet(MRI: MR))
7801 State.addKnownBits(Bits: NO_READS);
7802 if (!isModSet(MRI: MR))
7803 State.addKnownBits(Bits: NO_WRITES);
7804 }
7805 }
7806 }
7807
7808 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7809 SmallVectorImpl<Attribute> &Attrs) const override {
7810 assert(Attrs.size() == 0);
7811 if (isAssumedReadNone())
7812 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadNone));
7813 else if (isAssumedReadOnly())
7814 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadOnly));
7815 else if (isAssumedWriteOnly())
7816 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::WriteOnly));
7817 assert(Attrs.size() <= 1);
7818 }
7819
7820 ChangeStatus manifest(Attributor &A) override {
7821 const IRPosition &IRP = getIRPosition();
7822
7823 if (A.hasAttr(IRP, AKs: Attribute::ReadNone,
7824 /* IgnoreSubsumingPositions */ true))
7825 return ChangeStatus::UNCHANGED;
7826
7827 // Check if we would improve the existing attributes first.
7828 SmallVector<Attribute, 4> DeducedAttrs;
7829 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
7830 if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
7831 return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
7832 /* IgnoreSubsumingPositions */ true);
7833 }))
7834 return ChangeStatus::UNCHANGED;
7835
7836 // Clear existing attributes.
7837 A.removeAttrs(IRP, AttrKinds);
7838 // Clear conflicting writable attribute.
7839 if (isAssumedReadOnly())
7840 A.removeAttrs(IRP, AttrKinds: Attribute::Writable);
7841
7842 // Use the generic manifest method.
7843 return IRAttribute::manifest(A);
7844 }
7845
7846 const std::string getAsStr(Attributor *A) const override {
7847 if (isAssumedReadNone())
7848 return "readnone";
7849 if (isAssumedReadOnly())
7850 return "readonly";
7851 if (isAssumedWriteOnly())
7852 return "writeonly";
7853 return "may-read/write";
7854 }
7855
7856 /// The set of IR attributes AAMemoryBehavior deals with.
7857 static const Attribute::AttrKind AttrKinds[3];
7858};
7859
7860const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7861 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7862
7863/// Memory behavior attribute for a floating value.
7864struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7865 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7866 : AAMemoryBehaviorImpl(IRP, A) {}
7867
7868 ChangeStatus updateImpl(Attributor &A) override;
7869
7870 void trackStatistics() const override {
7871 if (isAssumedReadNone())
7872 STATS_DECLTRACK_FLOATING_ATTR(readnone)
7873 else if (isAssumedReadOnly())
7874 STATS_DECLTRACK_FLOATING_ATTR(readonly)
7875 else if (isAssumedWriteOnly())
7876 STATS_DECLTRACK_FLOATING_ATTR(writeonly)
7877 }
7878
7879private:
7880 /// Return true if users of \p UserI might access the underlying
7881 /// variable/location described by \p U and should therefore be analyzed.
7882 bool followUsersOfUseIn(Attributor &A, const Use &U,
7883 const Instruction *UserI);
7884
7885 /// Update the state according to the effect of use \p U in \p UserI.
7886 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
7887};
7888
7889/// Memory behavior attribute for function argument.
7890struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
7891 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
7892 : AAMemoryBehaviorFloating(IRP, A) {}
7893
7894 void initialize(Attributor &A) override {
7895 intersectAssumedBits(BitsEncoding: BEST_STATE);
7896 const IRPosition &IRP = getIRPosition();
7897 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
7898 // can query it when we use has/getAttr. That would allow us to reuse the
7899 // initialize of the base class here.
7900 bool HasByVal = A.hasAttr(IRP, AKs: {Attribute::ByVal},
7901 /* IgnoreSubsumingPositions */ true);
7902 getKnownStateFromValue(A, IRP, State&: getState(),
7903 /* IgnoreSubsumingPositions */ HasByVal);
7904 }
7905
7906 ChangeStatus manifest(Attributor &A) override {
7907 // TODO: Pointer arguments are not supported on vectors of pointers yet.
7908 if (!getAssociatedValue().getType()->isPointerTy())
7909 return ChangeStatus::UNCHANGED;
7910
7911 // TODO: From readattrs.ll: "inalloca parameters are always
7912 // considered written"
7913 if (A.hasAttr(IRP: getIRPosition(),
7914 AKs: {Attribute::InAlloca, Attribute::Preallocated})) {
7915 removeKnownBits(BitsEncoding: NO_WRITES);
7916 removeAssumedBits(BitsEncoding: NO_WRITES);
7917 }
7918 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
7919 return AAMemoryBehaviorFloating::manifest(A);
7920 }
7921
7922 void trackStatistics() const override {
7923 if (isAssumedReadNone())
7924 STATS_DECLTRACK_ARG_ATTR(readnone)
7925 else if (isAssumedReadOnly())
7926 STATS_DECLTRACK_ARG_ATTR(readonly)
7927 else if (isAssumedWriteOnly())
7928 STATS_DECLTRACK_ARG_ATTR(writeonly)
7929 }
7930};
7931
7932struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
7933 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
7934 : AAMemoryBehaviorArgument(IRP, A) {}
7935
7936 void initialize(Attributor &A) override {
7937 // If we don't have an associated attribute this is either a variadic call
7938 // or an indirect call, either way, nothing to do here.
7939 Argument *Arg = getAssociatedArgument();
7940 if (!Arg) {
7941 indicatePessimisticFixpoint();
7942 return;
7943 }
7944 if (Arg->hasByValAttr()) {
7945 addKnownBits(Bits: NO_WRITES);
7946 removeKnownBits(BitsEncoding: NO_READS);
7947 removeAssumedBits(BitsEncoding: NO_READS);
7948 }
7949 AAMemoryBehaviorArgument::initialize(A);
7950 if (getAssociatedFunction()->isDeclaration())
7951 indicatePessimisticFixpoint();
7952 }
7953
7954 ChangeStatus updateImpl(Attributor &A) override {
7955 // TODO: Once we have call site specific value information we can provide
7956 // call site specific liveness liveness information and then it makes
7957 // sense to specialize attributes for call sites arguments instead of
7958 // redirecting requests to the callee argument.
7959 Argument *Arg = getAssociatedArgument();
7960 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
7961 auto *ArgAA =
7962 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
7963 if (!ArgAA)
7964 return indicatePessimisticFixpoint();
7965 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
7966 }
7967
7968 void trackStatistics() const override {
7969 if (isAssumedReadNone())
7970 STATS_DECLTRACK_CSARG_ATTR(readnone)
7971 else if (isAssumedReadOnly())
7972 STATS_DECLTRACK_CSARG_ATTR(readonly)
7973 else if (isAssumedWriteOnly())
7974 STATS_DECLTRACK_CSARG_ATTR(writeonly)
7975 }
7976};
7977
7978/// Memory behavior attribute for a call site return position.
7979struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
7980 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
7981 : AAMemoryBehaviorFloating(IRP, A) {}
7982
7983 void initialize(Attributor &A) override {
7984 AAMemoryBehaviorImpl::initialize(A);
7985 }
7986 ChangeStatus manifest(Attributor &A) override {
7987 // We do not annotate returned values.
7988 return ChangeStatus::UNCHANGED;
7989 }
7990
7991 void trackStatistics() const override {}
7992};
7993
7994/// An AA to represent the memory behavior function attributes.
7995struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
7996 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
7997 : AAMemoryBehaviorImpl(IRP, A) {}
7998
7999 ChangeStatus updateImpl(Attributor &A) override;
8000
8001 ChangeStatus manifest(Attributor &A) override {
8002 // TODO: It would be better to merge this with AAMemoryLocation, so that
8003 // we could determine read/write per location. This would also have the
8004 // benefit of only one place trying to manifest the memory attribute.
8005 Function &F = cast<Function>(Val&: getAnchorValue());
8006 MemoryEffects ME = MemoryEffects::unknown();
8007 if (isAssumedReadNone())
8008 ME = MemoryEffects::none();
8009 else if (isAssumedReadOnly())
8010 ME = MemoryEffects::readOnly();
8011 else if (isAssumedWriteOnly())
8012 ME = MemoryEffects::writeOnly();
8013
8014 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8015 // Clear conflicting writable attribute.
8016 if (ME.onlyReadsMemory())
8017 for (Argument &Arg : F.args())
8018 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8019 return A.manifestAttrs(IRP: getIRPosition(),
8020 DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8021 }
8022
8023 void trackStatistics() const override {
8024 if (isAssumedReadNone())
8025 STATS_DECLTRACK_FN_ATTR(readnone)
8026 else if (isAssumedReadOnly())
8027 STATS_DECLTRACK_FN_ATTR(readonly)
8028 else if (isAssumedWriteOnly())
8029 STATS_DECLTRACK_FN_ATTR(writeonly)
8030 }
8031};
8032
8033/// AAMemoryBehavior attribute for call sites.
8034struct AAMemoryBehaviorCallSite final
8035 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8036 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8037 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8038
8039 ChangeStatus manifest(Attributor &A) override {
8040 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8041 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8042 MemoryEffects ME = MemoryEffects::unknown();
8043 if (isAssumedReadNone())
8044 ME = MemoryEffects::none();
8045 else if (isAssumedReadOnly())
8046 ME = MemoryEffects::readOnly();
8047 else if (isAssumedWriteOnly())
8048 ME = MemoryEffects::writeOnly();
8049
8050 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8051 // Clear conflicting writable attribute.
8052 if (ME.onlyReadsMemory())
8053 for (Use &U : CB.args())
8054 A.removeAttrs(IRP: IRPosition::callsite_argument(CB, ArgNo: U.getOperandNo()),
8055 AttrKinds: Attribute::Writable);
8056 return A.manifestAttrs(
8057 IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8058 }
8059
8060 void trackStatistics() const override {
8061 if (isAssumedReadNone())
8062 STATS_DECLTRACK_CS_ATTR(readnone)
8063 else if (isAssumedReadOnly())
8064 STATS_DECLTRACK_CS_ATTR(readonly)
8065 else if (isAssumedWriteOnly())
8066 STATS_DECLTRACK_CS_ATTR(writeonly)
8067 }
8068};
8069
8070ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8071
8072 // The current assumed state used to determine a change.
8073 auto AssumedState = getAssumed();
8074
8075 auto CheckRWInst = [&](Instruction &I) {
8076 // If the instruction has an own memory behavior state, use it to restrict
8077 // the local state. No further analysis is required as the other memory
8078 // state is as optimistic as it gets.
8079 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8080 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8081 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8082 if (MemBehaviorAA) {
8083 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8084 return !isAtFixpoint();
8085 }
8086 }
8087
8088 // Remove access kind modifiers if necessary.
8089 if (I.mayReadFromMemory())
8090 removeAssumedBits(BitsEncoding: NO_READS);
8091 if (I.mayWriteToMemory())
8092 removeAssumedBits(BitsEncoding: NO_WRITES);
8093 return !isAtFixpoint();
8094 };
8095
8096 bool UsedAssumedInformation = false;
8097 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8098 UsedAssumedInformation))
8099 return indicatePessimisticFixpoint();
8100
8101 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8102 : ChangeStatus::UNCHANGED;
8103}
8104
8105ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8106
8107 const IRPosition &IRP = getIRPosition();
8108 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8109 AAMemoryBehavior::StateType &S = getState();
8110
8111 // First, check the function scope. We take the known information and we avoid
8112 // work if the assumed information implies the current assumed information for
8113 // this attribute. This is a valid for all but byval arguments.
8114 Argument *Arg = IRP.getAssociatedArgument();
8115 AAMemoryBehavior::base_t FnMemAssumedState =
8116 AAMemoryBehavior::StateType::getWorstState();
8117 if (!Arg || !Arg->hasByValAttr()) {
8118 const auto *FnMemAA =
8119 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8120 if (FnMemAA) {
8121 FnMemAssumedState = FnMemAA->getAssumed();
8122 S.addKnownBits(Bits: FnMemAA->getKnown());
8123 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8124 return ChangeStatus::UNCHANGED;
8125 }
8126 }
8127
8128 // The current assumed state used to determine a change.
8129 auto AssumedState = S.getAssumed();
8130
8131 // Make sure the value is not captured (except through "return"), if
8132 // it is, any information derived would be irrelevant anyway as we cannot
8133 // check the potential aliases introduced by the capture. However, no need
8134 // to fall back to anythign less optimistic than the function state.
8135 bool IsKnownNoCapture;
8136 const AANoCapture *ArgNoCaptureAA = nullptr;
8137 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8138 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
8139 AAPtr: &ArgNoCaptureAA);
8140
8141 if (!IsAssumedNoCapture &&
8142 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8143 S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8144 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8145 : ChangeStatus::UNCHANGED;
8146 }
8147
8148 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8149 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8150 Instruction *UserI = cast<Instruction>(Val: U.getUser());
8151 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8152 << " \n");
8153
8154 // Droppable users, e.g., llvm::assume does not actually perform any action.
8155 if (UserI->isDroppable())
8156 return true;
8157
8158 // Check if the users of UserI should also be visited.
8159 Follow = followUsersOfUseIn(A, U, UserI);
8160
8161 // If UserI might touch memory we analyze the use in detail.
8162 if (UserI->mayReadOrWriteMemory())
8163 analyzeUseIn(A, U, UserI);
8164
8165 return !isAtFixpoint();
8166 };
8167
8168 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8169 return indicatePessimisticFixpoint();
8170
8171 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8172 : ChangeStatus::UNCHANGED;
8173}
8174
8175bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8176 const Instruction *UserI) {
8177 // The loaded value is unrelated to the pointer argument, no need to
8178 // follow the users of the load.
8179 if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8180 return false;
8181
8182 // By default we follow all uses assuming UserI might leak information on U,
8183 // we have special handling for call sites operands though.
8184 const auto *CB = dyn_cast<CallBase>(Val: UserI);
8185 if (!CB || !CB->isArgOperand(U: &U))
8186 return true;
8187
8188 // If the use is a call argument known not to be captured, the users of
8189 // the call do not need to be visited because they have to be unrelated to
8190 // the input. Note that this check is not trivial even though we disallow
8191 // general capturing of the underlying argument. The reason is that the
8192 // call might the argument "through return", which we allow and for which we
8193 // need to check call users.
8194 if (U.get()->getType()->isPointerTy()) {
8195 unsigned ArgNo = CB->getArgOperandNo(U: &U);
8196 bool IsKnownNoCapture;
8197 return !AA::hasAssumedIRAttr<Attribute::Captures>(
8198 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
8199 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
8200 }
8201
8202 return true;
8203}
8204
8205void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8206 const Instruction *UserI) {
8207 assert(UserI->mayReadOrWriteMemory());
8208
8209 switch (UserI->getOpcode()) {
8210 default:
8211 // TODO: Handle all atomics and other side-effect operations we know of.
8212 break;
8213 case Instruction::Load:
8214 // Loads cause the NO_READS property to disappear.
8215 removeAssumedBits(BitsEncoding: NO_READS);
8216 return;
8217
8218 case Instruction::Store:
8219 // Stores cause the NO_WRITES property to disappear if the use is the
8220 // pointer operand. Note that while capturing was taken care of somewhere
8221 // else we need to deal with stores of the value that is not looked through.
8222 if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8223 removeAssumedBits(BitsEncoding: NO_WRITES);
8224 else
8225 indicatePessimisticFixpoint();
8226 return;
8227
8228 case Instruction::Call:
8229 case Instruction::CallBr:
8230 case Instruction::Invoke: {
8231 // For call sites we look at the argument memory behavior attribute (this
8232 // could be recursive!) in order to restrict our own state.
8233 const auto *CB = cast<CallBase>(Val: UserI);
8234
8235 // Give up on operand bundles.
8236 if (CB->isBundleOperand(U: &U)) {
8237 indicatePessimisticFixpoint();
8238 return;
8239 }
8240
8241 // Calling a function does read the function pointer, maybe write it if the
8242 // function is self-modifying.
8243 if (CB->isCallee(U: &U)) {
8244 removeAssumedBits(BitsEncoding: NO_READS);
8245 break;
8246 }
8247
8248 // Adjust the possible access behavior based on the information on the
8249 // argument.
8250 IRPosition Pos;
8251 if (U.get()->getType()->isPointerTy())
8252 Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8253 else
8254 Pos = IRPosition::callsite_function(CB: *CB);
8255 const auto *MemBehaviorAA =
8256 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8257 if (!MemBehaviorAA)
8258 break;
8259 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8260 // and at least "known".
8261 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8262 return;
8263 }
8264 };
8265
8266 // Generally, look at the "may-properties" and adjust the assumed state if we
8267 // did not trigger special handling before.
8268 if (UserI->mayReadFromMemory())
8269 removeAssumedBits(BitsEncoding: NO_READS);
8270 if (UserI->mayWriteToMemory())
8271 removeAssumedBits(BitsEncoding: NO_WRITES);
8272}
8273} // namespace
8274
8275/// -------------------- Memory Locations Attributes ---------------------------
8276/// Includes read-none, argmemonly, inaccessiblememonly,
8277/// inaccessiblememorargmemonly
8278/// ----------------------------------------------------------------------------
8279
8280std::string AAMemoryLocation::getMemoryLocationsAsStr(
8281 AAMemoryLocation::MemoryLocationsKind MLK) {
8282 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8283 return "all memory";
8284 if (MLK == AAMemoryLocation::NO_LOCATIONS)
8285 return "no memory";
8286 std::string S = "memory:";
8287 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8288 S += "stack,";
8289 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8290 S += "constant,";
8291 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8292 S += "internal global,";
8293 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8294 S += "external global,";
8295 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8296 S += "argument,";
8297 if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8298 S += "inaccessible,";
8299 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8300 S += "malloced,";
8301 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8302 S += "unknown,";
8303 S.pop_back();
8304 return S;
8305}
8306
8307namespace {
8308struct AAMemoryLocationImpl : public AAMemoryLocation {
8309
8310 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8311 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8312 AccessKind2Accesses.fill(u: nullptr);
8313 }
8314
8315 ~AAMemoryLocationImpl() override {
8316 // The AccessSets are allocated via a BumpPtrAllocator, we call
8317 // the destructor manually.
8318 for (AccessSet *AS : AccessKind2Accesses)
8319 if (AS)
8320 AS->~AccessSet();
8321 }
8322
8323 void initialize(Attributor &A) override {
8324 intersectAssumedBits(BitsEncoding: BEST_STATE);
8325 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8326 AAMemoryLocation::initialize(A);
8327 }
8328
8329 /// Return the memory behavior information encoded in the IR for \p IRP.
8330 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8331 BitIntegerState &State,
8332 bool IgnoreSubsumingPositions = false) {
8333 // For internal functions we ignore `argmemonly` and
8334 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8335 // constant propagation. It is unclear if this is the best way but it is
8336 // unlikely this will cause real performance problems. If we are deriving
8337 // attributes for the anchor function we even remove the attribute in
8338 // addition to ignoring it.
8339 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8340 // MemoryEffects::Other as a possible location.
8341 bool UseArgMemOnly = true;
8342 Function *AnchorFn = IRP.getAnchorScope();
8343 if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8344 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8345
8346 SmallVector<Attribute, 2> Attrs;
8347 A.getAttrs(IRP, AKs: {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8348 for (const Attribute &Attr : Attrs) {
8349 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8350 MemoryEffects ME = Attr.getMemoryEffects();
8351 if (ME.doesNotAccessMemory()) {
8352 State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8353 continue;
8354 }
8355 if (ME.onlyAccessesInaccessibleMem()) {
8356 State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8357 continue;
8358 }
8359 if (ME.onlyAccessesArgPointees()) {
8360 if (UseArgMemOnly)
8361 State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8362 else {
8363 // Remove location information, only keep read/write info.
8364 ME = MemoryEffects(ME.getModRef());
8365 A.manifestAttrs(IRP,
8366 DeducedAttrs: Attribute::getWithMemoryEffects(
8367 Context&: IRP.getAnchorValue().getContext(), ME),
8368 /*ForceReplace*/ true);
8369 }
8370 continue;
8371 }
8372 if (ME.onlyAccessesInaccessibleOrArgMem()) {
8373 if (UseArgMemOnly)
8374 State.addKnownBits(Bits: inverseLocation(
8375 Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8376 else {
8377 // Remove location information, only keep read/write info.
8378 ME = MemoryEffects(ME.getModRef());
8379 A.manifestAttrs(IRP,
8380 DeducedAttrs: Attribute::getWithMemoryEffects(
8381 Context&: IRP.getAnchorValue().getContext(), ME),
8382 /*ForceReplace*/ true);
8383 }
8384 continue;
8385 }
8386 }
8387 }
8388
8389 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8390 SmallVectorImpl<Attribute> &Attrs) const override {
8391 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8392 assert(Attrs.size() == 0);
8393 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8394 if (isAssumedReadNone())
8395 Attrs.push_back(
8396 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8397 else if (isAssumedInaccessibleMemOnly())
8398 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8399 Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8400 else if (isAssumedArgMemOnly())
8401 Attrs.push_back(
8402 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8403 else if (isAssumedInaccessibleOrArgMemOnly())
8404 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8405 Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8406 }
8407 assert(Attrs.size() <= 1);
8408 }
8409
8410 ChangeStatus manifest(Attributor &A) override {
8411 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8412 // provide per-location modref information here.
8413 const IRPosition &IRP = getIRPosition();
8414
8415 SmallVector<Attribute, 1> DeducedAttrs;
8416 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8417 if (DeducedAttrs.size() != 1)
8418 return ChangeStatus::UNCHANGED;
8419 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8420
8421 // Clear conflicting writable attributes of the function arguments.
8422 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION &&
8423 !isModSet(MRI: ME.getModRef(Loc: IRMemLocation::ArgMem))) {
8424 for (Argument &Arg : IRP.getAnchorScope()->args())
8425 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8426 }
8427
8428 return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8429 Context&: IRP.getAnchorValue().getContext(), ME));
8430 }
8431
8432 bool checkForAllAccessesToMemoryKind(
8433 function_ref<bool(const Instruction *, const Value *, AccessKind,
8434 MemoryLocationsKind)>
8435 Pred,
8436 MemoryLocationsKind RequestedMLK) const override {
8437 if (!isValidState())
8438 return false;
8439
8440 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8441 if (AssumedMLK == NO_LOCATIONS)
8442 return true;
8443
8444 unsigned Idx = 0;
8445 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8446 CurMLK *= 2, ++Idx) {
8447 if (CurMLK & RequestedMLK)
8448 continue;
8449
8450 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8451 for (const AccessInfo &AI : *Accesses)
8452 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8453 return false;
8454 }
8455
8456 return true;
8457 }
8458
8459 ChangeStatus indicatePessimisticFixpoint() override {
8460 // If we give up and indicate a pessimistic fixpoint this instruction will
8461 // become an access for all potential access kinds:
8462 // TODO: Add pointers for argmemonly and globals to improve the results of
8463 // checkForAllAccessesToMemoryKind.
8464 bool Changed = false;
8465 MemoryLocationsKind KnownMLK = getKnown();
8466 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8467 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8468 if (!(CurMLK & KnownMLK))
8469 updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8470 AK: getAccessKindFromInst(I));
8471 return AAMemoryLocation::indicatePessimisticFixpoint();
8472 }
8473
8474protected:
8475 /// Helper struct to tie together an instruction that has a read or write
8476 /// effect with the pointer it accesses (if any).
8477 struct AccessInfo {
8478
8479 /// The instruction that caused the access.
8480 const Instruction *I;
8481
8482 /// The base pointer that is accessed, or null if unknown.
8483 const Value *Ptr;
8484
8485 /// The kind of access (read/write/read+write).
8486 AccessKind Kind;
8487
8488 bool operator==(const AccessInfo &RHS) const {
8489 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8490 }
8491 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8492 if (LHS.I != RHS.I)
8493 return LHS.I < RHS.I;
8494 if (LHS.Ptr != RHS.Ptr)
8495 return LHS.Ptr < RHS.Ptr;
8496 if (LHS.Kind != RHS.Kind)
8497 return LHS.Kind < RHS.Kind;
8498 return false;
8499 }
8500 };
8501
8502 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8503 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8504 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8505 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8506 AccessKind2Accesses;
8507
8508 /// Categorize the pointer arguments of CB that might access memory in
8509 /// AccessedLoc and update the state and access map accordingly.
8510 void
8511 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8512 AAMemoryLocation::StateType &AccessedLocs,
8513 bool &Changed);
8514
8515 /// Return the kind(s) of location that may be accessed by \p V.
8516 AAMemoryLocation::MemoryLocationsKind
8517 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8518
8519 /// Return the access kind as determined by \p I.
8520 AccessKind getAccessKindFromInst(const Instruction *I) {
8521 AccessKind AK = READ_WRITE;
8522 if (I) {
8523 AK = I->mayReadFromMemory() ? READ : NONE;
8524 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8525 }
8526 return AK;
8527 }
8528
8529 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8530 /// an access of kind \p AK to a \p MLK memory location with the access
8531 /// pointer \p Ptr.
8532 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8533 MemoryLocationsKind MLK, const Instruction *I,
8534 const Value *Ptr, bool &Changed,
8535 AccessKind AK = READ_WRITE) {
8536
8537 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8538 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8539 if (!Accesses)
8540 Accesses = new (Allocator) AccessSet();
8541 Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8542 if (MLK == NO_UNKOWN_MEM)
8543 MLK = NO_LOCATIONS;
8544 State.removeAssumedBits(BitsEncoding: MLK);
8545 }
8546
8547 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8548 /// arguments, and update the state and access map accordingly.
8549 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8550 AAMemoryLocation::StateType &State, bool &Changed,
8551 unsigned AccessAS = 0);
8552
8553 /// Used to allocate access sets.
8554 BumpPtrAllocator &Allocator;
8555};
8556
8557void AAMemoryLocationImpl::categorizePtrValue(
8558 Attributor &A, const Instruction &I, const Value &Ptr,
8559 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8560 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8561 << Ptr << " ["
8562 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8563
8564 auto Pred = [&](Value &Obj) {
8565 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8566 // TODO: recognize the TBAA used for constant accesses.
8567 MemoryLocationsKind MLK = NO_LOCATIONS;
8568
8569 // Filter accesses to constant (GPU) memory if we have an AS at the access
8570 // site or the object is known to actually have the associated AS.
8571 if (AA::isGPU(M: A.getModule())) {
8572 if (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: AccessAS) ||
8573 (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: ObjectAS) &&
8574 isIdentifiedObject(V: &Obj)))
8575 return true;
8576 }
8577
8578 if (isa<UndefValue>(Val: &Obj))
8579 return true;
8580 if (isa<Argument>(Val: &Obj)) {
8581 // TODO: For now we do not treat byval arguments as local copies performed
8582 // on the call edge, though, we should. To make that happen we need to
8583 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8584 // would also allow us to mark functions only accessing byval arguments as
8585 // readnone again, arguably their accesses have no effect outside of the
8586 // function, like accesses to allocas.
8587 MLK = NO_ARGUMENT_MEM;
8588 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8589 // Reading constant memory is not treated as a read "effect" by the
8590 // function attr pass so we won't neither. Constants defined by TBAA are
8591 // similar. (We know we do not write it because it is constant.)
8592 if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8593 if (GVar->isConstant())
8594 return true;
8595
8596 if (GV->hasLocalLinkage())
8597 MLK = NO_GLOBAL_INTERNAL_MEM;
8598 else
8599 MLK = NO_GLOBAL_EXTERNAL_MEM;
8600 } else if (isa<ConstantPointerNull>(Val: &Obj) && ObjectAS == AccessAS &&
8601 !NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS)) {
8602 // Accessing null is undefined, but not accessing a null pointer cast to
8603 // another address space, which may be a valid address there.
8604 return true;
8605 } else if (isa<AllocaInst>(Val: &Obj)) {
8606 MLK = NO_LOCAL_MEM;
8607 } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8608 bool IsKnownNoAlias;
8609 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8610 A, QueryingAA: this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::OPTIONAL,
8611 IsKnown&: IsKnownNoAlias))
8612 MLK = NO_MALLOCED_MEM;
8613 else
8614 MLK = NO_UNKOWN_MEM;
8615 } else {
8616 MLK = NO_UNKOWN_MEM;
8617 }
8618
8619 assert(MLK != NO_LOCATIONS && "No location specified!");
8620 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8621 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8622 updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8623 AK: getAccessKindFromInst(I: &I));
8624
8625 return true;
8626 };
8627
8628 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8629 QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8630 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8631 LLVM_DEBUG(
8632 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8633 updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8634 AK: getAccessKindFromInst(I: &I));
8635 return;
8636 }
8637
8638 LLVM_DEBUG(
8639 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8640 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8641}
8642
8643void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8644 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8645 bool &Changed) {
8646 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8647
8648 // Skip non-pointer arguments.
8649 const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8650 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8651 continue;
8652
8653 // Skip readnone arguments.
8654 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8655 const auto *ArgOpMemLocationAA =
8656 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8657
8658 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8659 continue;
8660
8661 // Categorize potentially accessed pointer arguments as if there was an
8662 // access instruction with them as pointer.
8663 categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8664 }
8665}
8666
8667AAMemoryLocation::MemoryLocationsKind
8668AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8669 bool &Changed) {
8670 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8671 << I << "\n");
8672
8673 AAMemoryLocation::StateType AccessedLocs;
8674 AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8675
8676 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8677
8678 // First check if we assume any memory is access is visible.
8679 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8680 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8681 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8682 << " [" << CBMemLocationAA << "]\n");
8683 if (!CBMemLocationAA) {
8684 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8685 Changed, AK: getAccessKindFromInst(I: &I));
8686 return NO_UNKOWN_MEM;
8687 }
8688
8689 if (CBMemLocationAA->isAssumedReadNone())
8690 return NO_LOCATIONS;
8691
8692 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8693 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8694 Changed, AK: getAccessKindFromInst(I: &I));
8695 return AccessedLocs.getAssumed();
8696 }
8697
8698 uint32_t CBAssumedNotAccessedLocs =
8699 CBMemLocationAA->getAssumedNotAccessedLocation();
8700
8701 // Set the argmemonly and global bit as we handle them separately below.
8702 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8703 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8704
8705 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8706 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8707 continue;
8708 updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8709 AK: getAccessKindFromInst(I: &I));
8710 }
8711
8712 // Now handle global memory if it might be accessed. This is slightly tricky
8713 // as NO_GLOBAL_MEM has multiple bits set.
8714 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8715 if (HasGlobalAccesses) {
8716 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8717 AccessKind Kind, MemoryLocationsKind MLK) {
8718 updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8719 AK: getAccessKindFromInst(I: &I));
8720 return true;
8721 };
8722 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8723 Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8724 return AccessedLocs.getWorstState();
8725 }
8726
8727 LLVM_DEBUG(
8728 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8729 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8730
8731 // Now handle argument memory if it might be accessed.
8732 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8733 if (HasArgAccesses)
8734 categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8735
8736 LLVM_DEBUG(
8737 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8738 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8739
8740 return AccessedLocs.getAssumed();
8741 }
8742
8743 if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8744 LLVM_DEBUG(
8745 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8746 << I << " [" << *Ptr << "]\n");
8747 categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8748 AccessAS: Ptr->getType()->getPointerAddressSpace());
8749 return AccessedLocs.getAssumed();
8750 }
8751
8752 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8753 << I << "\n");
8754 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8755 AK: getAccessKindFromInst(I: &I));
8756 return AccessedLocs.getAssumed();
8757}
8758
8759/// An AA to represent the memory behavior function attributes.
8760struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8761 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8762 : AAMemoryLocationImpl(IRP, A) {}
8763
8764 ChangeStatus updateImpl(Attributor &A) override {
8765
8766 const auto *MemBehaviorAA =
8767 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8768 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8769 if (MemBehaviorAA->isKnownReadNone())
8770 return indicateOptimisticFixpoint();
8771 assert(isAssumedReadNone() &&
8772 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8773 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8774 return ChangeStatus::UNCHANGED;
8775 }
8776
8777 // The current assumed state used to determine a change.
8778 auto AssumedState = getAssumed();
8779 bool Changed = false;
8780
8781 auto CheckRWInst = [&](Instruction &I) {
8782 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8783 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8784 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8785 removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8786 // Stop once only the valid bit set in the *not assumed location*, thus
8787 // once we don't actually exclude any memory locations in the state.
8788 return getAssumedNotAccessedLocation() != VALID_STATE;
8789 };
8790
8791 bool UsedAssumedInformation = false;
8792 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8793 UsedAssumedInformation))
8794 return indicatePessimisticFixpoint();
8795
8796 Changed |= AssumedState != getAssumed();
8797 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8798 }
8799
8800 void trackStatistics() const override {
8801 if (isAssumedReadNone())
8802 STATS_DECLTRACK_FN_ATTR(readnone)
8803 else if (isAssumedArgMemOnly())
8804 STATS_DECLTRACK_FN_ATTR(argmemonly)
8805 else if (isAssumedInaccessibleMemOnly())
8806 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8807 else if (isAssumedInaccessibleOrArgMemOnly())
8808 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8809 }
8810};
8811
8812/// AAMemoryLocation attribute for call sites.
8813struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8814 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8815 : AAMemoryLocationImpl(IRP, A) {}
8816
8817 void initialize(Attributor &A) override {
8818 AAMemoryLocationImpl::initialize(A);
8819
8820 // Volatile calls may also access inaccessible memory, which the callee's
8821 // attributes do not say.
8822 auto &CB = cast<CallBase>(Val&: getAnchorValue());
8823 if (CB.isVolatile()) {
8824 bool Changed = false;
8825 removeKnownBits(BitsEncoding: NO_INACCESSIBLE_MEM);
8826 updateStateAndAccessesMap(State&: getState(), MLK: NO_INACCESSIBLE_MEM, I: &CB, Ptr: nullptr,
8827 Changed, AK: getAccessKindFromInst(I: &CB));
8828 }
8829 }
8830
8831 ChangeStatus updateImpl(Attributor &A) override {
8832 // TODO: Once we have call site specific value information we can provide
8833 // call site specific liveness liveness information and then it makes
8834 // sense to specialize attributes for call sites arguments instead of
8835 // redirecting requests to the callee argument.
8836 Function *F = getAssociatedFunction();
8837 const IRPosition &FnPos = IRPosition::function(F: *F);
8838 auto *FnAA =
8839 A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
8840 if (!FnAA)
8841 return indicatePessimisticFixpoint();
8842 bool Changed = false;
8843 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8844 AccessKind Kind, MemoryLocationsKind MLK) {
8845 updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
8846 AK: getAccessKindFromInst(I));
8847 return true;
8848 };
8849 if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
8850 return indicatePessimisticFixpoint();
8851 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8852 }
8853
8854 void trackStatistics() const override {
8855 if (isAssumedReadNone())
8856 STATS_DECLTRACK_CS_ATTR(readnone)
8857 }
8858};
8859} // namespace
8860
8861/// ------------------ denormal-fp-math Attribute -------------------------
8862
8863namespace {
8864struct AADenormalFPMathImpl : public AADenormalFPMath {
8865 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8866 : AADenormalFPMath(IRP, A) {}
8867
8868 const std::string getAsStr(Attributor *A) const override {
8869 std::string Str("AADenormalFPMath[");
8870 raw_string_ostream OS(Str);
8871
8872 DenormalState Known = getKnown();
8873 if (Known.Mode.isValid())
8874 OS << "denormal-fp-math=" << Known.Mode;
8875 else
8876 OS << "invalid";
8877
8878 if (Known.ModeF32.isValid())
8879 OS << " denormal-fp-math-f32=" << Known.ModeF32;
8880 OS << ']';
8881 return Str;
8882 }
8883};
8884
8885struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
8886 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
8887 : AADenormalFPMathImpl(IRP, A) {}
8888
8889 void initialize(Attributor &A) override {
8890 const Function *F = getAnchorScope();
8891 DenormalFPEnv DenormEnv = F->getDenormalFPEnv();
8892
8893 Known = DenormalState{.Mode: DenormEnv.DefaultMode, .ModeF32: DenormEnv.F32Mode};
8894 if (isModeFixed())
8895 indicateFixpoint();
8896 }
8897
8898 ChangeStatus updateImpl(Attributor &A) override {
8899 ChangeStatus Change = ChangeStatus::UNCHANGED;
8900
8901 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
8902 Function *Caller = CS.getInstruction()->getFunction();
8903 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
8904 << "->" << getAssociatedFunction()->getName() << '\n');
8905
8906 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
8907 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
8908 if (!CallerInfo)
8909 return false;
8910
8911 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
8912 R: CallerInfo->getState());
8913 return true;
8914 };
8915
8916 bool AllCallSitesKnown = true;
8917 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
8918 return indicatePessimisticFixpoint();
8919
8920 if (Change == ChangeStatus::CHANGED && isModeFixed())
8921 indicateFixpoint();
8922 return Change;
8923 }
8924
8925 ChangeStatus manifest(Attributor &A) override {
8926 LLVMContext &Ctx = getAssociatedFunction()->getContext();
8927
8928 SmallVector<Attribute, 2> AttrToAdd;
8929 SmallVector<Attribute::AttrKind, 2> AttrToRemove;
8930
8931 // TODO: Change to use DenormalFPEnv everywhere.
8932 DenormalFPEnv KnownEnv(Known.Mode, Known.ModeF32);
8933
8934 if (KnownEnv == DenormalFPEnv::getDefault()) {
8935 AttrToRemove.push_back(Elt: Attribute::DenormalFPEnv);
8936 } else {
8937 AttrToAdd.push_back(Elt: Attribute::get(
8938 Context&: Ctx, Kind: Attribute::DenormalFPEnv,
8939 Val: DenormalFPEnv(Known.Mode, Known.ModeF32).toIntValue()));
8940 }
8941
8942 auto &IRP = getIRPosition();
8943
8944 // TODO: There should be a combined add and remove API.
8945 return A.removeAttrs(IRP, AttrKinds: AttrToRemove) |
8946 A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
8947 }
8948
8949 void trackStatistics() const override {
8950 STATS_DECLTRACK_FN_ATTR(denormal_fpenv)
8951 }
8952};
8953} // namespace
8954
8955/// ------------------ Value Constant Range Attribute -------------------------
8956
8957namespace {
8958struct AAValueConstantRangeImpl : AAValueConstantRange {
8959 using StateType = IntegerRangeState;
8960 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
8961 : AAValueConstantRange(IRP, A) {}
8962
8963 void initialize(Attributor &A) override {
8964 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
8965 indicatePessimisticFixpoint();
8966 return;
8967 }
8968
8969 // Intersect a range given by SCEV.
8970 intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
8971
8972 // Intersect a range given by LVI.
8973 intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
8974 }
8975
8976 const std::string getAsStr(Attributor *A) const override {
8977 std::string Str;
8978 llvm::raw_string_ostream OS(Str);
8979 OS << "range(" << getBitWidth() << ")<";
8980 getKnown().print(OS);
8981 OS << " / ";
8982 getAssumed().print(OS);
8983 OS << ">";
8984 return Str;
8985 }
8986
8987 /// Helper function to get a SCEV expr for the associated value at program
8988 /// point \p I.
8989 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
8990 if (!getAnchorScope())
8991 return nullptr;
8992
8993 ScalarEvolution *SE =
8994 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
8995 F: *getAnchorScope());
8996
8997 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
8998 F: *getAnchorScope());
8999
9000 if (!SE || !LI)
9001 return nullptr;
9002
9003 const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
9004 if (!I)
9005 return S;
9006
9007 return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
9008 }
9009
9010 /// Helper function to get a range from SCEV for the associated value at
9011 /// program point \p I.
9012 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9013 const Instruction *I = nullptr) const {
9014 if (!getAnchorScope())
9015 return getWorstState(BitWidth: getBitWidth());
9016
9017 ScalarEvolution *SE =
9018 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9019 F: *getAnchorScope());
9020
9021 const SCEV *S = getSCEV(A, I);
9022 if (!SE || !S)
9023 return getWorstState(BitWidth: getBitWidth());
9024
9025 return SE->getUnsignedRange(S);
9026 }
9027
9028 /// Helper function to get a range from LVI for the associated value at
9029 /// program point \p I.
9030 ConstantRange
9031 getConstantRangeFromLVI(Attributor &A,
9032 const Instruction *CtxI = nullptr) const {
9033 if (!getAnchorScope())
9034 return getWorstState(BitWidth: getBitWidth());
9035
9036 LazyValueInfo *LVI =
9037 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9038 F: *getAnchorScope());
9039
9040 if (!LVI || !CtxI)
9041 return getWorstState(BitWidth: getBitWidth());
9042 return LVI->getConstantRange(V: &getAssociatedValue(),
9043 CtxI: const_cast<Instruction *>(CtxI),
9044 /*UndefAllowed*/ false);
9045 }
9046
9047 /// Return true if \p CtxI is valid for querying outside analyses.
9048 /// This basically makes sure we do not ask intra-procedural analysis
9049 /// about a context in the wrong function or a context that violates
9050 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9051 /// if the original context of this AA is OK or should be considered invalid.
9052 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9053 const Instruction *CtxI,
9054 bool AllowAACtxI) const {
9055 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9056 return false;
9057
9058 // Our context might be in a different function, neither intra-procedural
9059 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9060 if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9061 return false;
9062
9063 // If the context is not dominated by the value there are paths to the
9064 // context that do not define the value. This cannot be handled by
9065 // LazyValueInfo so we need to bail.
9066 if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9067 InformationCache &InfoCache = A.getInfoCache();
9068 const DominatorTree *DT =
9069 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9070 F: *I->getFunction());
9071 return DT && DT->dominates(Def: I, User: CtxI);
9072 }
9073
9074 return true;
9075 }
9076
9077 ConstantRange
9078 getAssumedConstantRange(Attributor &A,
9079 const Instruction *CtxI = nullptr) const override {
9080 // TODO: Make SCEV use Attributor assumption.
9081 // We may be able to bound a variable range via assumptions in
9082 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9083 // evolve to x^2 + x, then we can say that y is in [2, 12].
9084 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9085 /* AllowAACtxI */ false))
9086 return getAssumed();
9087
9088 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9089 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9090 return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9091 }
9092
9093 /// Helper function to create MDNode for range metadata.
9094 static MDNode *
9095 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9096 const ConstantRange &AssumedConstantRange) {
9097 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9098 Ty, V: AssumedConstantRange.getLower())),
9099 ConstantAsMetadata::get(C: ConstantInt::get(
9100 Ty, V: AssumedConstantRange.getUpper()))};
9101 return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9102 }
9103
9104 /// Return true if \p Assumed is included in ranges from instruction \p I.
9105 static bool isBetterRange(const ConstantRange &Assumed,
9106 const Instruction &I) {
9107 if (Assumed.isFullSet())
9108 return false;
9109
9110 std::optional<ConstantRange> Known;
9111
9112 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
9113 Known = CB->getRange();
9114 } else if (MDNode *KnownRanges = I.getMetadata(KindID: LLVMContext::MD_range)) {
9115 // If multiple ranges are annotated in IR, we give up to annotate assumed
9116 // range for now.
9117
9118 // TODO: If there exists a known range which containts assumed range, we
9119 // can say assumed range is better.
9120 if (KnownRanges->getNumOperands() > 2)
9121 return false;
9122
9123 ConstantInt *Lower =
9124 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9125 ConstantInt *Upper =
9126 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9127
9128 Known.emplace(args: Lower->getValue(), args: Upper->getValue());
9129 }
9130 return !Known || (*Known != Assumed && Known->contains(CR: Assumed));
9131 }
9132
9133 /// Helper function to set range metadata.
9134 static bool
9135 setRangeMetadataIfisBetterRange(Instruction *I,
9136 const ConstantRange &AssumedConstantRange) {
9137 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9138 I->setMetadata(KindID: LLVMContext::MD_range,
9139 Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9140 AssumedConstantRange));
9141 return true;
9142 }
9143 return false;
9144 }
9145 /// Helper function to set range return attribute.
9146 static bool
9147 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9148 Instruction *I,
9149 const ConstantRange &AssumedConstantRange) {
9150 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9151 A.manifestAttrs(IRP,
9152 DeducedAttrs: Attribute::get(Context&: I->getContext(), Kind: Attribute::Range,
9153 CR: AssumedConstantRange),
9154 /*ForceReplace*/ true);
9155 return true;
9156 }
9157 return false;
9158 }
9159
9160 ChangeStatus manifest(Attributor &A) override {
9161 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9162 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9163 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9164
9165 auto &V = getAssociatedValue();
9166 if (!AssumedConstantRange.isEmptySet() &&
9167 !AssumedConstantRange.isSingleElement()) {
9168 if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9169 assert(I == getCtxI() && "Should not annotate an instruction which is "
9170 "not the context instruction");
9171 if (isa<LoadInst>(Val: I))
9172 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9173 Changed = ChangeStatus::CHANGED;
9174 if (isa<CallInst>(Val: I))
9175 if (setRangeRetAttrIfisBetterRange(A, IRP: getIRPosition(), I,
9176 AssumedConstantRange))
9177 Changed = ChangeStatus::CHANGED;
9178 }
9179 }
9180
9181 return Changed;
9182 }
9183};
9184
9185struct AAValueConstantRangeArgument final
9186 : AAArgumentFromCallSiteArguments<
9187 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9188 true /* BridgeCallBaseContext */> {
9189 using Base = AAArgumentFromCallSiteArguments<
9190 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9191 true /* BridgeCallBaseContext */>;
9192 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9193 : Base(IRP, A) {}
9194
9195 void trackStatistics() const override {
9196 STATS_DECLTRACK_ARG_ATTR(value_range)
9197 }
9198};
9199
9200struct AAValueConstantRangeReturned
9201 : AAReturnedFromReturnedValues<AAValueConstantRange,
9202 AAValueConstantRangeImpl,
9203 AAValueConstantRangeImpl::StateType,
9204 /* PropagateCallBaseContext */ true> {
9205 using Base =
9206 AAReturnedFromReturnedValues<AAValueConstantRange,
9207 AAValueConstantRangeImpl,
9208 AAValueConstantRangeImpl::StateType,
9209 /* PropagateCallBaseContext */ true>;
9210 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9211 : Base(IRP, A) {}
9212
9213 void initialize(Attributor &A) override {
9214 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9215 indicatePessimisticFixpoint();
9216 }
9217
9218 void trackStatistics() const override {
9219 STATS_DECLTRACK_FNRET_ATTR(value_range)
9220 }
9221};
9222
9223struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9224 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9225 : AAValueConstantRangeImpl(IRP, A) {}
9226
9227 void initialize(Attributor &A) override {
9228 AAValueConstantRangeImpl::initialize(A);
9229 if (isAtFixpoint())
9230 return;
9231
9232 Value &V = getAssociatedValue();
9233
9234 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9235 unionAssumed(R: ConstantRange(C->getValue()));
9236 indicateOptimisticFixpoint();
9237 return;
9238 }
9239
9240 if (isa<UndefValue>(Val: &V)) {
9241 // Collapse the undef state to 0.
9242 unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9243 indicateOptimisticFixpoint();
9244 return;
9245 }
9246
9247 if (isa<CallBase>(Val: &V))
9248 return;
9249
9250 if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9251 return;
9252
9253 // If it is a load instruction with range metadata, use it.
9254 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9255 if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9256 intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9257 return;
9258 }
9259
9260 // We can work with PHI and select instruction as we traverse their operands
9261 // during update.
9262 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9263 return;
9264
9265 // Otherwise we give up.
9266 indicatePessimisticFixpoint();
9267
9268 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9269 << getAssociatedValue() << "\n");
9270 }
9271
9272 bool calculateBinaryOperator(
9273 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9274 const Instruction *CtxI,
9275 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9276 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9277 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9278
9279 // Simplify the operands first.
9280 bool UsedAssumedInformation = false;
9281 const auto &SimplifiedLHS = A.getAssumedSimplified(
9282 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9283 UsedAssumedInformation, S: AA::Interprocedural);
9284 if (!SimplifiedLHS.has_value())
9285 return true;
9286 if (!*SimplifiedLHS)
9287 return false;
9288 LHS = *SimplifiedLHS;
9289
9290 const auto &SimplifiedRHS = A.getAssumedSimplified(
9291 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9292 UsedAssumedInformation, S: AA::Interprocedural);
9293 if (!SimplifiedRHS.has_value())
9294 return true;
9295 if (!*SimplifiedRHS)
9296 return false;
9297 RHS = *SimplifiedRHS;
9298
9299 // TODO: Allow non integers as well.
9300 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9301 return false;
9302
9303 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9304 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9305 DepClass: DepClassTy::REQUIRED);
9306 if (!LHSAA)
9307 return false;
9308 QuerriedAAs.push_back(Elt: LHSAA);
9309 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9310
9311 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9312 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9313 DepClass: DepClassTy::REQUIRED);
9314 if (!RHSAA)
9315 return false;
9316 QuerriedAAs.push_back(Elt: RHSAA);
9317 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9318
9319 auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9320
9321 T.unionAssumed(R: AssumedRange);
9322
9323 // TODO: Track a known state too.
9324
9325 return T.isValidState();
9326 }
9327
9328 bool calculateCastInst(
9329 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9330 const Instruction *CtxI,
9331 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9332 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9333 // TODO: Allow non integers as well.
9334 Value *OpV = CastI->getOperand(i_nocapture: 0);
9335
9336 // Simplify the operand first.
9337 bool UsedAssumedInformation = false;
9338 const auto &SimplifiedOpV = A.getAssumedSimplified(
9339 IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9340 UsedAssumedInformation, S: AA::Interprocedural);
9341 if (!SimplifiedOpV.has_value())
9342 return true;
9343 if (!*SimplifiedOpV)
9344 return false;
9345 OpV = *SimplifiedOpV;
9346
9347 if (!OpV->getType()->isIntegerTy())
9348 return false;
9349
9350 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9351 QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9352 DepClass: DepClassTy::REQUIRED);
9353 if (!OpAA)
9354 return false;
9355 QuerriedAAs.push_back(Elt: OpAA);
9356 T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9357 BitWidth: getState().getBitWidth()));
9358 return T.isValidState();
9359 }
9360
9361 bool
9362 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9363 const Instruction *CtxI,
9364 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9365 Value *LHS = CmpI->getOperand(i_nocapture: 0);
9366 Value *RHS = CmpI->getOperand(i_nocapture: 1);
9367
9368 // Simplify the operands first.
9369 bool UsedAssumedInformation = false;
9370 const auto &SimplifiedLHS = A.getAssumedSimplified(
9371 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9372 UsedAssumedInformation, S: AA::Interprocedural);
9373 if (!SimplifiedLHS.has_value())
9374 return true;
9375 if (!*SimplifiedLHS)
9376 return false;
9377 LHS = *SimplifiedLHS;
9378
9379 const auto &SimplifiedRHS = A.getAssumedSimplified(
9380 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9381 UsedAssumedInformation, S: AA::Interprocedural);
9382 if (!SimplifiedRHS.has_value())
9383 return true;
9384 if (!*SimplifiedRHS)
9385 return false;
9386 RHS = *SimplifiedRHS;
9387
9388 // TODO: Allow non integers as well.
9389 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9390 return false;
9391
9392 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9393 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9394 DepClass: DepClassTy::REQUIRED);
9395 if (!LHSAA)
9396 return false;
9397 QuerriedAAs.push_back(Elt: LHSAA);
9398 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9399 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9400 DepClass: DepClassTy::REQUIRED);
9401 if (!RHSAA)
9402 return false;
9403 QuerriedAAs.push_back(Elt: RHSAA);
9404 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9405 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9406
9407 // If one of them is empty set, we can't decide.
9408 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9409 return true;
9410
9411 bool MustTrue = false, MustFalse = false;
9412
9413 auto AllowedRegion =
9414 ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9415
9416 if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9417 MustFalse = true;
9418
9419 if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9420 MustTrue = true;
9421
9422 assert((!MustTrue || !MustFalse) &&
9423 "Either MustTrue or MustFalse should be false!");
9424
9425 if (MustTrue)
9426 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9427 else if (MustFalse)
9428 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9429 else
9430 T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9431
9432 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9433 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9434 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9435 << *RHSAA);
9436
9437 // TODO: Track a known state too.
9438 return T.isValidState();
9439 }
9440
9441 ChangeStatus updateImpl(Attributor &A) override {
9442
9443 IntegerRangeState T(getBitWidth());
9444 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9445 Instruction *I = dyn_cast<Instruction>(Val: &V);
9446 if (!I || isa<CallBase>(Val: I)) {
9447
9448 // Simplify the operand first.
9449 bool UsedAssumedInformation = false;
9450 const auto &SimplifiedOpV = A.getAssumedSimplified(
9451 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9452 UsedAssumedInformation, S: AA::Interprocedural);
9453 if (!SimplifiedOpV.has_value())
9454 return true;
9455 if (!*SimplifiedOpV)
9456 return false;
9457 Value *VPtr = *SimplifiedOpV;
9458
9459 // If the value is not instruction, we query AA to Attributor.
9460 const auto *AA = A.getAAFor<AAValueConstantRange>(
9461 QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9462 DepClass: DepClassTy::REQUIRED);
9463
9464 // Clamp operator is not used to utilize a program point CtxI.
9465 if (AA)
9466 T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9467 else
9468 return false;
9469
9470 return T.isValidState();
9471 }
9472
9473 SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9474 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9475 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9476 return false;
9477 } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9478 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9479 return false;
9480 } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9481 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9482 return false;
9483 } else {
9484 // Give up with other instructions.
9485 // TODO: Add other instructions
9486
9487 T.indicatePessimisticFixpoint();
9488 return false;
9489 }
9490
9491 // Catch circular reasoning in a pessimistic way for now.
9492 // TODO: Check how the range evolves and if we stripped anything, see also
9493 // AADereferenceable or AAAlign for similar situations.
9494 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9495 if (QueriedAA != this)
9496 continue;
9497 // If we are in a stady state we do not need to worry.
9498 if (T.getAssumed() == getState().getAssumed())
9499 continue;
9500 T.indicatePessimisticFixpoint();
9501 }
9502
9503 return T.isValidState();
9504 };
9505
9506 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9507 return indicatePessimisticFixpoint();
9508
9509 // Ensure that long def-use chains can't cause circular reasoning either by
9510 // introducing a cutoff below.
9511 if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9512 return ChangeStatus::UNCHANGED;
9513 if (++NumChanges > MaxNumChanges) {
9514 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9515 << " but only " << MaxNumChanges
9516 << " are allowed to avoid cyclic reasoning.");
9517 return indicatePessimisticFixpoint();
9518 }
9519 return ChangeStatus::CHANGED;
9520 }
9521
9522 void trackStatistics() const override {
9523 STATS_DECLTRACK_FLOATING_ATTR(value_range)
9524 }
9525
9526 /// Tracker to bail after too many widening steps of the constant range.
9527 int NumChanges = 0;
9528
9529 /// Upper bound for the number of allowed changes (=widening steps) for the
9530 /// constant range before we give up.
9531 static constexpr int MaxNumChanges = 5;
9532};
9533
9534struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9535 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9536 : AAValueConstantRangeImpl(IRP, A) {}
9537
9538 ChangeStatus updateImpl(Attributor &A) override {
9539 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9540 "not be called");
9541 }
9542
9543 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9544};
9545
9546struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9547 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9548 : AAValueConstantRangeFunction(IRP, A) {}
9549
9550 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9551};
9552
9553struct AAValueConstantRangeCallSiteReturned
9554 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9555 AAValueConstantRangeImpl::StateType,
9556 /* IntroduceCallBaseContext */ true> {
9557 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9558 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9559 AAValueConstantRangeImpl::StateType,
9560 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9561
9562 void initialize(Attributor &A) override {
9563 // If it is a call instruction with range attribute, use the range.
9564 if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) {
9565 if (std::optional<ConstantRange> Range = CI->getRange())
9566 intersectKnown(R: *Range);
9567 }
9568
9569 AAValueConstantRangeImpl::initialize(A);
9570 }
9571
9572 void trackStatistics() const override {
9573 STATS_DECLTRACK_CSRET_ATTR(value_range)
9574 }
9575};
9576struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9577 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9578 : AAValueConstantRangeFloating(IRP, A) {}
9579
9580 ChangeStatus manifest(Attributor &A) override {
9581 return ChangeStatus::UNCHANGED;
9582 }
9583
9584 void trackStatistics() const override {
9585 STATS_DECLTRACK_CSARG_ATTR(value_range)
9586 }
9587};
9588} // namespace
9589
9590/// ------------------ Potential Values Attribute -------------------------
9591
9592AAPotentialConstantValues::AAPotentialConstantValues(const IRPosition &IRP,
9593 Attributor &A)
9594 : Base(IRP, true, A.Opts.attributor_max_potential_values) {}
9595
9596namespace {
9597struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9598 using StateType = PotentialConstantIntValuesState;
9599
9600 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9601 : AAPotentialConstantValues(IRP, A) {}
9602
9603 void initialize(Attributor &A) override {
9604 if (A.hasSimplificationCallback(IRP: getIRPosition()))
9605 indicatePessimisticFixpoint();
9606 else
9607 AAPotentialConstantValues::initialize(A);
9608 }
9609
9610 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9611 bool &ContainsUndef, bool ForSelf) {
9612 SmallVector<AA::ValueAndContext> Values;
9613 bool UsedAssumedInformation = false;
9614 if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9615 UsedAssumedInformation)) {
9616 // Avoid recursion when the caller is computing constant values for this
9617 // IRP itself.
9618 if (ForSelf)
9619 return false;
9620 if (!IRP.getAssociatedType()->isIntegerTy())
9621 return false;
9622 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9623 QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9624 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9625 return false;
9626 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9627 S = PotentialValuesAA->getState().getAssumedSet();
9628 return true;
9629 }
9630
9631 // Copy all the constant values, except UndefValue. ContainsUndef is true
9632 // iff Values contains only UndefValue instances. If there are other known
9633 // constants, then UndefValue is dropped.
9634 ContainsUndef = false;
9635 for (auto &It : Values) {
9636 if (isa<UndefValue>(Val: It.getValue())) {
9637 ContainsUndef = true;
9638 continue;
9639 }
9640 auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9641 if (!CI)
9642 return false;
9643 S.insert(X: CI->getValue());
9644 }
9645 ContainsUndef &= S.empty();
9646
9647 return true;
9648 }
9649
9650 const std::string getAsStr(Attributor *A) const override {
9651 std::string Str;
9652 llvm::raw_string_ostream OS(Str);
9653 OS << getState();
9654 return Str;
9655 }
9656
9657 ChangeStatus updateImpl(Attributor &A) override {
9658 return indicatePessimisticFixpoint();
9659 }
9660};
9661
9662struct AAPotentialConstantValuesArgument final
9663 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9664 AAPotentialConstantValuesImpl,
9665 PotentialConstantIntValuesState> {
9666 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9667 AAPotentialConstantValuesImpl,
9668 PotentialConstantIntValuesState>;
9669 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9670 : Base(IRP, A) {}
9671
9672 void trackStatistics() const override {
9673 STATS_DECLTRACK_ARG_ATTR(potential_values)
9674 }
9675};
9676
9677struct AAPotentialConstantValuesReturned
9678 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9679 AAPotentialConstantValuesImpl> {
9680 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9681 AAPotentialConstantValuesImpl>;
9682 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9683 : Base(IRP, A) {}
9684
9685 void initialize(Attributor &A) override {
9686 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9687 indicatePessimisticFixpoint();
9688 Base::initialize(A);
9689 }
9690
9691 void trackStatistics() const override {
9692 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9693 }
9694};
9695
9696struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9697 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9698 : AAPotentialConstantValuesImpl(IRP, A) {}
9699
9700 void initialize(Attributor &A) override {
9701 AAPotentialConstantValuesImpl::initialize(A);
9702 if (isAtFixpoint())
9703 return;
9704
9705 Value &V = getAssociatedValue();
9706
9707 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9708 unionAssumed(C: C->getValue());
9709 indicateOptimisticFixpoint();
9710 return;
9711 }
9712
9713 if (isa<UndefValue>(Val: &V)) {
9714 unionAssumedWithUndef();
9715 indicateOptimisticFixpoint();
9716 return;
9717 }
9718
9719 if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9720 return;
9721
9722 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9723 return;
9724
9725 indicatePessimisticFixpoint();
9726
9727 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9728 << getAssociatedValue() << "\n");
9729 }
9730
9731 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9732 const APInt &RHS) {
9733 return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9734 }
9735
9736 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9737 uint32_t ResultBitWidth) {
9738 Instruction::CastOps CastOp = CI->getOpcode();
9739 switch (CastOp) {
9740 default:
9741 llvm_unreachable("unsupported or not integer cast");
9742 case Instruction::Trunc:
9743 return Src.trunc(width: ResultBitWidth);
9744 case Instruction::SExt:
9745 return Src.sext(width: ResultBitWidth);
9746 case Instruction::ZExt:
9747 return Src.zext(width: ResultBitWidth);
9748 case Instruction::BitCast:
9749 return Src;
9750 }
9751 }
9752
9753 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9754 const APInt &LHS, const APInt &RHS,
9755 bool &SkipOperation, bool &Unsupported) {
9756 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9757 // Unsupported is set to true when the binary operator is not supported.
9758 // SkipOperation is set to true when UB occur with the given operand pair
9759 // (LHS, RHS).
9760 // TODO: we should look at nsw and nuw keywords to handle operations
9761 // that create poison or undef value.
9762 switch (BinOpcode) {
9763 default:
9764 Unsupported = true;
9765 return LHS;
9766 case Instruction::Add:
9767 return LHS + RHS;
9768 case Instruction::Sub:
9769 return LHS - RHS;
9770 case Instruction::Mul:
9771 return LHS * RHS;
9772 case Instruction::UDiv:
9773 if (RHS.isZero()) {
9774 SkipOperation = true;
9775 return LHS;
9776 }
9777 return LHS.udiv(RHS);
9778 case Instruction::SDiv:
9779 if (RHS.isZero()) {
9780 SkipOperation = true;
9781 return LHS;
9782 }
9783 return LHS.sdiv(RHS);
9784 case Instruction::URem:
9785 if (RHS.isZero()) {
9786 SkipOperation = true;
9787 return LHS;
9788 }
9789 return LHS.urem(RHS);
9790 case Instruction::SRem:
9791 if (RHS.isZero()) {
9792 SkipOperation = true;
9793 return LHS;
9794 }
9795 return LHS.srem(RHS);
9796 case Instruction::Shl:
9797 return LHS.shl(ShiftAmt: RHS);
9798 case Instruction::LShr:
9799 return LHS.lshr(ShiftAmt: RHS);
9800 case Instruction::AShr:
9801 return LHS.ashr(ShiftAmt: RHS);
9802 case Instruction::And:
9803 return LHS & RHS;
9804 case Instruction::Or:
9805 return LHS | RHS;
9806 case Instruction::Xor:
9807 return LHS ^ RHS;
9808 }
9809 }
9810
9811 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9812 const APInt &LHS, const APInt &RHS) {
9813 bool SkipOperation = false;
9814 bool Unsupported = false;
9815 APInt Result =
9816 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9817 if (Unsupported)
9818 return false;
9819 // If SkipOperation is true, we can ignore this operand pair (L, R).
9820 if (!SkipOperation)
9821 unionAssumed(C: Result);
9822 return isValidState();
9823 }
9824
9825 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9826 auto AssumedBefore = getAssumed();
9827 Value *LHS = ICI->getOperand(i_nocapture: 0);
9828 Value *RHS = ICI->getOperand(i_nocapture: 1);
9829
9830 bool LHSContainsUndef = false, RHSContainsUndef = false;
9831 SetTy LHSAAPVS, RHSAAPVS;
9832 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9833 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
9834 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9835 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9836 return indicatePessimisticFixpoint();
9837
9838 // TODO: make use of undef flag to limit potential values aggressively.
9839 bool MaybeTrue = false, MaybeFalse = false;
9840 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
9841 if (LHSContainsUndef && RHSContainsUndef) {
9842 // The result of any comparison between undefs can be soundly replaced
9843 // with undef.
9844 unionAssumedWithUndef();
9845 } else if (LHSContainsUndef) {
9846 for (const APInt &R : RHSAAPVS) {
9847 bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
9848 MaybeTrue |= CmpResult;
9849 MaybeFalse |= !CmpResult;
9850 if (MaybeTrue & MaybeFalse)
9851 return indicatePessimisticFixpoint();
9852 }
9853 } else if (RHSContainsUndef) {
9854 for (const APInt &L : LHSAAPVS) {
9855 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
9856 MaybeTrue |= CmpResult;
9857 MaybeFalse |= !CmpResult;
9858 if (MaybeTrue & MaybeFalse)
9859 return indicatePessimisticFixpoint();
9860 }
9861 } else {
9862 for (const APInt &L : LHSAAPVS) {
9863 for (const APInt &R : RHSAAPVS) {
9864 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
9865 MaybeTrue |= CmpResult;
9866 MaybeFalse |= !CmpResult;
9867 if (MaybeTrue & MaybeFalse)
9868 return indicatePessimisticFixpoint();
9869 }
9870 }
9871 }
9872 if (MaybeTrue)
9873 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
9874 if (MaybeFalse)
9875 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
9876 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9877 : ChangeStatus::CHANGED;
9878 }
9879
9880 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
9881 auto AssumedBefore = getAssumed();
9882 Value *LHS = SI->getTrueValue();
9883 Value *RHS = SI->getFalseValue();
9884
9885 bool UsedAssumedInformation = false;
9886 std::optional<Constant *> C = A.getAssumedConstant(
9887 V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
9888
9889 // Check if we only need one operand.
9890 bool OnlyLeft = false, OnlyRight = false;
9891 if (C && *C && (*C)->isOneValue())
9892 OnlyLeft = true;
9893 else if (C && *C && (*C)->isNullValue())
9894 OnlyRight = true;
9895
9896 bool LHSContainsUndef = false, RHSContainsUndef = false;
9897 SetTy LHSAAPVS, RHSAAPVS;
9898 if (!OnlyRight &&
9899 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9900 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
9901 return indicatePessimisticFixpoint();
9902
9903 if (!OnlyLeft &&
9904 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9905 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9906 return indicatePessimisticFixpoint();
9907
9908 if (OnlyLeft || OnlyRight) {
9909 // select (true/false), lhs, rhs
9910 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
9911 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
9912
9913 if (Undef)
9914 unionAssumedWithUndef();
9915 else {
9916 for (const auto &It : *OpAA)
9917 unionAssumed(C: It);
9918 }
9919
9920 } else if (LHSContainsUndef && RHSContainsUndef) {
9921 // select i1 *, undef , undef => undef
9922 unionAssumedWithUndef();
9923 } else {
9924 for (const auto &It : LHSAAPVS)
9925 unionAssumed(C: It);
9926 for (const auto &It : RHSAAPVS)
9927 unionAssumed(C: It);
9928 }
9929 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9930 : ChangeStatus::CHANGED;
9931 }
9932
9933 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
9934 auto AssumedBefore = getAssumed();
9935 if (!CI->isIntegerCast())
9936 return indicatePessimisticFixpoint();
9937 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
9938 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
9939 Value *Src = CI->getOperand(i_nocapture: 0);
9940
9941 bool SrcContainsUndef = false;
9942 SetTy SrcPVS;
9943 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
9944 ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
9945 return indicatePessimisticFixpoint();
9946
9947 if (SrcContainsUndef)
9948 unionAssumedWithUndef();
9949 else {
9950 for (const APInt &S : SrcPVS) {
9951 APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
9952 unionAssumed(C: T);
9953 }
9954 }
9955 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9956 : ChangeStatus::CHANGED;
9957 }
9958
9959 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
9960 auto AssumedBefore = getAssumed();
9961 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9962 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9963
9964 bool LHSContainsUndef = false, RHSContainsUndef = false;
9965 SetTy LHSAAPVS, RHSAAPVS;
9966 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9967 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
9968 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9969 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9970 return indicatePessimisticFixpoint();
9971
9972 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
9973
9974 // TODO: make use of undef flag to limit potential values aggressively.
9975 if (LHSContainsUndef && RHSContainsUndef) {
9976 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
9977 return indicatePessimisticFixpoint();
9978 } else if (LHSContainsUndef) {
9979 for (const APInt &R : RHSAAPVS) {
9980 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
9981 return indicatePessimisticFixpoint();
9982 }
9983 } else if (RHSContainsUndef) {
9984 for (const APInt &L : LHSAAPVS) {
9985 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
9986 return indicatePessimisticFixpoint();
9987 }
9988 } else {
9989 for (const APInt &L : LHSAAPVS) {
9990 for (const APInt &R : RHSAAPVS) {
9991 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
9992 return indicatePessimisticFixpoint();
9993 }
9994 }
9995 }
9996 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9997 : ChangeStatus::CHANGED;
9998 }
9999
10000 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10001 auto AssumedBefore = getAssumed();
10002 SetTy Incoming;
10003 bool ContainsUndef;
10004 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10005 ContainsUndef, /* ForSelf */ true))
10006 return indicatePessimisticFixpoint();
10007 if (ContainsUndef) {
10008 unionAssumedWithUndef();
10009 } else {
10010 for (const auto &It : Incoming)
10011 unionAssumed(C: It);
10012 }
10013 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10014 : ChangeStatus::CHANGED;
10015 }
10016
10017 ChangeStatus updateImpl(Attributor &A) override {
10018 Value &V = getAssociatedValue();
10019 Instruction *I = dyn_cast<Instruction>(Val: &V);
10020
10021 if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10022 return updateWithICmpInst(A, ICI);
10023
10024 if (auto *SI = dyn_cast<SelectInst>(Val: I))
10025 return updateWithSelectInst(A, SI);
10026
10027 if (auto *CI = dyn_cast<CastInst>(Val: I))
10028 return updateWithCastInst(A, CI);
10029
10030 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10031 return updateWithBinaryOperator(A, BinOp);
10032
10033 if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10034 return updateWithInstruction(A, Inst: I);
10035
10036 return indicatePessimisticFixpoint();
10037 }
10038
10039 void trackStatistics() const override {
10040 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10041 }
10042};
10043
10044struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10045 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10046 : AAPotentialConstantValuesImpl(IRP, A) {}
10047
10048 ChangeStatus updateImpl(Attributor &A) override {
10049 llvm_unreachable(
10050 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10051 "not be called");
10052 }
10053
10054 void trackStatistics() const override {
10055 STATS_DECLTRACK_FN_ATTR(potential_values)
10056 }
10057};
10058
10059struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10060 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10061 : AAPotentialConstantValuesFunction(IRP, A) {}
10062
10063 void trackStatistics() const override {
10064 STATS_DECLTRACK_CS_ATTR(potential_values)
10065 }
10066};
10067
10068struct AAPotentialConstantValuesCallSiteReturned
10069 : AACalleeToCallSite<AAPotentialConstantValues,
10070 AAPotentialConstantValuesImpl> {
10071 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10072 Attributor &A)
10073 : AACalleeToCallSite<AAPotentialConstantValues,
10074 AAPotentialConstantValuesImpl>(IRP, A) {}
10075
10076 void trackStatistics() const override {
10077 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10078 }
10079};
10080
10081struct AAPotentialConstantValuesCallSiteArgument
10082 : AAPotentialConstantValuesFloating {
10083 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10084 Attributor &A)
10085 : AAPotentialConstantValuesFloating(IRP, A) {}
10086
10087 void initialize(Attributor &A) override {
10088 AAPotentialConstantValuesImpl::initialize(A);
10089 if (isAtFixpoint())
10090 return;
10091
10092 Value &V = getAssociatedValue();
10093
10094 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10095 unionAssumed(C: C->getValue());
10096 indicateOptimisticFixpoint();
10097 return;
10098 }
10099
10100 if (isa<UndefValue>(Val: &V)) {
10101 unionAssumedWithUndef();
10102 indicateOptimisticFixpoint();
10103 return;
10104 }
10105 }
10106
10107 ChangeStatus updateImpl(Attributor &A) override {
10108 Value &V = getAssociatedValue();
10109 auto AssumedBefore = getAssumed();
10110 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10111 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10112 if (!AA)
10113 return indicatePessimisticFixpoint();
10114 const auto &S = AA->getAssumed();
10115 unionAssumed(PVS: S);
10116 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10117 : ChangeStatus::CHANGED;
10118 }
10119
10120 void trackStatistics() const override {
10121 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10122 }
10123};
10124} // namespace
10125
10126/// ------------------------ NoUndef Attribute ---------------------------------
10127bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10128 Attribute::AttrKind ImpliedAttributeKind,
10129 bool IgnoreSubsumingPositions) {
10130 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10131 "Unexpected attribute kind");
10132 if (A.hasAttr(IRP, AKs: {Attribute::NoUndef}, IgnoreSubsumingPositions,
10133 ImpliedAttributeKind: Attribute::NoUndef))
10134 return true;
10135
10136 Value &Val = IRP.getAssociatedValue();
10137 if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10138 isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10139 LLVMContext &Ctx = Val.getContext();
10140 A.manifestAttrs(IRP, DeducedAttrs: Attribute::get(Context&: Ctx, Kind: Attribute::NoUndef));
10141 return true;
10142 }
10143
10144 return false;
10145}
10146
10147namespace {
10148struct AANoUndefImpl : AANoUndef {
10149 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10150
10151 void initialize(Attributor &A) override {
10152 Value &V = getAssociatedValue();
10153 if (isa<UndefValue>(Val: V))
10154 indicatePessimisticFixpoint();
10155 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10156 }
10157
10158 /// See followUsesInMBEC
10159 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10160 AANoUndef::StateType &State) {
10161 const Value *UseV = U->get();
10162 const DominatorTree *DT = nullptr;
10163 AssumptionCache *AC = nullptr;
10164 InformationCache &InfoCache = A.getInfoCache();
10165 if (Function *F = getAnchorScope()) {
10166 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10167 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10168 }
10169 State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10170 bool TrackUse = false;
10171 // Track use for instructions which must produce undef or poison bits when
10172 // at least one operand contains such bits.
10173 if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10174 TrackUse = true;
10175 return TrackUse;
10176 }
10177
10178 const std::string getAsStr(Attributor *A) const override {
10179 return getAssumed() ? "noundef" : "may-undef-or-poison";
10180 }
10181
10182 ChangeStatus manifest(Attributor &A) override {
10183 // We don't manifest noundef attribute for dead positions because the
10184 // associated values with dead positions would be replaced with undef
10185 // values.
10186 bool UsedAssumedInformation = false;
10187 if (A.isAssumedDead(IRP: getIRPosition(), QueryingAA: nullptr, FnLivenessAA: nullptr,
10188 UsedAssumedInformation))
10189 return ChangeStatus::UNCHANGED;
10190 // A position whose simplified value does not have any value is
10191 // considered to be dead. We don't manifest noundef in such positions for
10192 // the same reason above.
10193 if (!A.getAssumedSimplified(IRP: getIRPosition(), AA: *this, UsedAssumedInformation,
10194 S: AA::Interprocedural)
10195 .has_value())
10196 return ChangeStatus::UNCHANGED;
10197 return AANoUndef::manifest(A);
10198 }
10199};
10200
10201struct AANoUndefFloating : public AANoUndefImpl {
10202 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10203 : AANoUndefImpl(IRP, A) {}
10204
10205 void initialize(Attributor &A) override {
10206 AANoUndefImpl::initialize(A);
10207 if (!getState().isAtFixpoint() && getAnchorScope() &&
10208 !getAnchorScope()->isDeclaration())
10209 if (Instruction *CtxI = getCtxI())
10210 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10211 }
10212
10213 ChangeStatus updateImpl(Attributor &A) override {
10214 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10215 bool IsKnownNoUndef;
10216 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10217 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoUndef);
10218 };
10219
10220 bool Stripped;
10221 bool UsedAssumedInformation = false;
10222 Value *AssociatedValue = &getAssociatedValue();
10223 SmallVector<AA::ValueAndContext> Values;
10224 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10225 S: AA::AnyScope, UsedAssumedInformation))
10226 Stripped = false;
10227 else
10228 Stripped =
10229 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10230
10231 if (!Stripped) {
10232 // If we haven't stripped anything we might still be able to use a
10233 // different AA, but only if the IRP changes. Effectively when we
10234 // interpret this not as a call site value but as a floating/argument
10235 // value.
10236 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10237 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10238 return indicatePessimisticFixpoint();
10239 return ChangeStatus::UNCHANGED;
10240 }
10241
10242 for (const auto &VAC : Values)
10243 if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10244 return indicatePessimisticFixpoint();
10245
10246 return ChangeStatus::UNCHANGED;
10247 }
10248
10249 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10250};
10251
10252/// Return true if the poison-generating attributes in \p Attrs are assumed to
10253/// hold for the value at \p Pos.
10254static bool checkPoisonGeneratingAttrsHold(Attributor &A,
10255 const AbstractAttribute &QueryingAA,
10256 AttributeSet Attrs,
10257 const IRPosition &Pos,
10258 const Instruction *CtxI) {
10259 AttributeMask AM = AttributeFuncs::getPoisonGeneratingAttributes();
10260 for (Attribute Attr : Attrs) {
10261 if (!AM.contains(A: Attr))
10262 continue;
10263
10264 switch (Attr.getKindAsEnum()) {
10265 case Attribute::NonNull: {
10266 bool IsKnown;
10267 if (!AA::hasAssumedIRAttr<Attribute::NonNull>(
10268 A, QueryingAA: &QueryingAA, IRP: Pos, DepClass: DepClassTy::REQUIRED, IsKnown))
10269 return false;
10270 break;
10271 }
10272 case Attribute::Alignment: {
10273 const auto *AlignAA =
10274 A.getAAFor<AAAlign>(QueryingAA, IRP: Pos, DepClass: DepClassTy::REQUIRED);
10275 if (!AlignAA || AlignAA->getAssumedAlign() < *Attr.getAlignment())
10276 return false;
10277 break;
10278 }
10279 case Attribute::Range: {
10280 const auto *RangeAA = A.getAAFor<AAValueConstantRange>(
10281 QueryingAA, IRP: Pos, DepClass: DepClassTy::REQUIRED);
10282 if (!RangeAA ||
10283 !Attr.getRange().contains(CR: RangeAA->getAssumedConstantRange(A, CtxI)))
10284 return false;
10285 break;
10286 }
10287 case Attribute::NoFPClass: {
10288 FPClassTest NoFPClass = Attr.getNoFPClass();
10289 const auto *NoFPClassAA =
10290 A.getAAFor<AANoFPClass>(QueryingAA, IRP: Pos, DepClass: DepClassTy::REQUIRED);
10291 if (!NoFPClassAA ||
10292 (NoFPClassAA->getAssumedNoFPClass() & NoFPClass) != NoFPClass)
10293 return false;
10294 break;
10295 }
10296 default:
10297 // Conservatively assume unhandled attributes do not hold.
10298 return false;
10299 }
10300 }
10301
10302 return true;
10303}
10304
10305struct AANoUndefReturned final
10306 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10307 using Base = AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>;
10308 AANoUndefReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
10309
10310 void initialize(Attributor &A) override {
10311 Base::initialize(A);
10312 RetAttrs = getAssociatedFunction()->getAttributes().getRetAttrs();
10313 }
10314
10315 ChangeStatus updateImpl(Attributor &A) override {
10316 // A returned value that violates a poison-generating return attribute
10317 // becomes poison, so the attributes must hold for every returned value.
10318 if (RetAttrs.hasAttributes()) {
10319 auto AttrsHold = [&](Instruction &I) {
10320 Value &V = *cast<ReturnInst>(Val&: I).getReturnValue();
10321 return checkPoisonGeneratingAttrsHold(A, QueryingAA: *this, Attrs: RetAttrs,
10322 Pos: IRPosition::value(V), CtxI: &I);
10323 };
10324 bool UsedAssumedInformation = false;
10325 if (!A.checkForAllInstructions(Pred: AttrsHold, QueryingAA: *this, Opcodes: {Instruction::Ret},
10326 UsedAssumedInformation))
10327 return indicatePessimisticFixpoint();
10328 }
10329
10330 return Base::updateImpl(A);
10331 }
10332
10333 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10334
10335private:
10336 /// The function return attributes.
10337 AttributeSet RetAttrs;
10338};
10339
10340struct AANoUndefArgument final
10341 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10342 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10343 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10344
10345 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10346};
10347
10348struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10349 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10350 : AANoUndefFloating(IRP, A) {}
10351
10352 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10353};
10354
10355struct AANoUndefCallSiteReturned final
10356 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10357 using Base = AACalleeToCallSite<AANoUndef, AANoUndefImpl>;
10358 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10359 : Base(IRP, A) {}
10360
10361 void initialize(Attributor &A) override {
10362 Base::initialize(A);
10363 RetAttrs = cast<CallBase>(Val&: getAnchorValue()).getRetAttributes();
10364 }
10365
10366 ChangeStatus updateImpl(Attributor &A) override {
10367 // The call returns poison if the callee return value violates a
10368 // poison-generating call site return attribute, so the attributes must
10369 // hold for every callee.
10370 if (RetAttrs.hasAttributes()) {
10371 auto AttrsHold = [&](ArrayRef<const Function *> Callees) {
10372 return all_of(Range&: Callees, P: [&](const Function *Callee) {
10373 return checkPoisonGeneratingAttrsHold(
10374 A, QueryingAA: *this, Attrs: RetAttrs, Pos: IRPosition::returned(F: *Callee), CtxI: nullptr);
10375 });
10376 };
10377 if (!A.checkForAllCallees(Pred: AttrsHold, QueryingAA: *this,
10378 CB: cast<CallBase>(Val&: getAnchorValue())))
10379 return indicatePessimisticFixpoint();
10380 }
10381
10382 return Base::updateImpl(A);
10383 }
10384
10385 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10386
10387private:
10388 /// The call site return attributes.
10389 AttributeSet RetAttrs;
10390};
10391
10392/// ------------------------ NoFPClass Attribute -------------------------------
10393
10394struct AANoFPClassImpl : AANoFPClass {
10395 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10396
10397 void initialize(Attributor &A) override {
10398 const IRPosition &IRP = getIRPosition();
10399
10400 Value &V = IRP.getAssociatedValue();
10401 if (isa<UndefValue>(Val: V)) {
10402 indicateOptimisticFixpoint();
10403 return;
10404 }
10405
10406 SmallVector<Attribute> Attrs;
10407 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::NoFPClass}, Attrs, IgnoreSubsumingPositions: false);
10408 for (const auto &Attr : Attrs) {
10409 addKnownBits(Bits: Attr.getNoFPClass());
10410 }
10411
10412 Instruction *CtxI = getCtxI();
10413
10414 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10415 const DataLayout &DL = A.getDataLayout();
10416 InformationCache &InfoCache = A.getInfoCache();
10417
10418 const DominatorTree *DT = nullptr;
10419 AssumptionCache *AC = nullptr;
10420 const TargetLibraryInfo *TLI = nullptr;
10421 Function *F = getAnchorScope();
10422 if (F) {
10423 TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10424 if (!F->isDeclaration()) {
10425 DT =
10426 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10427 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10428 }
10429 }
10430
10431 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10432
10433 KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, InterestedClasses: fcAllFlags, SQ: Q);
10434 addKnownBits(Bits: ~KnownFPClass.getKnownFPClasses());
10435 }
10436
10437 if (CtxI)
10438 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10439 }
10440
10441 /// See followUsesInMBEC
10442 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10443 AANoFPClass::StateType &State) {
10444 // TODO: Determine what instructions can be looked through.
10445 auto *CB = dyn_cast<CallBase>(Val: I);
10446 if (!CB)
10447 return false;
10448
10449 if (!CB->isArgOperand(U))
10450 return false;
10451
10452 unsigned ArgNo = CB->getArgOperandNo(U);
10453 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
10454 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP, DepClass: DepClassTy::NONE))
10455 State.addKnownBits(Bits: NoFPAA->getState().getKnown());
10456 return false;
10457 }
10458
10459 const std::string getAsStr(Attributor *A) const override {
10460 std::string Result = "nofpclass";
10461 raw_string_ostream OS(Result);
10462 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10463 return Result;
10464 }
10465
10466 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10467 SmallVectorImpl<Attribute> &Attrs) const override {
10468 Attrs.emplace_back(Args: Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10469 }
10470};
10471
10472struct AANoFPClassFloating : public AANoFPClassImpl {
10473 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10474 : AANoFPClassImpl(IRP, A) {}
10475
10476 ChangeStatus updateImpl(Attributor &A) override {
10477 SmallVector<AA::ValueAndContext> Values;
10478 bool UsedAssumedInformation = false;
10479 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10480 S: AA::AnyScope, UsedAssumedInformation)) {
10481 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10482 }
10483
10484 StateType T;
10485 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10486 const auto *AA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP: IRPosition::value(V),
10487 DepClass: DepClassTy::REQUIRED);
10488 if (!AA || this == AA) {
10489 T.indicatePessimisticFixpoint();
10490 } else {
10491 const AANoFPClass::StateType &S =
10492 static_cast<const AANoFPClass::StateType &>(AA->getState());
10493 T ^= S;
10494 }
10495 return T.isValidState();
10496 };
10497
10498 for (const auto &VAC : Values)
10499 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10500 return indicatePessimisticFixpoint();
10501
10502 return clampStateAndIndicateChange(S&: getState(), R: T);
10503 }
10504
10505 void trackStatistics() const override {
10506 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10507 }
10508};
10509
10510struct AANoFPClassReturned final
10511 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10512 AANoFPClassImpl::StateType, false,
10513 Attribute::None, false> {
10514 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10515 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10516 AANoFPClassImpl::StateType, false,
10517 Attribute::None, false>(IRP, A) {}
10518
10519 void trackStatistics() const override {
10520 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10521 }
10522};
10523
10524struct AANoFPClassArgument final
10525 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10526 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10527 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10528
10529 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10530};
10531
10532struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10533 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10534 : AANoFPClassFloating(IRP, A) {}
10535
10536 void trackStatistics() const override {
10537 STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10538 }
10539};
10540
10541struct AANoFPClassCallSiteReturned final
10542 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10543 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10544 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10545
10546 void trackStatistics() const override {
10547 STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10548 }
10549};
10550
10551struct AACallEdgesImpl : public AACallEdges {
10552 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10553
10554 const SetVector<Function *> &getOptimisticEdges() const override {
10555 return CalledFunctions;
10556 }
10557
10558 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10559
10560 bool hasNonAsmUnknownCallee() const override {
10561 return HasUnknownCalleeNonAsm;
10562 }
10563
10564 const std::string getAsStr(Attributor *A) const override {
10565 return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10566 std::to_string(val: CalledFunctions.size()) + "]";
10567 }
10568
10569 void trackStatistics() const override {}
10570
10571protected:
10572 /// Return the callee of \p CB if it is a direct call whose callee has no
10573 /// callback metadata. Such a call only calls its callee.
10574 static Function *getOnlyCallee(const CallBase &CB) {
10575 auto *Callee = dyn_cast<Function>(Val: CB.getCalledOperand());
10576 if (Callee && !Callee->hasMetadata(KindID: LLVMContext::MD_callback))
10577 return Callee;
10578 return nullptr;
10579 }
10580
10581 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10582 if (CalledFunctions.insert(X: Fn)) {
10583 Change = ChangeStatus::CHANGED;
10584 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10585 << "\n");
10586 }
10587 }
10588
10589 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10590 if (!HasUnknownCallee)
10591 Change = ChangeStatus::CHANGED;
10592 if (NonAsm && !HasUnknownCalleeNonAsm)
10593 Change = ChangeStatus::CHANGED;
10594 HasUnknownCalleeNonAsm |= NonAsm;
10595 HasUnknownCallee = true;
10596 }
10597
10598private:
10599 /// Optimistic set of functions that might be called by this position.
10600 SetVector<Function *> CalledFunctions;
10601
10602 /// Is there any call with a unknown callee.
10603 bool HasUnknownCallee = false;
10604
10605 /// Is there any call with a unknown callee, excluding any inline asm.
10606 bool HasUnknownCalleeNonAsm = false;
10607};
10608
10609struct AACallEdgesCallSite : public AACallEdgesImpl {
10610 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10611 : AACallEdgesImpl(IRP, A) {}
10612
10613 void initialize(Attributor &A) override {
10614 // The only callee of a direct call without callbacks never changes.
10615 if (Function *Callee = getOnlyCallee(CB: cast<CallBase>(Val&: getAnchorValue()))) {
10616 ChangeStatus Change = ChangeStatus::UNCHANGED;
10617 addCalledFunction(Fn: Callee, Change);
10618 indicateOptimisticFixpoint();
10619 }
10620 }
10621
10622 ChangeStatus updateImpl(Attributor &A) override {
10623 ChangeStatus Change = ChangeStatus::UNCHANGED;
10624
10625 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10626 if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10627 addCalledFunction(Fn, Change);
10628 } else {
10629 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10630 setHasUnknownCallee(NonAsm: true, Change);
10631 }
10632
10633 // Explore all values.
10634 return true;
10635 };
10636
10637 SmallVector<AA::ValueAndContext> Values;
10638 // Process any value that we might call.
10639 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10640 if (isa<Constant>(Val: V)) {
10641 VisitValue(*V, CtxI);
10642 return;
10643 }
10644
10645 bool UsedAssumedInformation = false;
10646 Values.clear();
10647 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10648 S: AA::AnyScope, UsedAssumedInformation)) {
10649 Values.push_back(Elt: {*V, CtxI});
10650 }
10651 for (auto &VAC : Values)
10652 VisitValue(*VAC.getValue(), VAC.getCtxI());
10653 };
10654
10655 CallBase *CB = cast<CallBase>(Val: getCtxI());
10656
10657 if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10658 if (IA->hasSideEffects() &&
10659 !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10660 !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10661 setHasUnknownCallee(NonAsm: false, Change);
10662 }
10663 return Change;
10664 }
10665
10666 if (CB->isIndirectCall())
10667 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10668 QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10669 if (IndirectCallAA->foreachCallee(
10670 CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10671 return Change;
10672
10673 // The most simple case.
10674 ProcessCalledOperand(CB->getCalledOperand(), CB);
10675
10676 // Process callback functions.
10677 SmallVector<const Use *, 4u> CallbackUses;
10678 AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10679 for (const Use *U : CallbackUses)
10680 ProcessCalledOperand(U->get(), CB);
10681
10682 return Change;
10683 }
10684};
10685
10686struct AACallEdgesFunction : public AACallEdgesImpl {
10687 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10688 : AACallEdgesImpl(IRP, A) {}
10689
10690 ChangeStatus updateImpl(Attributor &A) override {
10691 ChangeStatus Change = ChangeStatus::UNCHANGED;
10692 bool AllCallsDirect = true;
10693
10694 auto ProcessCallInst = [&](Instruction &Inst) {
10695 CallBase &CB = cast<CallBase>(Val&: Inst);
10696
10697 // A direct call without callbacks only calls its callee; there is no
10698 // need for a call site attribute.
10699 if (Function *Callee = getOnlyCallee(CB)) {
10700 addCalledFunction(Fn: Callee, Change);
10701 return true;
10702 }
10703
10704 AllCallsDirect = false;
10705 auto *CBEdges = A.getAAFor<AACallEdges>(
10706 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10707 if (!CBEdges)
10708 return false;
10709 if (CBEdges->hasNonAsmUnknownCallee())
10710 setHasUnknownCallee(NonAsm: true, Change);
10711 if (CBEdges->hasUnknownCallee())
10712 setHasUnknownCallee(NonAsm: false, Change);
10713
10714 for (Function *F : CBEdges->getOptimisticEdges())
10715 addCalledFunction(Fn: F, Change);
10716
10717 return true;
10718 };
10719
10720 // Visit all callable instructions.
10721 bool UsedAssumedInformation = false;
10722 if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10723 UsedAssumedInformation,
10724 /* CheckBBLivenessOnly */ true)) {
10725 // If we haven't looked at all call like instructions, assume that there
10726 // are unknown callees.
10727 setHasUnknownCallee(NonAsm: true, Change);
10728 }
10729
10730 // The callees of direct calls in code known to be live do not change.
10731 if (AllCallsDirect && !UsedAssumedInformation)
10732 indicateOptimisticFixpoint();
10733
10734 return Change;
10735 }
10736};
10737
10738/// -------------------AAInterFnReachability Attribute--------------------------
10739
10740struct AAInterFnReachabilityFunction
10741 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10742 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10743 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10744 : Base(IRP, A) {}
10745
10746 bool instructionCanReach(
10747 Attributor &A, const Instruction &From, const Function &To,
10748 const AA::InstExclusionSetTy *ExclusionSet) const override {
10749 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10750 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10751
10752 RQITy StackRQI(A, From, To, ExclusionSet, false);
10753 RQITy::Reachable Result;
10754 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10755 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10756 /*IsTemporaryRQI=*/true);
10757 return Result == RQITy::Reachable::Yes;
10758 }
10759
10760 bool isReachableImpl(Attributor &A, RQITy &RQI,
10761 bool IsTemporaryRQI) override {
10762 const Instruction *EntryI =
10763 &RQI.From->getFunction()->getEntryBlock().front();
10764 if (EntryI != RQI.From &&
10765 !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10766 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10767 IsTemporaryRQI);
10768
10769 auto CheckReachableCallBase = [&](CallBase *CB) {
10770 auto *CBEdges = A.getAAFor<AACallEdges>(
10771 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10772 if (!CBEdges || !CBEdges->getState().isValidState())
10773 return false;
10774 // TODO Check To backwards in this case.
10775 if (CBEdges->hasUnknownCallee())
10776 return false;
10777
10778 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10779 if (Fn == RQI.To)
10780 return false;
10781
10782 if (Fn->isDeclaration()) {
10783 if (Fn->hasFnAttribute(Kind: Attribute::NoCallback))
10784 continue;
10785 // TODO Check To backwards in this case.
10786 return false;
10787 }
10788
10789 if (Fn == getAnchorScope()) {
10790 if (EntryI == RQI.From)
10791 continue;
10792 return false;
10793 }
10794
10795 const AAInterFnReachability *InterFnReachability =
10796 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10797 DepClass: DepClassTy::OPTIONAL);
10798
10799 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10800 if (!InterFnReachability ||
10801 InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10802 ExclusionSet: RQI.ExclusionSet))
10803 return false;
10804 }
10805 return true;
10806 };
10807
10808 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10809 QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10810 DepClass: DepClassTy::OPTIONAL);
10811
10812 // Determine call like instructions that we can reach from the inst.
10813 auto CheckCallBase = [&](Instruction &CBInst) {
10814 // There are usually less nodes in the call graph, check inter function
10815 // reachability first.
10816 if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10817 return true;
10818 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10819 A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10820 };
10821
10822 bool UsedExclusionSet = /* conservative */ true;
10823 bool UsedAssumedInformation = false;
10824 if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10825 UsedAssumedInformation,
10826 /* CheckBBLivenessOnly */ true))
10827 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10828 IsTemporaryRQI);
10829
10830 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10831 IsTemporaryRQI);
10832 }
10833
10834 void trackStatistics() const override {}
10835};
10836} // namespace
10837
10838template <typename AAType>
10839static std::optional<Constant *>
10840askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10841 const IRPosition &IRP, Type &Ty) {
10842 if (!Ty.isIntegerTy())
10843 return nullptr;
10844
10845 // This will also pass the call base context.
10846 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10847 if (!AA)
10848 return nullptr;
10849
10850 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10851
10852 if (!COpt.has_value()) {
10853 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10854 return std::nullopt;
10855 }
10856 if (auto *C = *COpt) {
10857 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10858 return C;
10859 }
10860 return nullptr;
10861}
10862
10863Value *AAPotentialValues::getSingleValue(
10864 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10865 SmallVectorImpl<AA::ValueAndContext> &Values) {
10866 Type &Ty = *IRP.getAssociatedType();
10867 std::optional<Value *> V;
10868 for (auto &It : Values) {
10869 V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10870 if (V.has_value() && !*V)
10871 break;
10872 }
10873 if (!V.has_value())
10874 return UndefValue::get(T: &Ty);
10875 return *V;
10876}
10877
10878namespace {
10879struct AAPotentialValuesImpl : AAPotentialValues {
10880 using StateType = PotentialLLVMValuesState;
10881
10882 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10883 : AAPotentialValues(IRP, A) {}
10884
10885 void initialize(Attributor &A) override {
10886 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10887 indicatePessimisticFixpoint();
10888 return;
10889 }
10890 Value *Stripped = getAssociatedValue().stripPointerCasts();
10891 if (isa<Constant>(Val: Stripped) && !isa<ConstantExpr>(Val: Stripped)) {
10892 addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10893 AnchorScope: getAnchorScope());
10894 indicateOptimisticFixpoint();
10895 return;
10896 }
10897 AAPotentialValues::initialize(A);
10898 }
10899
10900 const std::string getAsStr(Attributor *A) const override {
10901 std::string Str;
10902 llvm::raw_string_ostream OS(Str);
10903 OS << getState();
10904 return Str;
10905 }
10906
10907 template <typename AAType>
10908 static std::optional<Value *> askOtherAA(Attributor &A,
10909 const AbstractAttribute &AA,
10910 const IRPosition &IRP, Type &Ty) {
10911 if (isa<Constant>(Val: IRP.getAssociatedValue()))
10912 return &IRP.getAssociatedValue();
10913 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10914 if (!C)
10915 return std::nullopt;
10916 if (*C)
10917 if (auto *CC = AA::getWithType(V&: **C, Ty))
10918 return CC;
10919 return nullptr;
10920 }
10921
10922 virtual void addValue(Attributor &A, StateType &State, Value &V,
10923 const Instruction *CtxI, AA::ValueScope S,
10924 Function *AnchorScope) const {
10925
10926 IRPosition ValIRP = IRPosition::value(V);
10927 if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
10928 for (const auto &U : CB->args()) {
10929 if (U.get() != &V)
10930 continue;
10931 ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
10932 break;
10933 }
10934 }
10935
10936 Value *VPtr = &V;
10937 if (ValIRP.getAssociatedType()->isIntegerTy()) {
10938 Type &Ty = *getAssociatedType();
10939 std::optional<Value *> SimpleV =
10940 askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
10941 if (SimpleV.has_value() && !*SimpleV) {
10942 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10943 QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
10944 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10945 for (const auto &It : PotentialConstantsAA->getAssumedSet())
10946 State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
10947 if (PotentialConstantsAA->undefIsContained())
10948 State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
10949 return;
10950 }
10951 }
10952 if (!SimpleV.has_value())
10953 return;
10954
10955 if (*SimpleV)
10956 VPtr = *SimpleV;
10957 }
10958
10959 if (isa<ConstantInt>(Val: VPtr))
10960 CtxI = nullptr;
10961 if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
10962 S = AA::ValueScope(S | AA::Interprocedural);
10963
10964 State.unionAssumed(C: {{*VPtr, CtxI}, S});
10965 }
10966
10967 /// Helper struct to tie a value+context pair together with the scope for
10968 /// which this is the simplified version.
10969 struct ItemInfo {
10970 AA::ValueAndContext I;
10971 AA::ValueScope S;
10972
10973 bool operator==(const ItemInfo &II) const {
10974 return II.I == I && II.S == S;
10975 };
10976 bool operator<(const ItemInfo &II) const {
10977 return std::tie(args: I, args: S) < std::tie(args: II.I, args: II.S);
10978 };
10979 };
10980
10981 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
10982 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
10983 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
10984 if (!(CS & S))
10985 continue;
10986
10987 bool UsedAssumedInformation = false;
10988 SmallVector<AA::ValueAndContext> Values;
10989 if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
10990 UsedAssumedInformation))
10991 return false;
10992
10993 for (auto &It : Values)
10994 ValueScopeMap[It] += CS;
10995 }
10996 for (auto &It : ValueScopeMap)
10997 addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
10998 S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
10999
11000 return true;
11001 }
11002
11003 void giveUpOnIntraprocedural(Attributor &A) {
11004 auto NewS = StateType::getBestState(PVS: getState());
11005 for (const auto &It : getAssumedSet()) {
11006 if (It.second == AA::Intraprocedural)
11007 continue;
11008 addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11009 S: AA::Interprocedural, AnchorScope: getAnchorScope());
11010 }
11011 assert(!undefIsContained() && "Undef should be an explicit value!");
11012 addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
11013 AnchorScope: getAnchorScope());
11014 getState() = NewS;
11015 }
11016
11017 ChangeStatus indicatePessimisticFixpoint() override {
11018 getState() = StateType::getBestState(PVS: getState());
11019 getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11020 AAPotentialValues::indicateOptimisticFixpoint();
11021 return ChangeStatus::CHANGED;
11022 }
11023
11024 ChangeStatus updateImpl(Attributor &A) override {
11025 return indicatePessimisticFixpoint();
11026 }
11027
11028 ChangeStatus manifest(Attributor &A) override {
11029 SmallVector<AA::ValueAndContext> Values;
11030 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11031 Values.clear();
11032 if (!getAssumedSimplifiedValues(A, Values, S))
11033 continue;
11034 Value &OldV = getAssociatedValue();
11035 if (isa<UndefValue>(Val: OldV))
11036 continue;
11037 Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11038 if (!NewV || NewV == &OldV)
11039 continue;
11040 if (getCtxI() &&
11041 !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
11042 continue;
11043 if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
11044 return ChangeStatus::CHANGED;
11045 }
11046 return ChangeStatus::UNCHANGED;
11047 }
11048
11049 bool getAssumedSimplifiedValues(
11050 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11051 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11052 if (!isValidState())
11053 return false;
11054 bool UsedAssumedInformation = false;
11055 for (const auto &It : getAssumedSet())
11056 if (It.second & S) {
11057 if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
11058 isa<SelectInst>(Val: It.first.getValue()))) {
11059 if (A.getAssumedSimplifiedValues(
11060 IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
11061 AA: this, Values, S, UsedAssumedInformation))
11062 continue;
11063 }
11064 Values.push_back(Elt: It.first);
11065 }
11066 assert(!undefIsContained() && "Undef should be an explicit value!");
11067 return true;
11068 }
11069};
11070
11071struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11072 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11073 : AAPotentialValuesImpl(IRP, A) {}
11074
11075 ChangeStatus updateImpl(Attributor &A) override {
11076 auto AssumedBefore = getAssumed();
11077
11078 genericValueTraversal(A, InitialV: &getAssociatedValue());
11079
11080 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11081 : ChangeStatus::CHANGED;
11082 }
11083
11084 /// Helper struct to remember which AAIsDead instances we actually used.
11085 struct LivenessInfo {
11086 const AAIsDead *LivenessAA = nullptr;
11087 bool AnyDead = false;
11088 };
11089
11090 /// Check if \p Cmp is a comparison we can simplify.
11091 ///
11092 /// We handle multiple cases, one in which at least one operand is an
11093 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11094 /// operand. Return true if successful, in that case Worklist will be updated.
11095 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11096 CmpInst::Predicate Pred, ItemInfo II,
11097 SmallVectorImpl<ItemInfo> &Worklist) {
11098
11099 // Simplify the operands first.
11100 bool UsedAssumedInformation = false;
11101 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11102 auto GetSimplifiedValues = [&](Value &V,
11103 SmallVector<AA::ValueAndContext> &Values) {
11104 if (!A.getAssumedSimplifiedValues(
11105 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11106 S: AA::Intraprocedural, UsedAssumedInformation)) {
11107 Values.clear();
11108 Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11109 }
11110 return Values.empty();
11111 };
11112 if (GetSimplifiedValues(*LHS, LHSValues))
11113 return true;
11114 if (GetSimplifiedValues(*RHS, RHSValues))
11115 return true;
11116
11117 LLVMContext &Ctx = LHS->getContext();
11118
11119 InformationCache &InfoCache = A.getInfoCache();
11120 Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11121 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11122 const auto *DT =
11123 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11124 : nullptr;
11125 const auto *TLI =
11126 F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11127 auto *AC =
11128 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11129 : nullptr;
11130
11131 const DataLayout &DL = A.getDataLayout();
11132 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11133
11134 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11135 if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11136 addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11137 /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11138 return true;
11139 }
11140
11141 // Handle the trivial case first in which we don't even need to think
11142 // about null or non-null.
11143 if (&LHSV == &RHSV &&
11144 (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11145 Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11146 V: CmpInst::isTrueWhenEqual(predicate: Pred));
11147 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11148 AnchorScope: getAnchorScope());
11149 return true;
11150 }
11151
11152 auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11153 auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11154 if (TypedLHS && TypedRHS) {
11155 Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11156 if (NewV && NewV != &Cmp) {
11157 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11158 AnchorScope: getAnchorScope());
11159 return true;
11160 }
11161 }
11162
11163 // From now on we only handle equalities (==, !=).
11164 if (!CmpInst::isEquality(pred: Pred))
11165 return false;
11166
11167 bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11168 bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11169 if (!LHSIsNull && !RHSIsNull)
11170 return false;
11171
11172 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11173 // non-nullptr operand and if we assume it's non-null we can conclude the
11174 // result of the comparison.
11175 assert((LHSIsNull || RHSIsNull) &&
11176 "Expected nullptr versus non-nullptr comparison at this point");
11177
11178 // The index is the operand that we assume is not null.
11179 unsigned PtrIdx = LHSIsNull;
11180 bool IsKnownNonNull;
11181 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11182 A, QueryingAA: this, IRP: IRPosition::value(V: *(PtrIdx ? &RHSV : &LHSV)),
11183 DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNonNull);
11184 if (!IsAssumedNonNull)
11185 return false;
11186
11187 // The new value depends on the predicate, true for != and false for ==.
11188 Constant *NewV =
11189 ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11190 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11191 AnchorScope: getAnchorScope());
11192 return true;
11193 };
11194
11195 for (auto &LHSValue : LHSValues)
11196 for (auto &RHSValue : RHSValues)
11197 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11198 return false;
11199 return true;
11200 }
11201
11202 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11203 SmallVectorImpl<ItemInfo> &Worklist) {
11204 const Instruction *CtxI = II.I.getCtxI();
11205 bool UsedAssumedInformation = false;
11206
11207 std::optional<Constant *> C =
11208 A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11209 bool NoValueYet = !C.has_value();
11210 if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11211 return true;
11212 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11213 if (CI->isZero())
11214 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11215 else
11216 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11217 } else if (&SI == &getAssociatedValue()) {
11218 // We could not simplify the condition, assume both values.
11219 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11220 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11221 } else {
11222 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11223 IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11224 if (!SimpleV.has_value())
11225 return true;
11226 if (*SimpleV) {
11227 addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11228 return true;
11229 }
11230 return false;
11231 }
11232 return true;
11233 }
11234
11235 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11236 SmallVectorImpl<ItemInfo> &Worklist) {
11237 SmallSetVector<Value *, 4> PotentialCopies;
11238 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11239 bool UsedAssumedInformation = false;
11240 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11241 PotentialValueOrigins, QueryingAA: *this,
11242 UsedAssumedInformation,
11243 /* OnlyExact */ true)) {
11244 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11245 "loaded values for load instruction "
11246 << LI << "\n");
11247 return false;
11248 }
11249
11250 // Do not simplify loads that are only used in llvm.assume if we cannot also
11251 // remove all stores that may feed into the load. The reason is that the
11252 // assume is probably worth something as long as the stores are around.
11253 InformationCache &InfoCache = A.getInfoCache();
11254 if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11255 if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11256 if (!I || isa<AssumeInst>(Val: I))
11257 return true;
11258 if (auto *SI = dyn_cast<StoreInst>(Val: I))
11259 return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11260 /* LivenessAA */ FnLivenessAA: nullptr,
11261 UsedAssumedInformation,
11262 /* CheckBBLivenessOnly */ false);
11263 return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11264 UsedAssumedInformation,
11265 /* CheckBBLivenessOnly */ false);
11266 })) {
11267 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11268 "and we cannot delete all the stores: "
11269 << LI << "\n");
11270 return false;
11271 }
11272 }
11273
11274 // Values have to be dynamically unique or we loose the fact that a
11275 // single llvm::Value might represent two runtime values (e.g.,
11276 // stack locations in different recursive calls).
11277 const Instruction *CtxI = II.I.getCtxI();
11278 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11279 bool AllLocal = ScopeIsLocal;
11280 bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11281 AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11282 return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11283 });
11284 if (!DynamicallyUnique) {
11285 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11286 "values are dynamically unique: "
11287 << LI << "\n");
11288 return false;
11289 }
11290
11291 for (auto *PotentialCopy : PotentialCopies) {
11292 if (AllLocal) {
11293 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11294 } else {
11295 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11296 }
11297 }
11298 if (!AllLocal && ScopeIsLocal)
11299 addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11300 return true;
11301 }
11302
11303 bool handlePHINode(
11304 Attributor &A, PHINode &PHI, ItemInfo II,
11305 SmallVectorImpl<ItemInfo> &Worklist,
11306 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11307 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11308 LivenessInfo &LI = LivenessAAs[&F];
11309 if (!LI.LivenessAA)
11310 LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11311 DepClass: DepClassTy::NONE);
11312 return LI;
11313 };
11314
11315 if (&PHI == &getAssociatedValue()) {
11316 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11317 const auto *CI =
11318 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11319 F: *PHI.getFunction());
11320
11321 CycleRef C;
11322 bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11323 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11324 BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11325 if (LI.LivenessAA &&
11326 LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11327 LI.AnyDead = true;
11328 continue;
11329 }
11330 Value *V = PHI.getIncomingValue(i: u);
11331 if (V == &PHI)
11332 continue;
11333
11334 // If the incoming value is not the PHI but an instruction in the same
11335 // cycle we might have multiple versions of it flying around.
11336 if (CyclePHI && isa<Instruction>(Val: V) &&
11337 (!C || CI->contains(C, Block: cast<Instruction>(Val: V)->getParent())))
11338 return false;
11339
11340 Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11341 }
11342 return true;
11343 }
11344
11345 bool UsedAssumedInformation = false;
11346 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11347 IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11348 if (!SimpleV.has_value())
11349 return true;
11350 if (!(*SimpleV))
11351 return false;
11352 addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11353 return true;
11354 }
11355
11356 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11357 /// simplify any operand of the instruction \p I. Return true if successful,
11358 /// in that case Worklist will be updated.
11359 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11360 SmallVectorImpl<ItemInfo> &Worklist) {
11361 bool SomeSimplified = false;
11362 bool UsedAssumedInformation = false;
11363
11364 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11365 int Idx = 0;
11366 for (Value *Op : I.operands()) {
11367 const auto &SimplifiedOp = A.getAssumedSimplified(
11368 IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11369 UsedAssumedInformation, S: AA::Intraprocedural);
11370 // If we are not sure about any operand we are not sure about the entire
11371 // instruction, we'll wait.
11372 if (!SimplifiedOp.has_value())
11373 return true;
11374
11375 if (*SimplifiedOp)
11376 NewOps[Idx] = *SimplifiedOp;
11377 else
11378 NewOps[Idx] = Op;
11379
11380 SomeSimplified |= (NewOps[Idx] != Op);
11381 ++Idx;
11382 }
11383
11384 // We won't bother with the InstSimplify interface if we didn't simplify any
11385 // operand ourselves.
11386 if (!SomeSimplified)
11387 return false;
11388
11389 InformationCache &InfoCache = A.getInfoCache();
11390 Function *F = I.getFunction();
11391 const auto *DT =
11392 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11393 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11394 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11395
11396 const DataLayout &DL = I.getDataLayout();
11397 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11398 Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11399 if (!NewV || NewV == &I)
11400 return false;
11401
11402 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11403 << *NewV << "\n");
11404 Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11405 return true;
11406 }
11407
11408 bool simplifyInstruction(
11409 Attributor &A, Instruction &I, ItemInfo II,
11410 SmallVectorImpl<ItemInfo> &Worklist,
11411 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11412 if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11413 return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11414 Pred: CI->getPredicate(), II, Worklist);
11415
11416 switch (I.getOpcode()) {
11417 case Instruction::Select:
11418 return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11419 case Instruction::PHI:
11420 return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11421 case Instruction::Load:
11422 return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11423 default:
11424 return handleGenericInst(A, I, II, Worklist);
11425 };
11426 return false;
11427 }
11428
11429 void genericValueTraversal(Attributor &A, Value *InitialV) {
11430 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11431
11432 SmallSet<ItemInfo, 16> Visited;
11433 SmallVector<ItemInfo, 16> Worklist;
11434 Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11435
11436 int Iteration = 0;
11437 do {
11438 ItemInfo II = Worklist.pop_back_val();
11439 Value *V = II.I.getValue();
11440 assert(V);
11441 const Instruction *CtxI = II.I.getCtxI();
11442 AA::ValueScope S = II.S;
11443
11444 // Check if we should process the current value. To prevent endless
11445 // recursion keep a record of the values we followed!
11446 if (!Visited.insert(V: II).second)
11447 continue;
11448
11449 // Make sure we limit the compile time for complex expressions.
11450 if (Iteration++ >= A.Opts.attributor_max_potential_values_iterations) {
11451 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11452 << Iteration << "!\n");
11453 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11454 continue;
11455 }
11456
11457 // Explicitly look through calls with a "returned" attribute if we do
11458 // not have a pointer as stripPointerCasts only works on them.
11459 Value *NewV = nullptr;
11460 if (V->getType()->isPointerTy()) {
11461 NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11462 } else {
11463 if (auto *CB = dyn_cast<CallBase>(Val: V))
11464 if (auto *Callee =
11465 dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11466 for (Argument &Arg : Callee->args())
11467 if (Arg.hasReturnedAttr()) {
11468 NewV = CB->getArgOperand(i: Arg.getArgNo());
11469 break;
11470 }
11471 }
11472 }
11473 if (NewV && NewV != V) {
11474 Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11475 continue;
11476 }
11477
11478 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11479 if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11480 continue;
11481 }
11482
11483 if (V != InitialV || isa<Argument>(Val: V))
11484 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11485 continue;
11486
11487 // If we haven't stripped anything we give up.
11488 if (V == InitialV && CtxI == getCtxI()) {
11489 indicatePessimisticFixpoint();
11490 return;
11491 }
11492
11493 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11494 } while (!Worklist.empty());
11495
11496 // If we actually used liveness information so we have to record a
11497 // dependence.
11498 for (auto &It : LivenessAAs)
11499 if (It.second.AnyDead)
11500 A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11501 }
11502
11503 void trackStatistics() const override {
11504 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11505 }
11506};
11507
11508struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11509 using Base = AAPotentialValuesImpl;
11510 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11511 : Base(IRP, A) {}
11512
11513 void initialize(Attributor &A) override {
11514 auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11515 if (Arg.hasPointeeInMemoryValueAttr())
11516 indicatePessimisticFixpoint();
11517 }
11518
11519 ChangeStatus updateImpl(Attributor &A) override {
11520 auto AssumedBefore = getAssumed();
11521
11522 unsigned ArgNo = getCalleeArgNo();
11523
11524 bool UsedAssumedInformation = false;
11525 SmallVector<AA::ValueAndContext> Values;
11526 auto CallSitePred = [&](AbstractCallSite ACS) {
11527 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11528 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11529 return false;
11530
11531 if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11532 S: AA::Interprocedural,
11533 UsedAssumedInformation))
11534 return false;
11535
11536 return isValidState();
11537 };
11538
11539 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11540 /* RequireAllCallSites */ true,
11541 UsedAssumedInformation))
11542 return indicatePessimisticFixpoint();
11543
11544 Function *Fn = getAssociatedFunction();
11545 bool AnyNonLocal = false;
11546 for (auto &It : Values) {
11547 if (isa<Constant>(Val: It.getValue())) {
11548 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11549 AnchorScope: getAnchorScope());
11550 continue;
11551 }
11552 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11553 return indicatePessimisticFixpoint();
11554
11555 if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11556 if (Arg->getParent() == Fn) {
11557 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11558 AnchorScope: getAnchorScope());
11559 continue;
11560 }
11561 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11562 AnchorScope: getAnchorScope());
11563 AnyNonLocal = true;
11564 }
11565 assert(!undefIsContained() && "Undef should be an explicit value!");
11566 if (AnyNonLocal)
11567 giveUpOnIntraprocedural(A);
11568
11569 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11570 : ChangeStatus::CHANGED;
11571 }
11572
11573 void trackStatistics() const override {
11574 STATS_DECLTRACK_ARG_ATTR(potential_values)
11575 }
11576};
11577
11578struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11579 using Base = AAPotentialValuesFloating;
11580 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11581 : Base(IRP, A) {}
11582
11583 void initialize(Attributor &A) override {
11584 Function *F = getAssociatedFunction();
11585 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11586 indicatePessimisticFixpoint();
11587 return;
11588 }
11589
11590 for (Argument &Arg : F->args())
11591 if (Arg.hasReturnedAttr()) {
11592 addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11593 ReturnedArg = &Arg;
11594 break;
11595 }
11596 if (!A.isFunctionIPOAmendable(F: *F) ||
11597 A.hasSimplificationCallback(IRP: getIRPosition())) {
11598 if (!ReturnedArg)
11599 indicatePessimisticFixpoint();
11600 else
11601 indicateOptimisticFixpoint();
11602 }
11603 }
11604
11605 ChangeStatus updateImpl(Attributor &A) override {
11606 auto AssumedBefore = getAssumed();
11607 bool UsedAssumedInformation = false;
11608
11609 SmallVector<AA::ValueAndContext> Values;
11610 Function *AnchorScope = getAnchorScope();
11611 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11612 bool AddValues) {
11613 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11614 Values.clear();
11615 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11616 UsedAssumedInformation,
11617 /* RecurseForSelectAndPHI */ true))
11618 return false;
11619 if (!AddValues)
11620 continue;
11621
11622 bool AllInterAreIntra = false;
11623 if (S == AA::Interprocedural)
11624 AllInterAreIntra =
11625 llvm::all_of(Range&: Values, P: [&](const AA::ValueAndContext &VAC) {
11626 return AA::isValidInScope(V: *VAC.getValue(), Scope: AnchorScope);
11627 });
11628
11629 for (const AA::ValueAndContext &VAC : Values) {
11630 addValue(A, State&: getState(), V&: *VAC.getValue(),
11631 CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11632 S: AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11633 }
11634 if (AllInterAreIntra)
11635 break;
11636 }
11637 return true;
11638 };
11639
11640 if (ReturnedArg) {
11641 HandleReturnedValue(*ReturnedArg, nullptr, true);
11642 } else {
11643 auto RetInstPred = [&](Instruction &RetI) {
11644 bool AddValues = true;
11645 if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11646 isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11647 addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11648 AnchorScope);
11649 AddValues = false;
11650 }
11651 return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11652 };
11653
11654 if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11655 UsedAssumedInformation,
11656 /* CheckBBLivenessOnly */ true))
11657 return indicatePessimisticFixpoint();
11658 }
11659
11660 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11661 : ChangeStatus::CHANGED;
11662 }
11663
11664 ChangeStatus manifest(Attributor &A) override {
11665 if (ReturnedArg)
11666 return ChangeStatus::UNCHANGED;
11667 SmallVector<AA::ValueAndContext> Values;
11668 if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11669 /* RecurseForSelectAndPHI */ true))
11670 return ChangeStatus::UNCHANGED;
11671 Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11672 if (!NewVal)
11673 return ChangeStatus::UNCHANGED;
11674
11675 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11676 if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11677 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11678 "Number of function with unique return");
11679 Changed |= A.manifestAttrs(
11680 IRP: IRPosition::argument(Arg: *Arg),
11681 DeducedAttrs: {Attribute::get(Context&: Arg->getContext(), Kind: Attribute::Returned)});
11682 STATS_DECLTRACK_ARG_ATTR(returned);
11683 }
11684
11685 auto RetInstPred = [&](Instruction &RetI) {
11686 Value *RetOp = RetI.getOperand(i: 0);
11687 if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11688 return true;
11689 if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11690 if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11691 Changed = ChangeStatus::CHANGED;
11692 return true;
11693 };
11694 bool UsedAssumedInformation = false;
11695 (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11696 UsedAssumedInformation,
11697 /* CheckBBLivenessOnly */ true);
11698 return Changed;
11699 }
11700
11701 ChangeStatus indicatePessimisticFixpoint() override {
11702 return AAPotentialValues::indicatePessimisticFixpoint();
11703 }
11704
11705 void trackStatistics() const override{
11706 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11707
11708 /// The argumented with an existing `returned` attribute.
11709 Argument *ReturnedArg = nullptr;
11710};
11711
11712struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11713 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11714 : AAPotentialValuesImpl(IRP, A) {}
11715
11716 ChangeStatus updateImpl(Attributor &A) override {
11717 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11718 "not be called");
11719 }
11720
11721 void trackStatistics() const override {
11722 STATS_DECLTRACK_FN_ATTR(potential_values)
11723 }
11724};
11725
11726struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11727 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11728 : AAPotentialValuesFunction(IRP, A) {}
11729
11730 void trackStatistics() const override {
11731 STATS_DECLTRACK_CS_ATTR(potential_values)
11732 }
11733};
11734
11735struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11736 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11737 : AAPotentialValuesImpl(IRP, A) {}
11738
11739 ChangeStatus updateImpl(Attributor &A) override {
11740 auto AssumedBefore = getAssumed();
11741
11742 Function *Callee = getAssociatedFunction();
11743 if (!Callee)
11744 return indicatePessimisticFixpoint();
11745
11746 bool UsedAssumedInformation = false;
11747 auto *CB = cast<CallBase>(Val: getCtxI());
11748 if (CB->isMustTailCall() &&
11749 !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11750 UsedAssumedInformation))
11751 return indicatePessimisticFixpoint();
11752
11753 Function *Caller = CB->getCaller();
11754
11755 auto AddScope = [&](AA::ValueScope S) {
11756 SmallVector<AA::ValueAndContext> Values;
11757 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11758 Values, S, UsedAssumedInformation))
11759 return false;
11760
11761 for (auto &It : Values) {
11762 Value *V = It.getValue();
11763 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11764 V, CB&: *CB, AA: *this, UsedAssumedInformation);
11765 if (!CallerV.has_value()) {
11766 // Nothing to do as long as no value was determined.
11767 continue;
11768 }
11769 V = *CallerV ? *CallerV : V;
11770 if (*CallerV && AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) {
11771 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S))
11772 continue;
11773 }
11774 if (S == AA::Intraprocedural && !AA::isValidInScope(V: *V, Scope: Caller)) {
11775 giveUpOnIntraprocedural(A);
11776 return true;
11777 }
11778 addValue(A, State&: getState(), V&: *V, CtxI: CB, S, AnchorScope: getAnchorScope());
11779 }
11780 return true;
11781 };
11782 if (!AddScope(AA::Intraprocedural))
11783 return indicatePessimisticFixpoint();
11784 if (!AddScope(AA::Interprocedural))
11785 return indicatePessimisticFixpoint();
11786 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11787 : ChangeStatus::CHANGED;
11788 }
11789
11790 ChangeStatus indicatePessimisticFixpoint() override {
11791 return AAPotentialValues::indicatePessimisticFixpoint();
11792 }
11793
11794 void trackStatistics() const override {
11795 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11796 }
11797};
11798
11799struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11800 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11801 : AAPotentialValuesFloating(IRP, A) {}
11802
11803 void trackStatistics() const override {
11804 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11805 }
11806};
11807} // namespace
11808
11809/// ---------------------- Assumption Propagation ------------------------------
11810namespace {
11811struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11812 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11813 const DenseSet<StringRef> &Known)
11814 : AAAssumptionInfo(IRP, A, Known) {}
11815
11816 ChangeStatus manifest(Attributor &A) override {
11817 // Don't manifest a universal set if it somehow made it here.
11818 if (getKnown().isUniversal())
11819 return ChangeStatus::UNCHANGED;
11820
11821 const IRPosition &IRP = getIRPosition();
11822 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11823 getAssumed().getSet().end());
11824 llvm::sort(C&: Set);
11825 return A.manifestAttrs(IRP,
11826 DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(),
11827 Kind: AssumptionAttrKey,
11828 Val: llvm::join(R&: Set, Separator: ",")),
11829 /*ForceReplace=*/true);
11830 }
11831
11832 bool hasAssumption(const StringRef Assumption) const override {
11833 return isValidState() && setContains(Assumption);
11834 }
11835
11836 const std::string getAsStr(Attributor *A) const override {
11837 const SetContents &Known = getKnown();
11838 const SetContents &Assumed = getAssumed();
11839
11840 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11841 llvm::sort(C&: Set);
11842 const std::string KnownStr = llvm::join(R&: Set, Separator: ",");
11843
11844 std::string AssumedStr = "Universal";
11845 if (!Assumed.isUniversal()) {
11846 Set.assign(in_start: Assumed.getSet().begin(), in_end: Assumed.getSet().end());
11847 AssumedStr = llvm::join(R&: Set, Separator: ",");
11848 }
11849 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11850 }
11851};
11852
11853/// Propagates assumption information from parent functions to all of their
11854/// successors. An assumption can be propagated if the containing function
11855/// dominates the called function.
11856///
11857/// We start with a "known" set of assumptions already valid for the associated
11858/// function and an "assumed" set that initially contains all possible
11859/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11860/// contents as concrete values are known. The concrete values are seeded by the
11861/// first nodes that are either entries into the call graph, or contains no
11862/// assumptions. Each node is updated as the intersection of the assumed state
11863/// with all of its predecessors.
11864struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11865 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11866 : AAAssumptionInfoImpl(IRP, A,
11867 getAssumptions(F: *IRP.getAssociatedFunction())) {}
11868
11869 ChangeStatus updateImpl(Attributor &A) override {
11870 bool Changed = false;
11871
11872 auto CallSitePred = [&](AbstractCallSite ACS) {
11873 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11874 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11875 DepClass: DepClassTy::REQUIRED);
11876 if (!AssumptionAA)
11877 return false;
11878 // Get the set of assumptions shared by all of this function's callers.
11879 Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11880 return !getAssumed().empty() || !getKnown().empty();
11881 };
11882
11883 bool UsedAssumedInformation = false;
11884 // Get the intersection of all assumptions held by this node's predecessors.
11885 // If we don't know all the call sites then this is either an entry into the
11886 // call graph or an empty node. This node is known to only contain its own
11887 // assumptions and can be propagated to its successors.
11888 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11889 UsedAssumedInformation))
11890 return indicatePessimisticFixpoint();
11891
11892 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11893 }
11894
11895 void trackStatistics() const override {}
11896};
11897
11898/// Assumption Info defined for call sites.
11899struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11900
11901 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11902 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11903
11904 void initialize(Attributor &A) override {
11905 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11906 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11907 }
11908
11909 ChangeStatus updateImpl(Attributor &A) override {
11910 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11911 auto *AssumptionAA =
11912 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11913 if (!AssumptionAA)
11914 return indicatePessimisticFixpoint();
11915 bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
11916 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11917 }
11918
11919 void trackStatistics() const override {}
11920
11921private:
11922 /// Helper to initialized the known set as all the assumptions this call and
11923 /// the callee contain.
11924 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
11925 const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
11926 auto Assumptions = getAssumptions(CB);
11927 if (const Function *F = CB.getCaller())
11928 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
11929 if (Function *F = IRP.getAssociatedFunction())
11930 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
11931 return Assumptions;
11932 }
11933};
11934} // namespace
11935
11936AACallGraphNode *AACallEdgeIterator::operator*() const {
11937 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
11938 A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
11939}
11940
11941void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
11942
11943/// ------------------------ UnderlyingObjects ---------------------------------
11944
11945namespace {
11946struct AAUnderlyingObjectsImpl
11947 : StateWrapper<BooleanState, AAUnderlyingObjects> {
11948 using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
11949 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
11950
11951 const std::string getAsStr(Attributor *A) const override {
11952 if (!isValidState())
11953 return "<invalid>";
11954 std::string Str;
11955 llvm::raw_string_ostream OS(Str);
11956 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
11957 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
11958 << " objects.\n";
11959 if (!InterAssumedUnderlyingObjects.empty()) {
11960 OS << "inter objects:\n";
11961 for (auto *Obj : InterAssumedUnderlyingObjects)
11962 OS << *Obj << '\n';
11963 }
11964 if (!IntraAssumedUnderlyingObjects.empty()) {
11965 OS << "intra objects:\n";
11966 for (auto *Obj : IntraAssumedUnderlyingObjects)
11967 OS << *Obj << '\n';
11968 }
11969 return Str;
11970 }
11971
11972 void trackStatistics() const override {}
11973
11974 ChangeStatus updateImpl(Attributor &A) override {
11975 auto &Ptr = getAssociatedValue();
11976
11977 bool UsedAssumedInformation = false;
11978 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
11979 AA::ValueScope Scope) {
11980 SmallPtrSet<Value *, 8> SeenObjects;
11981 SmallVector<AA::ValueAndContext> Values;
11982
11983 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
11984 S: Scope, UsedAssumedInformation))
11985 return UnderlyingObjects.insert(X: &Ptr);
11986
11987 bool Changed = false;
11988
11989 for (unsigned I = 0; I < Values.size(); ++I) {
11990 auto &VAC = Values[I];
11991 auto *Obj = VAC.getValue();
11992 Value *UO = getUnderlyingObject(V: Obj);
11993 if (!SeenObjects.insert(Ptr: UO ? UO : Obj).second)
11994 continue;
11995 if (UO && UO != Obj) {
11996 if (isa<AllocaInst>(Val: UO) || isa<GlobalValue>(Val: UO)) {
11997 Changed |= UnderlyingObjects.insert(X: UO);
11998 continue;
11999 }
12000
12001 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12002 QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
12003 auto Pred = [&](Value &V) {
12004 if (&V == UO)
12005 Changed |= UnderlyingObjects.insert(X: UO);
12006 else
12007 Values.emplace_back(Args&: V, Args: nullptr);
12008 return true;
12009 };
12010
12011 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12012 llvm_unreachable(
12013 "The forall call should not return false at this position");
12014 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12015 continue;
12016 }
12017
12018 if (isa<SelectInst>(Val: Obj)) {
12019 Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope,
12020 UsedAssumedInformation);
12021 continue;
12022 }
12023 if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
12024 // Explicitly look through PHIs as we do not care about dynamically
12025 // uniqueness.
12026 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12027 Changed |=
12028 handleIndirect(A, V&: *PHI->getIncomingValue(i: u), UnderlyingObjects,
12029 Scope, UsedAssumedInformation);
12030 }
12031 continue;
12032 }
12033
12034 Changed |= UnderlyingObjects.insert(X: Obj);
12035 }
12036
12037 return Changed;
12038 };
12039
12040 bool Changed = false;
12041 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12042 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12043 if (!UsedAssumedInformation)
12044 indicateOptimisticFixpoint();
12045 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12046 }
12047
12048 bool forallUnderlyingObjects(
12049 function_ref<bool(Value &)> Pred,
12050 AA::ValueScope Scope = AA::Interprocedural) const override {
12051 if (!isValidState())
12052 return Pred(getAssociatedValue());
12053
12054 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12055 ? IntraAssumedUnderlyingObjects
12056 : InterAssumedUnderlyingObjects;
12057 for (Value *Obj : AssumedUnderlyingObjects)
12058 if (!Pred(*Obj))
12059 return false;
12060
12061 return true;
12062 }
12063
12064private:
12065 /// Handle the case where the value is not the actual underlying value, such
12066 /// as a phi node or a select instruction.
12067 bool handleIndirect(Attributor &A, Value &V,
12068 SmallSetVector<Value *, 8> &UnderlyingObjects,
12069 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12070 bool Changed = false;
12071 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12072 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
12073 auto Pred = [&](Value &V) {
12074 Changed |= UnderlyingObjects.insert(X: &V);
12075 return true;
12076 };
12077 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12078 llvm_unreachable(
12079 "The forall call should not return false at this position");
12080 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12081 return Changed;
12082 }
12083
12084 /// All the underlying objects collected so far via intra procedural scope.
12085 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12086 /// All the underlying objects collected so far via inter procedural scope.
12087 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12088};
12089
12090struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12091 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12092 : AAUnderlyingObjectsImpl(IRP, A) {}
12093};
12094
12095struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12096 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12097 : AAUnderlyingObjectsImpl(IRP, A) {}
12098};
12099
12100struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12101 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12102 : AAUnderlyingObjectsImpl(IRP, A) {}
12103};
12104
12105struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12106 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12107 : AAUnderlyingObjectsImpl(IRP, A) {}
12108};
12109
12110struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12111 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12112 : AAUnderlyingObjectsImpl(IRP, A) {}
12113};
12114
12115struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12116 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12117 : AAUnderlyingObjectsImpl(IRP, A) {}
12118};
12119
12120struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12121 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12122 : AAUnderlyingObjectsImpl(IRP, A) {}
12123};
12124} // namespace
12125
12126/// ------------------------ Global Value Info -------------------------------
12127namespace {
12128struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12129 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12130 : AAGlobalValueInfo(IRP, A) {}
12131
12132 void initialize(Attributor &A) override {}
12133
12134 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12135 SmallVectorImpl<const Value *> &Worklist) {
12136 Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12137 if (!UInst) {
12138 // Outside a closed world, code outside the module can read an
12139 // externally visible global, so the value escapes through it.
12140 if (auto *GV = dyn_cast<GlobalValue>(Val: U.getUser()))
12141 if (!GV->hasLocalLinkage() && !A.isClosedWorldModule())
12142 return false;
12143 Follow = true;
12144 return true;
12145 }
12146
12147 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12148 << *UInst << "\n");
12149
12150 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12151 int Idx = &Cmp->getOperandUse(i: 0) == &U;
12152 if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12153 return true;
12154 return U == &getAnchorValue();
12155 }
12156
12157 // Explicitly catch return instructions.
12158 if (isa<ReturnInst>(Val: UInst)) {
12159 auto CallSitePred = [&](AbstractCallSite ACS) {
12160 Worklist.push_back(Elt: ACS.getInstruction());
12161 return true;
12162 };
12163 bool UsedAssumedInformation = false;
12164 // TODO: We should traverse the uses or add a "non-call-site" CB.
12165 if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12166 /*RequireAllCallSites=*/true, QueryingAA: this,
12167 UsedAssumedInformation))
12168 return false;
12169 return true;
12170 }
12171
12172 // For now we only use special logic for call sites. However, the tracker
12173 // itself knows about a lot of other non-capturing cases already.
12174 auto *CB = dyn_cast<CallBase>(Val: UInst);
12175 if (!CB)
12176 return false;
12177 // Direct calls are OK uses.
12178 if (CB->isCallee(U: &U))
12179 return true;
12180 // Non-argument uses are scary.
12181 if (!CB->isArgOperand(U: &U))
12182 return false;
12183 // TODO: Iterate callees.
12184 auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12185 if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12186 return false;
12187
12188 unsigned ArgNo = CB->getArgOperandNo(U: &U);
12189 Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12190 return true;
12191 }
12192
12193 ChangeStatus updateImpl(Attributor &A) override {
12194 unsigned NumUsesBefore = Uses.size();
12195
12196 SmallPtrSet<const Value *, 8> Visited;
12197 SmallVector<const Value *> Worklist;
12198 Worklist.push_back(Elt: &getAnchorValue());
12199
12200 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12201 Uses.insert(Ptr: &U);
12202 // TODO(captures): Make this more precise.
12203 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12204 if (CI.isPassthrough()) {
12205 Follow = true;
12206 return true;
12207 }
12208 return checkUse(A, U, Follow, Worklist);
12209 };
12210 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12211 Uses.insert(Ptr: &OldU);
12212 return true;
12213 };
12214
12215 while (!Worklist.empty()) {
12216 const Value *V = Worklist.pop_back_val();
12217 if (!Visited.insert(Ptr: V).second)
12218 continue;
12219 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12220 /* CheckBBLivenessOnly */ true,
12221 LivenessDepClass: DepClassTy::OPTIONAL,
12222 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12223 return indicatePessimisticFixpoint();
12224 }
12225 }
12226
12227 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12228 : ChangeStatus::CHANGED;
12229 }
12230
12231 bool isPotentialUse(const Use &U) const override {
12232 return !isValidState() || Uses.contains(Ptr: &U);
12233 }
12234
12235 ChangeStatus manifest(Attributor &A) override {
12236 return ChangeStatus::UNCHANGED;
12237 }
12238
12239 const std::string getAsStr(Attributor *A) const override {
12240 return "[" + std::to_string(val: Uses.size()) + " uses]";
12241 }
12242
12243 void trackStatistics() const override {
12244 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12245 }
12246
12247private:
12248 /// Set of (transitive) uses of this GlobalValue.
12249 SmallPtrSet<const Use *, 8> Uses;
12250};
12251} // namespace
12252
12253/// ------------------------ Indirect Call Info -------------------------------
12254namespace {
12255struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12256 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12257 : AAIndirectCallInfo(IRP, A) {}
12258
12259 void initialize(Attributor &A) override {
12260 auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12261 if (!MD && !A.isClosedWorldModule())
12262 return;
12263
12264 if (MD) {
12265 for (const auto &Op : MD->operands())
12266 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12267 PotentialCallees.insert(X: Callee);
12268 } else if (A.isClosedWorldModule()) {
12269 ArrayRef<Function *> IndirectlyCallableFunctions =
12270 A.getInfoCache().getIndirectlyCallableFunctions(A);
12271 PotentialCallees.insert_range(R&: IndirectlyCallableFunctions);
12272 }
12273
12274 if (PotentialCallees.empty())
12275 indicateOptimisticFixpoint();
12276 }
12277
12278 ChangeStatus updateImpl(Attributor &A) override {
12279 CallBase *CB = cast<CallBase>(Val: getCtxI());
12280 const Use &CalleeUse = CB->getCalledOperandUse();
12281 Value *FP = CB->getCalledOperand();
12282
12283 SmallSetVector<Function *, 4> AssumedCalleesNow;
12284 bool AllCalleesKnownNow = AllCalleesKnown;
12285
12286 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12287 bool &UsedAssumedInformation) {
12288 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12289 QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12290 if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12291 return true;
12292 UsedAssumedInformation = !GIAA->isAtFixpoint();
12293 return false;
12294 };
12295
12296 auto AddPotentialCallees = [&]() {
12297 for (auto *PotentialCallee : PotentialCallees) {
12298 bool UsedAssumedInformation = false;
12299 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12300 AssumedCalleesNow.insert(X: PotentialCallee);
12301 }
12302 };
12303
12304 // Use simplification to find potential callees, if !callees was present,
12305 // fallback to that set if necessary.
12306 bool UsedAssumedInformation = false;
12307 SmallVector<AA::ValueAndContext> Values;
12308 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12309 S: AA::ValueScope::AnyScope,
12310 UsedAssumedInformation)) {
12311 if (PotentialCallees.empty())
12312 return indicatePessimisticFixpoint();
12313 AddPotentialCallees();
12314 }
12315
12316 // Try to find a reason for \p Fn not to be a potential callee. If none was
12317 // found, add it to the assumed callees set.
12318 auto CheckPotentialCallee = [&](Function &Fn) {
12319 if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12320 return false;
12321
12322 auto &CachedResult = FilterResults[&Fn];
12323 if (CachedResult.has_value())
12324 return CachedResult.value();
12325
12326 bool UsedAssumedInformation = false;
12327 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12328 if (!UsedAssumedInformation)
12329 CachedResult = false;
12330 return false;
12331 }
12332
12333 int NumFnArgs = Fn.arg_size();
12334 int NumCBArgs = CB->arg_size();
12335
12336 // Check if any excess argument (which we fill up with poison) is known to
12337 // be UB on undef.
12338 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12339 bool IsKnown = false;
12340 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12341 A, QueryingAA: this, IRP: IRPosition::argument(Arg: *Fn.getArg(i: I)),
12342 DepClass: DepClassTy::OPTIONAL, IsKnown)) {
12343 if (IsKnown)
12344 CachedResult = false;
12345 return false;
12346 }
12347 }
12348
12349 CachedResult = true;
12350 return true;
12351 };
12352
12353 // Check simplification result, prune known UB callees, also restrict it to
12354 // the !callees set, if present.
12355 for (auto &VAC : Values) {
12356 if (isa<UndefValue>(Val: VAC.getValue()))
12357 continue;
12358 if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12359 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12360 continue;
12361 // TODO: Check for known UB, e.g., poison + noundef.
12362 if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12363 if (CheckPotentialCallee(*VACFn))
12364 AssumedCalleesNow.insert(X: VACFn);
12365 continue;
12366 }
12367 if (!PotentialCallees.empty()) {
12368 AddPotentialCallees();
12369 break;
12370 }
12371 AllCalleesKnownNow = false;
12372 }
12373
12374 if (AssumedCalleesNow == AssumedCallees &&
12375 AllCalleesKnown == AllCalleesKnownNow)
12376 return ChangeStatus::UNCHANGED;
12377
12378 std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12379 AllCalleesKnown = AllCalleesKnownNow;
12380 return ChangeStatus::CHANGED;
12381 }
12382
12383 ChangeStatus manifest(Attributor &A) override {
12384 // If we can't specialize at all, give up now.
12385 if (!AllCalleesKnown && AssumedCallees.empty())
12386 return ChangeStatus::UNCHANGED;
12387
12388 CallBase *CB = cast<CallBase>(Val: getCtxI());
12389 bool UsedAssumedInformation = false;
12390 if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12391 UsedAssumedInformation))
12392 return ChangeStatus::UNCHANGED;
12393
12394 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12395 unsigned ProgramAS = CB->getDataLayout().getProgramAddressSpace();
12396 Value *FP = CB->getCalledOperand();
12397 if (FP->getType()->getPointerAddressSpace() != ProgramAS)
12398 FP = new AddrSpaceCastInst(
12399 FP, PointerType::get(C&: FP->getContext(), AddressSpace: ProgramAS),
12400 FP->getName() + ".as" + Twine(ProgramAS), CB->getIterator());
12401
12402 bool CBIsVoid = CB->getType()->isVoidTy();
12403 BasicBlock::iterator IP = CB->getIterator();
12404 FunctionType *CSFT = CB->getFunctionType();
12405 SmallVector<Value *> CSArgs(CB->args());
12406
12407 // If we know all callees and there are none, the call site is (effectively)
12408 // dead (or UB).
12409 if (AssumedCallees.empty()) {
12410 assert(AllCalleesKnown &&
12411 "Expected all callees to be known if there are none.");
12412 A.changeToUnreachableAfterManifest(I: CB);
12413 return ChangeStatus::CHANGED;
12414 }
12415
12416 // Special handling for the single callee case.
12417 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12418 auto *NewCallee = AssumedCallees.front();
12419 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12420 promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12421 NumIndirectCallsPromoted++;
12422 return ChangeStatus::CHANGED;
12423 }
12424 Instruction *NewCall =
12425 CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12426 NameStr: CB->getName(), InsertBefore: CB->getIterator());
12427 NewCall->setDebugLoc(CB->getDebugLoc());
12428 if (!CBIsVoid)
12429 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12430 A.deleteAfterManifest(I&: *CB);
12431 return ChangeStatus::CHANGED;
12432 }
12433
12434 // For each potential value we create a conditional
12435 //
12436 // ```
12437 // if (ptr == value) value(args);
12438 // else ...
12439 // ```
12440 //
12441 bool SpecializedForAnyCallees = false;
12442 bool SpecializedForAllCallees = AllCalleesKnown;
12443 ICmpInst *LastCmp = nullptr;
12444 SmallVector<Function *, 8> SkippedAssumedCallees;
12445 SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12446 for (Function *NewCallee : AssumedCallees) {
12447 if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee,
12448 NumAssumedCallees: AssumedCallees.size())) {
12449 SkippedAssumedCallees.push_back(Elt: NewCallee);
12450 SpecializedForAllCallees = false;
12451 continue;
12452 }
12453 SpecializedForAnyCallees = true;
12454
12455 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12456 Instruction *ThenTI =
12457 SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12458 BasicBlock *CBBB = CB->getParent();
12459 A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12460 A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12461 auto *SplitTI = cast<CondBrInst>(Val: LastCmp->getNextNode());
12462 BasicBlock *ElseBB;
12463 if (&*IP == CB) {
12464 ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12465 Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12466 A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12467 IP = UncondBrInst::Create(Target: CBBB, InsertBefore: ElseBB)->getIterator();
12468 SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12469 } else {
12470 ElseBB = IP->getParent();
12471 ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12472 }
12473 CastInst *RetBC = nullptr;
12474 CallInst *NewCall = nullptr;
12475 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12476 auto *CBClone = cast<CallBase>(Val: CB->clone());
12477 CBClone->insertBefore(InsertPos: ThenTI->getIterator());
12478 NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12479 NumIndirectCallsPromoted++;
12480 } else {
12481 NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12482 NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12483 NewCall->setDebugLoc(CB->getDebugLoc());
12484 }
12485 NewCalls.push_back(Elt: {NewCall, RetBC});
12486 }
12487
12488 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12489 if (!AllCalleesKnown)
12490 return ChangeStatus::UNCHANGED;
12491 MDBuilder MDB(IndirectCB.getContext());
12492 MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12493 IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12494 return ChangeStatus::CHANGED;
12495 };
12496
12497 if (!SpecializedForAnyCallees)
12498 return AttachCalleeMetadata(*CB);
12499
12500 // Check if we need the fallback indirect call still.
12501 if (SpecializedForAllCallees) {
12502 LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12503 LastCmp->eraseFromParent();
12504 new UnreachableInst(IP->getContext(), IP);
12505 IP->eraseFromParent();
12506 } else {
12507 auto *CBClone = cast<CallInst>(Val: CB->clone());
12508 CBClone->setName(CB->getName());
12509 CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12510 NewCalls.push_back(Elt: {CBClone, nullptr});
12511 AttachCalleeMetadata(*CBClone);
12512 }
12513
12514 // Check if we need a PHI to merge the results.
12515 if (!CBIsVoid) {
12516 auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12517 NameStr: CB->getName() + ".phi",
12518 InsertBefore: CB->getParent()->getFirstInsertionPt());
12519 for (auto &It : NewCalls) {
12520 CallBase *NewCall = It.first;
12521 Instruction *CallRet = It.second ? It.second : It.first;
12522 if (CallRet->getType() == CB->getType())
12523 PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12524 else if (NewCall->getType()->isVoidTy())
12525 PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12526 BB: NewCall->getParent());
12527 else
12528 llvm_unreachable("Call return should match or be void!");
12529 }
12530 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12531 }
12532
12533 A.deleteAfterManifest(I&: *CB);
12534 Changed = ChangeStatus::CHANGED;
12535
12536 return Changed;
12537 }
12538
12539 const std::string getAsStr(Attributor *A) const override {
12540 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12541 " indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12542 " functions";
12543 }
12544
12545 void trackStatistics() const override {
12546 if (AllCalleesKnown) {
12547 STATS_DECLTRACK(
12548 Eliminated, CallSites,
12549 "Number of indirect call sites eliminated via specialization")
12550 } else {
12551 STATS_DECLTRACK(Specialized, CallSites,
12552 "Number of indirect call sites specialized")
12553 }
12554 }
12555
12556 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12557 return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12558 }
12559
12560private:
12561 /// Map to remember filter results.
12562 DenseMap<Function *, std::optional<bool>> FilterResults;
12563
12564 /// If the !callee metadata was present, this set will contain all potential
12565 /// callees (superset).
12566 SmallSetVector<Function *, 4> PotentialCallees;
12567
12568 /// This set contains all currently assumed calllees, which might grow over
12569 /// time.
12570 SmallSetVector<Function *, 4> AssumedCallees;
12571
12572 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12573 /// if there could be others.
12574 bool AllCalleesKnown = true;
12575};
12576} // namespace
12577
12578/// --------------------- Invariant Load Pointer -------------------------------
12579namespace {
12580
12581struct AAInvariantLoadPointerImpl
12582 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12583 AAInvariantLoadPointer> {
12584
12585 enum {
12586 // pointer does not alias within the bounds of the function
12587 IS_NOALIAS = 1 << 0,
12588 // pointer is not involved in any effectful instructions within the bounds
12589 // of the function
12590 IS_NOEFFECT = 1 << 1,
12591 // loads are invariant within the bounds of the function
12592 IS_LOCALLY_INVARIANT = 1 << 2,
12593 // memory lifetime is constrained within the bounds of the function
12594 IS_LOCALLY_CONSTRAINED = 1 << 3,
12595
12596 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12597 IS_LOCALLY_CONSTRAINED,
12598 };
12599 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12600
12601 using Base =
12602 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12603
12604 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12605 // pessimistic about IS_KNOWN_INVARIANT
12606 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12607 : Base(IRP) {}
12608
12609 bool isKnownInvariant() const final {
12610 return isKnownLocallyInvariant() && isKnown(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12611 }
12612
12613 bool isKnownLocallyInvariant() const final {
12614 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT))
12615 return true;
12616 return isKnown(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12617 }
12618
12619 bool isAssumedInvariant() const final {
12620 return isAssumedLocallyInvariant() && isAssumed(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12621 }
12622
12623 bool isAssumedLocallyInvariant() const final {
12624 if (isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12625 return true;
12626 return isAssumed(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12627 }
12628
12629 ChangeStatus updateImpl(Attributor &A) override {
12630 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12631
12632 Changed |= updateNoAlias(A);
12633 if (requiresNoAlias() && !isAssumed(BitsEncoding: IS_NOALIAS))
12634 return indicatePessimisticFixpoint();
12635
12636 Changed |= updateNoEffect(A);
12637
12638 Changed |= updateLocalInvariance(A);
12639
12640 return Changed;
12641 }
12642
12643 ChangeStatus manifest(Attributor &A) override {
12644 if (!isKnownInvariant())
12645 return ChangeStatus::UNCHANGED;
12646
12647 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12648 const Value *Ptr = &getAssociatedValue();
12649 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12650 if (U.get() != Ptr)
12651 return true;
12652 auto *I = dyn_cast<Instruction>(Val: U.getUser());
12653 if (!I)
12654 return true;
12655
12656 // Ensure that we are only changing uses from the corresponding callgraph
12657 // SSC in the case that the AA isn't run on the entire module
12658 if (!A.isRunOn(Fn: I->getFunction()))
12659 return true;
12660
12661 if (I->hasMetadata(KindID: LLVMContext::MD_invariant_load))
12662 return true;
12663
12664 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
12665 LI->setMetadata(KindID: LLVMContext::MD_invariant_load,
12666 Node: MDNode::get(Context&: LI->getContext(), MDs: {}));
12667 Changed = ChangeStatus::CHANGED;
12668 }
12669 return true;
12670 };
12671
12672 (void)A.checkForAllUses(Pred: TagInvariantLoads, QueryingAA: *this, V: *Ptr);
12673 return Changed;
12674 }
12675
12676 const std::string getAsStr(Attributor *) const override {
12677 if (isKnownInvariant())
12678 return "load-invariant pointer";
12679 return "non-invariant pointer";
12680 }
12681
12682 void trackStatistics() const override {}
12683
12684private:
12685 /// Indicate that noalias is required for the pointer to be invariant.
12686 bool requiresNoAlias() const {
12687 switch (getPositionKind()) {
12688 default:
12689 // Conservatively default to require noalias.
12690 return true;
12691 case IRP_FLOAT:
12692 case IRP_RETURNED:
12693 case IRP_CALL_SITE:
12694 return false;
12695 case IRP_CALL_SITE_RETURNED: {
12696 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12697 return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12698 Call: &CB, /*MustPreserveOffset=*/false);
12699 }
12700 case IRP_ARGUMENT: {
12701 const Function *F = getAssociatedFunction();
12702 assert(F && "no associated function for argument");
12703 return !isCallableCC(CC: F->getCallingConv());
12704 }
12705 }
12706 }
12707
12708 bool isExternal() const {
12709 const Function *F = getAssociatedFunction();
12710 if (!F)
12711 return true;
12712 return isCallableCC(CC: F->getCallingConv()) &&
12713 getPositionKind() != IRP_CALL_SITE_RETURNED;
12714 }
12715
12716 ChangeStatus updateNoAlias(Attributor &A) {
12717 if (isKnown(BitsEncoding: IS_NOALIAS) || !isAssumed(BitsEncoding: IS_NOALIAS))
12718 return ChangeStatus::UNCHANGED;
12719
12720 // Try to use AANoAlias.
12721 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12722 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12723 if (ANoAlias->isKnownNoAlias()) {
12724 addKnownBits(Bits: IS_NOALIAS);
12725 return ChangeStatus::CHANGED;
12726 }
12727
12728 if (!ANoAlias->isAssumedNoAlias()) {
12729 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12730 return ChangeStatus::CHANGED;
12731 }
12732
12733 return ChangeStatus::UNCHANGED;
12734 }
12735
12736 // Try to infer noalias from argument attribute, since it is applicable for
12737 // the duration of the function.
12738 if (const Argument *Arg = getAssociatedArgument()) {
12739 if (Arg->hasNoAliasAttr()) {
12740 addKnownBits(Bits: IS_NOALIAS);
12741 return ChangeStatus::UNCHANGED;
12742 }
12743
12744 // Noalias information is not provided, and cannot be inferred,
12745 // so we conservatively assume the pointer aliases.
12746 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12747 return ChangeStatus::CHANGED;
12748 }
12749
12750 return ChangeStatus::UNCHANGED;
12751 }
12752
12753 ChangeStatus updateNoEffect(Attributor &A) {
12754 if (isKnown(BitsEncoding: IS_NOEFFECT) || !isAssumed(BitsEncoding: IS_NOEFFECT))
12755 return ChangeStatus::UNCHANGED;
12756
12757 if (!getAssociatedFunction())
12758 return indicatePessimisticFixpoint();
12759
12760 if (isa<AllocaInst>(Val: &getAssociatedValue()))
12761 return indicatePessimisticFixpoint();
12762
12763 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12764 const auto *LI = dyn_cast<LoadInst>(Val: U.getUser());
12765 return !LI || !LI->mayHaveSideEffects();
12766 };
12767 if (!A.checkForAllUses(Pred: HasNoEffectLoads, QueryingAA: *this, V: getAssociatedValue()))
12768 return indicatePessimisticFixpoint();
12769
12770 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12771 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12772 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12773 // attribute
12774 if (!AMemoryBehavior->isAssumedReadOnly())
12775 return indicatePessimisticFixpoint();
12776
12777 if (AMemoryBehavior->isKnownReadOnly()) {
12778 addKnownBits(Bits: IS_NOEFFECT);
12779 return ChangeStatus::UNCHANGED;
12780 }
12781
12782 return ChangeStatus::UNCHANGED;
12783 }
12784
12785 if (const Argument *Arg = getAssociatedArgument()) {
12786 if (Arg->onlyReadsMemory()) {
12787 addKnownBits(Bits: IS_NOEFFECT);
12788 return ChangeStatus::UNCHANGED;
12789 }
12790
12791 // Readonly information is not provided, and cannot be inferred from
12792 // AAMemoryBehavior.
12793 return indicatePessimisticFixpoint();
12794 }
12795
12796 return ChangeStatus::UNCHANGED;
12797 }
12798
12799 ChangeStatus updateLocalInvariance(Attributor &A) {
12800 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT) || !isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12801 return ChangeStatus::UNCHANGED;
12802
12803 // try to infer invariance from underlying objects
12804 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12805 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
12806 if (!AUO)
12807 return ChangeStatus::UNCHANGED;
12808
12809 bool UsedAssumedInformation = false;
12810 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12811 if (!V.getType()->isPointerTy())
12812 return true;
12813 const auto *IsInvariantLoadPointer =
12814 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRP: IRPosition::value(V), QueryingAA: this,
12815 DepClass: DepClassTy::REQUIRED);
12816 // Conservatively fail if invariance cannot be inferred.
12817 if (!IsInvariantLoadPointer)
12818 return false;
12819
12820 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12821 return true;
12822 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12823 return false;
12824
12825 UsedAssumedInformation = true;
12826 return true;
12827 };
12828 if (!AUO->forallUnderlyingObjects(Pred: IsLocallyInvariantLoadIfPointer))
12829 return indicatePessimisticFixpoint();
12830
12831 if (const auto *CB = dyn_cast<CallBase>(Val: &getAnchorValue())) {
12832 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12833 Call: CB, /*MustPreserveOffset=*/false)) {
12834 for (const Value *Arg : CB->args()) {
12835 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12836 return indicatePessimisticFixpoint();
12837 }
12838 }
12839 }
12840
12841 if (!UsedAssumedInformation) {
12842 // Pointer is known and not just assumed to be locally invariant.
12843 addKnownBits(Bits: IS_LOCALLY_INVARIANT);
12844 return ChangeStatus::CHANGED;
12845 }
12846
12847 return ChangeStatus::UNCHANGED;
12848 }
12849};
12850
12851struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12852 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12853 : AAInvariantLoadPointerImpl(IRP, A) {}
12854};
12855
12856struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12857 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12858 : AAInvariantLoadPointerImpl(IRP, A) {}
12859
12860 void initialize(Attributor &) override {
12861 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12862 }
12863};
12864
12865struct AAInvariantLoadPointerCallSiteReturned final
12866 : AAInvariantLoadPointerImpl {
12867 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12868 : AAInvariantLoadPointerImpl(IRP, A) {}
12869
12870 void initialize(Attributor &A) override {
12871 // Nothing is known about the result of an indirect call.
12872 const Function *F = getAssociatedFunction();
12873 if (!F) {
12874 indicatePessimisticFixpoint();
12875 return;
12876 }
12877
12878 if (!F->isDeclaration() && !F->isIntrinsic())
12879 return AAInvariantLoadPointerImpl::initialize(A);
12880
12881 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12882 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12883 Call: &CB, /*MustPreserveOffset=*/false))
12884 return AAInvariantLoadPointerImpl::initialize(A);
12885
12886 if (F->onlyReadsMemory() && F->hasNoSync())
12887 return AAInvariantLoadPointerImpl::initialize(A);
12888
12889 // At this point, the function is opaque, so we conservatively assume
12890 // non-invariance.
12891 indicatePessimisticFixpoint();
12892 }
12893};
12894
12895struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
12896 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
12897 : AAInvariantLoadPointerImpl(IRP, A) {}
12898
12899 void initialize(Attributor &) override {
12900 const Function *F = getAssociatedFunction();
12901 assert(F && "no associated function for argument");
12902
12903 if (!isCallableCC(CC: F->getCallingConv())) {
12904 addKnownBits(Bits: IS_LOCALLY_CONSTRAINED);
12905 return;
12906 }
12907
12908 if (!F->hasLocalLinkage())
12909 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12910 }
12911};
12912
12913struct AAInvariantLoadPointerCallSiteArgument final
12914 : AAInvariantLoadPointerImpl {
12915 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
12916 : AAInvariantLoadPointerImpl(IRP, A) {}
12917};
12918} // namespace
12919
12920/// ------------------------ Address Space ------------------------------------
12921namespace {
12922
12923template <typename InstType>
12924static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
12925 Value *OriginalValue, PointerType *NewPtrTy,
12926 bool UseOriginalValue) {
12927 if (U.getOperandNo() != InstType::getPointerOperandIndex())
12928 return false;
12929
12930 if (MemInst->isVolatile()) {
12931 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
12932 *MemInst->getFunction());
12933 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
12934 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
12935 return false;
12936 }
12937
12938 if (UseOriginalValue) {
12939 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *OriginalValue);
12940 return true;
12941 }
12942
12943 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
12944 CastInst->insertBefore(MemInst->getIterator());
12945 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *CastInst);
12946 return true;
12947}
12948
12949struct AAAddressSpaceImpl : public AAAddressSpace {
12950 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
12951 : AAAddressSpace(IRP, A) {}
12952
12953 uint32_t getAddressSpace() const override {
12954 assert(isValidState() && "the AA is invalid");
12955 return AssumedAddressSpace;
12956 }
12957
12958 void initialize(Attributor &A) override {
12959 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
12960 "Associated value is not a pointer");
12961
12962 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
12963 indicatePessimisticFixpoint();
12964 return;
12965 }
12966
12967 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
12968 unsigned AS = getAssociatedType()->getPointerAddressSpace();
12969 if (AS != FlatAS) {
12970 [[maybe_unused]] bool R = takeAddressSpace(AS);
12971 assert(R && "The take should happen");
12972 indicateOptimisticFixpoint();
12973 }
12974 }
12975
12976 ChangeStatus updateImpl(Attributor &A) override {
12977 uint32_t OldAddressSpace = AssumedAddressSpace;
12978 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
12979
12980 auto CheckAddressSpace = [&](Value &Obj) {
12981 // Ignore undef.
12982 if (isa<UndefValue>(Val: &Obj))
12983 return true;
12984
12985 // If the object already has a non-flat address space, we simply take it.
12986 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
12987 if (ObjAS != FlatAS)
12988 return takeAddressSpace(AS: ObjAS);
12989
12990 // At this point, we know Obj is in the flat address space. For a final
12991 // attempt, we want to use getAssumedAddrSpace, but first we must get the
12992 // associated function, if possible.
12993 Function *F = nullptr;
12994 if (auto *Arg = dyn_cast<Argument>(Val: &Obj))
12995 F = Arg->getParent();
12996 else if (auto *I = dyn_cast<Instruction>(Val: &Obj))
12997 F = I->getFunction();
12998
12999 // Use getAssumedAddrSpace if the associated function exists.
13000 if (F) {
13001 auto *TTI =
13002 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: *F);
13003 unsigned AssumedAS = TTI->getAssumedAddrSpace(V: &Obj);
13004 if (AssumedAS != ~0U)
13005 return takeAddressSpace(AS: AssumedAS);
13006 }
13007
13008 // Now we can't do anything else but to take the flat AS.
13009 return takeAddressSpace(AS: FlatAS);
13010 };
13011
13012 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
13013 DepClass: DepClassTy::REQUIRED);
13014 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13015 return indicatePessimisticFixpoint();
13016
13017 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13018 : ChangeStatus::CHANGED;
13019 }
13020
13021 ChangeStatus manifest(Attributor &A) override {
13022 unsigned NewAS = getAddressSpace();
13023
13024 if (NewAS == InvalidAddressSpace ||
13025 NewAS == getAssociatedType()->getPointerAddressSpace())
13026 return ChangeStatus::UNCHANGED;
13027
13028 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13029
13030 Value *AssociatedValue = &getAssociatedValue();
13031 Value *OriginalValue = peelAddrspacecast(V: AssociatedValue, FlatAS);
13032
13033 PointerType *NewPtrTy =
13034 PointerType::get(C&: getAssociatedType()->getContext(), AddressSpace: NewAS);
13035 bool UseOriginalValue =
13036 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13037
13038 bool Changed = false;
13039
13040 auto Pred = [&](const Use &U, bool &) {
13041 if (U.get() != AssociatedValue)
13042 return true;
13043 auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
13044 if (!Inst)
13045 return true;
13046 // This is a WA to make sure we only change uses from the corresponding
13047 // CGSCC if the AA is run on CGSCC instead of the entire module.
13048 if (!A.isRunOn(Fn: Inst->getFunction()))
13049 return true;
13050 if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
13051 Changed |=
13052 makeChange(A, MemInst: LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13053 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
13054 Changed |=
13055 makeChange(A, MemInst: SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13056 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
13057 Changed |=
13058 makeChange(A, MemInst: RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13059 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
13060 Changed |=
13061 makeChange(A, MemInst: CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13062 }
13063 return true;
13064 };
13065
13066 // It doesn't matter if we can't check all uses as we can simply
13067 // conservatively ignore those that can not be visited.
13068 (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
13069 /* CheckBBLivenessOnly */ true);
13070
13071 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13072 }
13073
13074 const std::string getAsStr(Attributor *A) const override {
13075 if (!isValidState())
13076 return "addrspace(<invalid>)";
13077 return "addrspace(" +
13078 (AssumedAddressSpace == InvalidAddressSpace
13079 ? "none"
13080 : std::to_string(val: AssumedAddressSpace)) +
13081 ")";
13082 }
13083
13084private:
13085 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13086
13087 bool takeAddressSpace(uint32_t AS) {
13088 if (AssumedAddressSpace == InvalidAddressSpace) {
13089 AssumedAddressSpace = AS;
13090 return true;
13091 }
13092 return AssumedAddressSpace == AS;
13093 }
13094
13095 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13096 if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) {
13097 assert(I->getSrcAddressSpace() != FlatAS &&
13098 "there should not be flat AS -> non-flat AS");
13099 return I->getPointerOperand();
13100 }
13101 if (auto *C = dyn_cast<ConstantExpr>(Val: V))
13102 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13103 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13104 FlatAS &&
13105 "there should not be flat AS -> non-flat AS X");
13106 return C->getOperand(i_nocapture: 0);
13107 }
13108 return V;
13109 }
13110};
13111
13112struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13113 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13114 : AAAddressSpaceImpl(IRP, A) {}
13115
13116 void trackStatistics() const override {
13117 STATS_DECLTRACK_FLOATING_ATTR(addrspace);
13118 }
13119};
13120
13121struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13122 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13123 : AAAddressSpaceImpl(IRP, A) {}
13124
13125 void initialize(Attributor &A) override {
13126 // TODO: we don't rewrite function argument for now because it will need to
13127 // rewrite the function signature and all call sites.
13128 (void)indicatePessimisticFixpoint();
13129 }
13130
13131 void trackStatistics() const override {
13132 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13133 }
13134};
13135
13136struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13137 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13138 : AAAddressSpaceImpl(IRP, A) {}
13139
13140 void trackStatistics() const override {
13141 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13142 }
13143};
13144
13145struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13146 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13147 : AAAddressSpaceImpl(IRP, A) {}
13148
13149 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13150};
13151
13152struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13153 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13154 : AAAddressSpaceImpl(IRP, A) {}
13155
13156 void initialize(Attributor &A) override {
13157 // TODO: we don't rewrite call site argument for now because it will need to
13158 // rewrite the function signature of the callee.
13159 (void)indicatePessimisticFixpoint();
13160 }
13161
13162 void trackStatistics() const override {
13163 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13164 }
13165};
13166} // namespace
13167
13168/// ------------------------ No Alias Address Space ---------------------------
13169// This attribute assumes flat address space can alias all other address space
13170
13171// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13172// But merging it created failing cased on gateway test that cannot be
13173// reproduced locally. So should open a separated PR to handle the merge of
13174// AANoAliasAddrSpace and AAAddressSpace attribute
13175
13176namespace {
13177struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13178 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13179 : AANoAliasAddrSpace(IRP, A) {}
13180
13181 void initialize(Attributor &A) override {
13182 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13183 "Associated value is not a pointer");
13184
13185 resetASRanges(A);
13186
13187 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13188 if (!FlatAS.has_value()) {
13189 indicatePessimisticFixpoint();
13190 return;
13191 }
13192
13193 removeAS(AS: *FlatAS);
13194
13195 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13196 if (AS != *FlatAS) {
13197 removeAS(AS);
13198 indicateOptimisticFixpoint();
13199 }
13200 }
13201
13202 ChangeStatus updateImpl(Attributor &A) override {
13203 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13204 uint32_t OldAssumed = getAssumed();
13205
13206 auto CheckAddressSpace = [&](Value &Obj) {
13207 if (isa<PoisonValue>(Val: &Obj))
13208 return true;
13209
13210 unsigned AS = Obj.getType()->getPointerAddressSpace();
13211 if (AS == FlatAS)
13212 return false;
13213
13214 removeAS(AS: Obj.getType()->getPointerAddressSpace());
13215 return true;
13216 };
13217
13218 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13219 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
13220 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13221 return indicatePessimisticFixpoint();
13222
13223 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13224 : ChangeStatus::CHANGED;
13225 }
13226
13227 ChangeStatus manifest(Attributor &A) override {
13228 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13229
13230 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13231 if (AS != FlatAS || Map.empty())
13232 return ChangeStatus::UNCHANGED;
13233
13234 LLVMContext &Ctx = getAssociatedValue().getContext();
13235 MDNode *NoAliasASNode = nullptr;
13236 MDBuilder MDB(Ctx);
13237 // Has to use iterator to get the range info.
13238 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13239 if (!I.value())
13240 continue;
13241 unsigned Upper = I.stop();
13242 unsigned Lower = I.start();
13243 if (!NoAliasASNode) {
13244 NoAliasASNode = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13245 continue;
13246 }
13247 MDNode *ASRange = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13248 NoAliasASNode = MDNode::getMostGenericRange(A: NoAliasASNode, B: ASRange);
13249 }
13250
13251 Value *AssociatedValue = &getAssociatedValue();
13252 bool Changed = false;
13253
13254 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13255 if (U.get() != AssociatedValue)
13256 return true;
13257 Instruction *Inst = dyn_cast<Instruction>(Val: U.getUser());
13258 if (!Inst || Inst->hasMetadata(KindID: LLVMContext::MD_noalias_addrspace))
13259 return true;
13260 if (!isa<LoadInst>(Val: Inst) && !isa<StoreInst>(Val: Inst) &&
13261 !isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
13262 return true;
13263 if (!A.isRunOn(Fn: Inst->getFunction()))
13264 return true;
13265 Inst->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: NoAliasASNode);
13266 Changed = true;
13267 return true;
13268 };
13269 (void)A.checkForAllUses(Pred: AddNoAliasAttr, QueryingAA: *this, V: *AssociatedValue,
13270 /*CheckBBLivenessOnly=*/true);
13271 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13272 }
13273
13274 const std::string getAsStr(Attributor *A) const override {
13275 if (!isValidState())
13276 return "<invalid>";
13277 std::string Str;
13278 raw_string_ostream OS(Str);
13279 OS << "CanNotBeAddrSpace(";
13280 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13281 unsigned Upper = I.stop();
13282 unsigned Lower = I.start();
13283 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13284 }
13285 OS << " )";
13286 return OS.str();
13287 }
13288
13289private:
13290 void removeAS(unsigned AS) {
13291 RangeMap::iterator I = Map.find(x: AS);
13292
13293 if (I != Map.end()) {
13294 unsigned Upper = I.stop();
13295 unsigned Lower = I.start();
13296 I.erase();
13297 if (Upper == Lower)
13298 return;
13299 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13300 Map.insert(a: AS + 1, b: Upper, /*what ever this variable name is=*/y: true);
13301 if (AS != 0 && Lower <= AS - 1)
13302 Map.insert(a: Lower, b: AS - 1, y: true);
13303 }
13304 }
13305
13306 void resetASRanges(Attributor &A) {
13307 Map.clear();
13308 Map.insert(a: 0, b: A.getInfoCache().getMaxAddrSpace(), y: true);
13309 }
13310};
13311
13312struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13313 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13314 : AANoAliasAddrSpaceImpl(IRP, A) {}
13315
13316 void trackStatistics() const override {
13317 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13318 }
13319};
13320
13321struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13322 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13323 : AANoAliasAddrSpaceImpl(IRP, A) {}
13324
13325 void trackStatistics() const override {
13326 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13327 }
13328};
13329
13330struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13331 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13332 : AANoAliasAddrSpaceImpl(IRP, A) {}
13333
13334 void trackStatistics() const override {
13335 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13336 }
13337};
13338
13339struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13340 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13341 : AANoAliasAddrSpaceImpl(IRP, A) {}
13342
13343 void trackStatistics() const override {
13344 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13345 }
13346};
13347
13348struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13349 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13350 : AANoAliasAddrSpaceImpl(IRP, A) {}
13351
13352 void trackStatistics() const override {
13353 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13354 }
13355};
13356} // namespace
13357/// ----------- Allocation Info ----------
13358namespace {
13359struct AAAllocationInfoImpl : public AAAllocationInfo {
13360 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13361 : AAAllocationInfo(IRP, A) {}
13362
13363 std::optional<TypeSize> getAllocatedSize() const override {
13364 assert(isValidState() && "the AA is invalid");
13365 return AssumedAllocatedSize;
13366 }
13367
13368 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13369 const DataLayout &DL) {
13370
13371 // TODO: implement case for malloc like instructions
13372 switch (I->getOpcode()) {
13373 case Instruction::Alloca: {
13374 AllocaInst *AI = cast<AllocaInst>(Val: I);
13375 return AI->getAllocationSize(DL);
13376 }
13377 default:
13378 return std::nullopt;
13379 }
13380 }
13381
13382 ChangeStatus updateImpl(Attributor &A) override {
13383
13384 const IRPosition &IRP = getIRPosition();
13385 Instruction *I = IRP.getCtxI();
13386
13387 // TODO: update check for malloc like calls
13388 if (!isa<AllocaInst>(Val: I))
13389 return indicatePessimisticFixpoint();
13390
13391 bool IsKnownNoCapture;
13392 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
13393 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
13394 return indicatePessimisticFixpoint();
13395
13396 const AAPointerInfo *PI =
13397 A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
13398
13399 if (!PI)
13400 return indicatePessimisticFixpoint();
13401
13402 if (!PI->getState().isValidState() || PI->reachesReturn())
13403 return indicatePessimisticFixpoint();
13404
13405 const DataLayout &DL = A.getDataLayout();
13406 const auto AllocationSize = findInitialAllocationSize(I, DL);
13407
13408 // If allocation size is nullopt, we give up.
13409 if (!AllocationSize)
13410 return indicatePessimisticFixpoint();
13411
13412 // For zero sized allocations, we give up.
13413 // Since we can't reduce further
13414 if (*AllocationSize == 0)
13415 return indicatePessimisticFixpoint();
13416
13417 int64_t BinSize = PI->numOffsetBins();
13418
13419 // TODO: implement for multiple bins
13420 if (BinSize > 1)
13421 return indicatePessimisticFixpoint();
13422
13423 if (BinSize == 0) {
13424 auto NewAllocationSize = std::make_optional<TypeSize>(args: 0, args: false);
13425 if (!changeAllocationSize(Size: NewAllocationSize))
13426 return ChangeStatus::UNCHANGED;
13427 return ChangeStatus::CHANGED;
13428 }
13429
13430 // TODO: refactor this to be part of multiple bin case
13431 const auto &It = PI->begin();
13432
13433 // TODO: handle if Offset is not zero
13434 if (It->first.Offset != 0)
13435 return indicatePessimisticFixpoint();
13436
13437 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13438
13439 if (SizeOfBin >= *AllocationSize)
13440 return indicatePessimisticFixpoint();
13441
13442 auto NewAllocationSize = std::make_optional<TypeSize>(args: SizeOfBin * 8, args: false);
13443
13444 if (!changeAllocationSize(Size: NewAllocationSize))
13445 return ChangeStatus::UNCHANGED;
13446
13447 return ChangeStatus::CHANGED;
13448 }
13449
13450 ChangeStatus manifest(Attributor &A) override {
13451
13452 assert(isValidState() &&
13453 "Manifest should only be called if the state is valid.");
13454
13455 Instruction *I = getIRPosition().getCtxI();
13456
13457 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13458
13459 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13460
13461 switch (I->getOpcode()) {
13462 // TODO: add case for malloc like calls
13463 case Instruction::Alloca: {
13464
13465 AllocaInst *AI = cast<AllocaInst>(Val: I);
13466
13467 Type *CharType = Type::getInt8Ty(C&: I->getContext());
13468
13469 auto *NumBytesToValue =
13470 ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
13471
13472 BasicBlock::iterator insertPt = AI->getIterator();
13473 insertPt = std::next(x: insertPt);
13474 AllocaInst *NewAllocaInst =
13475 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13476 AI->getAlign(), AI->getName(), insertPt);
13477
13478 if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
13479 return ChangeStatus::CHANGED;
13480
13481 break;
13482 }
13483 default:
13484 break;
13485 }
13486
13487 return ChangeStatus::UNCHANGED;
13488 }
13489
13490 const std::string getAsStr(Attributor *A) const override {
13491 if (!isValidState())
13492 return "allocationinfo(<invalid>)";
13493 return "allocationinfo(" +
13494 (AssumedAllocatedSize == HasNoAllocationSize
13495 ? "none"
13496 : std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
13497 ")";
13498 }
13499
13500private:
13501 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13502
13503 // Maintain the computed allocation size of the object.
13504 // Returns (bool) weather the size of the allocation was modified or not.
13505 bool changeAllocationSize(std::optional<TypeSize> Size) {
13506 if (AssumedAllocatedSize == HasNoAllocationSize ||
13507 AssumedAllocatedSize != Size) {
13508 AssumedAllocatedSize = Size;
13509 return true;
13510 }
13511 return false;
13512 }
13513};
13514
13515struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13516 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13517 : AAAllocationInfoImpl(IRP, A) {}
13518
13519 void trackStatistics() const override {
13520 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13521 }
13522};
13523
13524struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13525 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13526 : AAAllocationInfoImpl(IRP, A) {}
13527
13528 void initialize(Attributor &A) override {
13529 // TODO: we don't rewrite function argument for now because it will need to
13530 // rewrite the function signature and all call sites
13531 (void)indicatePessimisticFixpoint();
13532 }
13533
13534 void trackStatistics() const override {
13535 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13536 }
13537};
13538
13539struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13540 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13541 : AAAllocationInfoImpl(IRP, A) {}
13542
13543 void trackStatistics() const override {
13544 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13545 }
13546};
13547
13548struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13549 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13550 : AAAllocationInfoImpl(IRP, A) {}
13551
13552 void trackStatistics() const override {
13553 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13554 }
13555};
13556
13557struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13558 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13559 : AAAllocationInfoImpl(IRP, A) {}
13560
13561 void initialize(Attributor &A) override {
13562
13563 (void)indicatePessimisticFixpoint();
13564 }
13565
13566 void trackStatistics() const override {
13567 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13568 }
13569};
13570} // namespace
13571
13572const char AANoUnwind::ID = 0;
13573const char AANoSync::ID = 0;
13574const char AANoFree::ID = 0;
13575const char AANonNull::ID = 0;
13576const char AAMustProgress::ID = 0;
13577const char AANoRecurse::ID = 0;
13578const char AANonConvergent::ID = 0;
13579const char AAWillReturn::ID = 0;
13580const char AAUndefinedBehavior::ID = 0;
13581const char AANoAlias::ID = 0;
13582const char AAIntraFnReachability::ID = 0;
13583const char AANoReturn::ID = 0;
13584const char AAIsDead::ID = 0;
13585const char AADereferenceable::ID = 0;
13586const char AAAlign::ID = 0;
13587const char AAInstanceInfo::ID = 0;
13588const char AANoCapture::ID = 0;
13589const char AAValueSimplify::ID = 0;
13590const char AAHeapToStack::ID = 0;
13591const char AAPrivatizablePtr::ID = 0;
13592const char AAMemoryBehavior::ID = 0;
13593const char AAMemoryLocation::ID = 0;
13594const char AAValueConstantRange::ID = 0;
13595const char AAPotentialConstantValues::ID = 0;
13596const char AAPotentialValues::ID = 0;
13597const char AANoUndef::ID = 0;
13598const char AANoFPClass::ID = 0;
13599const char AACallEdges::ID = 0;
13600const char AAInterFnReachability::ID = 0;
13601const char AAPointerInfo::ID = 0;
13602const char AAAssumptionInfo::ID = 0;
13603const char AAUnderlyingObjects::ID = 0;
13604const char AAInvariantLoadPointer::ID = 0;
13605const char AAAddressSpace::ID = 0;
13606const char AANoAliasAddrSpace::ID = 0;
13607const char AAAllocationInfo::ID = 0;
13608const char AAIndirectCallInfo::ID = 0;
13609const char AAGlobalValueInfo::ID = 0;
13610const char AADenormalFPMath::ID = 0;
13611
13612// Macro magic to create the static generator function for attributes that
13613// follow the naming scheme.
13614
13615#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13616 case IRPosition::PK: \
13617 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13618
13619#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13620 case IRPosition::PK: \
13621 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13622 ++NumAAs; \
13623 break;
13624
13625#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13626 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13627 CLASS *AA = nullptr; \
13628 switch (IRP.getPositionKind()) { \
13629 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13630 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13631 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13632 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13633 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13634 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13635 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13636 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13637 } \
13638 return *AA; \
13639 }
13640
13641#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13642 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13643 CLASS *AA = nullptr; \
13644 switch (IRP.getPositionKind()) { \
13645 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13646 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13647 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13648 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13649 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13650 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13651 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13652 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13653 } \
13654 return *AA; \
13655 }
13656
13657#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13658 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13659 CLASS *AA = nullptr; \
13660 switch (IRP.getPositionKind()) { \
13661 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13662 default: \
13663 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13664 " position!"); \
13665 } \
13666 return *AA; \
13667 }
13668
13669#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13670 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13671 CLASS *AA = nullptr; \
13672 switch (IRP.getPositionKind()) { \
13673 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13674 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13675 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13676 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13677 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13678 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13679 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13680 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13681 } \
13682 return *AA; \
13683 }
13684
13685#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13686 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13687 CLASS *AA = nullptr; \
13688 switch (IRP.getPositionKind()) { \
13689 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13690 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13691 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13692 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13693 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13694 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13695 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13696 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13697 } \
13698 return *AA; \
13699 }
13700
13701#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13702 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13703 CLASS *AA = nullptr; \
13704 switch (IRP.getPositionKind()) { \
13705 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13706 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13707 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13708 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13709 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13710 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13711 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13712 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13713 } \
13714 return *AA; \
13715 }
13716
13717CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13718CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13719CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13720CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13721CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13722CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13723CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13724CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13725CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13726
13727CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13728CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13729CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13730CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13731CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13732CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13733CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13734CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13735CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13736CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13737CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13738CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13739CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13740CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
13741CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13742CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
13743CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13744
13745CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13746CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13747CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13748CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13749
13750CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13751 AAIndirectCallInfo)
13752CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13753 AAGlobalValueInfo)
13754
13755CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13756CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13757CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13758CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13759CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13760CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13761
13762CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13763
13764#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13765#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13766#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13767#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13768#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13769#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13770#undef SWITCH_PK_CREATE
13771#undef SWITCH_PK_INV
13772