1//===- AssumptionCache.cpp - Cache finding @llvm.assume calls -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that keeps track of @llvm.assume intrinsics in
10// the functions of a module.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Analysis/AssumptionCache.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallPtrSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/Analysis/AssumeBundleQueries.h"
19#include "llvm/Analysis/TargetTransformInfo.h"
20#include "llvm/Analysis/ValueTracking.h"
21#include "llvm/IR/BasicBlock.h"
22#include "llvm/IR/Function.h"
23#include "llvm/IR/InstrTypes.h"
24#include "llvm/IR/Instruction.h"
25#include "llvm/IR/Instructions.h"
26#include "llvm/IR/PassManager.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/InitializePasses.h"
29#include "llvm/Pass.h"
30#include "llvm/Support/Casting.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/raw_ostream.h"
34#include <cassert>
35#include <utility>
36
37using namespace llvm;
38using namespace llvm::PatternMatch;
39
40static cl::opt<bool>
41 VerifyAssumptionCache("verify-assumption-cache", cl::Hidden,
42 cl::desc("Enable verification of assumption cache"),
43 cl::init(Val: false));
44
45SmallVector<AssumptionCache::ResultElem, 1> &
46AssumptionCache::getOrInsertAffectedValues(Value *V) {
47 // Try using find_as first to avoid creating extra value handles just for the
48 // purpose of doing the lookup.
49 auto AVI = AffectedValues.find_as(Val: V);
50 if (AVI != AffectedValues.end())
51 return AVI->second;
52
53 return AffectedValues[AffectedValueCallbackVH(V, this)];
54}
55
56static void
57findAffectedValues(CallBase *CI, TargetTransformInfo *TTI,
58 SmallVectorImpl<AssumptionCache::ResultElem> &Affected) {
59 // Note: This code must be kept in-sync with the code in
60 // computeKnownBitsFromAssume in ValueTracking.
61
62 auto InsertAffected = [&Affected](Value *V) {
63 Affected.push_back(Elt: {.Assume: V, .Index: AssumptionCache::ExprResultIdx});
64 };
65
66 auto AddAffectedVal = [&Affected](Value *V, unsigned Idx) {
67 if (isa<Argument>(Val: V) || isa<GlobalValue>(Val: V) || isa<Instruction>(Val: V)) {
68 Affected.push_back(Elt: {.Assume: V, .Index: Idx});
69 }
70 };
71
72 for (unsigned Idx = 0; Idx != CI->getNumOperandBundles(); Idx++) {
73 OperandBundleUse Bundle = CI->getOperandBundleAt(Index: Idx);
74 if (Bundle.getTagName() == "separate_storage") {
75 assert(Bundle.Inputs.size() == 2 &&
76 "separate_storage must have two args");
77 AddAffectedVal(getUnderlyingObject(V: Bundle.Inputs[0]), Idx);
78 AddAffectedVal(getUnderlyingObject(V: Bundle.Inputs[1]), Idx);
79 } else if (Bundle.Inputs.size() > ABA_WasOn &&
80 Bundle.getTagName() != IgnoreBundleTag)
81 AddAffectedVal(Bundle.Inputs[ABA_WasOn], Idx);
82 }
83
84 Value *Cond = CI->getArgOperand(i: 0);
85 findValuesAffectedByCondition(Cond, /*IsAssume=*/true, InsertAffected);
86
87 if (TTI) {
88 const Value *Ptr;
89 unsigned AS;
90 std::tie(args&: Ptr, args&: AS) = TTI->getPredicatedAddrSpace(V: Cond);
91 if (Ptr)
92 AddAffectedVal(const_cast<Value *>(Ptr->stripInBoundsOffsets()),
93 AssumptionCache::ExprResultIdx);
94 }
95}
96
97void AssumptionCache::updateAffectedValues(AssumeInst *CI) {
98 SmallVector<AssumptionCache::ResultElem, 16> Affected;
99 findAffectedValues(CI, TTI, Affected);
100
101 for (auto &AV : Affected) {
102 auto &AVV = getOrInsertAffectedValues(V: AV.Assume);
103 if (llvm::none_of(Range&: AVV, P: [&](ResultElem &Elem) {
104 return Elem.Assume == CI && Elem.Index == AV.Index;
105 }))
106 AVV.push_back(Elt: {.Assume: CI, .Index: AV.Index});
107 }
108}
109
110void AssumptionCache::unregisterAssumption(AssumeInst *CI) {
111 SmallVector<AssumptionCache::ResultElem, 16> Affected;
112 findAffectedValues(CI, TTI, Affected);
113
114 for (auto &AV : Affected) {
115 auto AVI = AffectedValues.find_as(Val: AV.Assume);
116 if (AVI == AffectedValues.end())
117 continue;
118 bool Found = false;
119 bool HasNonnull = false;
120 for (ResultElem &Elem : AVI->second) {
121 if (Elem.Assume == CI) {
122 Found = true;
123 Elem.Assume = nullptr;
124 }
125 HasNonnull |= !!Elem.Assume;
126 if (HasNonnull && Found)
127 break;
128 }
129 assert(Found && "already unregistered or incorrect cache state");
130 if (!HasNonnull)
131 AffectedValues.erase(I: AVI);
132 }
133
134 llvm::erase(C&: AssumeHandles, V: CI);
135}
136
137void AssumptionCache::AffectedValueCallbackVH::deleted() {
138 AC->AffectedValues.erase(Val: getValPtr());
139 // 'this' now dangles!
140}
141
142void AssumptionCache::transferAffectedValuesInCache(Value *OV, Value *NV) {
143 auto &NAVV = getOrInsertAffectedValues(V: NV);
144 auto AVI = AffectedValues.find(Val: OV);
145 if (AVI == AffectedValues.end())
146 return;
147
148 for (auto &A : AVI->second)
149 if (!llvm::is_contained(Range&: NAVV, Element: A))
150 NAVV.push_back(Elt: A);
151 AffectedValues.erase(Val: OV);
152}
153
154void AssumptionCache::AffectedValueCallbackVH::allUsesReplacedWith(Value *NV) {
155 if (!isa<Instruction>(Val: NV) && !isa<Argument>(Val: NV))
156 return;
157
158 // Any assumptions that affected this value now affect the new value.
159
160 AC->transferAffectedValuesInCache(OV: getValPtr(), NV);
161 // 'this' now might dangle! If the AffectedValues map was resized to add an
162 // entry for NV then this object might have been destroyed in favor of some
163 // copy in the grown map.
164}
165
166void AssumptionCache::scanFunction() {
167 assert(!Scanned && "Tried to scan the function twice!");
168 assert(AssumeHandles.empty() && "Already have assumes when scanning!");
169
170 // Go through all instructions in all blocks, add all calls to @llvm.assume
171 // to this cache.
172 for (BasicBlock &B : F)
173 for (Instruction &I : B)
174 if (isa<AssumeInst>(Val: &I))
175 AssumeHandles.push_back(Elt: {.Assume: &I, .Index: ExprResultIdx});
176
177 // Mark the scan as complete.
178 Scanned = true;
179
180 // Update affected values.
181 for (auto &A : AssumeHandles)
182 updateAffectedValues(CI: cast<AssumeInst>(Val&: A));
183}
184
185void AssumptionCache::registerAssumption(AssumeInst *CI) {
186 // If we haven't scanned the function yet, just drop this assumption. It will
187 // be found when we scan later.
188 if (!Scanned)
189 return;
190
191 AssumeHandles.push_back(Elt: {.Assume: CI, .Index: ExprResultIdx});
192
193#ifndef NDEBUG
194 assert(CI->getParent() &&
195 "Cannot register @llvm.assume call not in a basic block");
196 assert(&F == CI->getParent()->getParent() &&
197 "Cannot register @llvm.assume call not in this function");
198
199 // We expect the number of assumptions to be small, so in an asserts build
200 // check that we don't accumulate duplicates and that all assumptions point
201 // to the same function.
202 SmallPtrSet<Value *, 16> AssumptionSet;
203 for (auto &VH : AssumeHandles) {
204 if (!VH)
205 continue;
206
207 assert(&F == cast<Instruction>(VH)->getParent()->getParent() &&
208 "Cached assumption not inside this function!");
209 assert(match(cast<CallInst>(VH), m_Intrinsic<Intrinsic::assume>()) &&
210 "Cached something other than a call to @llvm.assume!");
211 assert(AssumptionSet.insert(VH).second &&
212 "Cache contains multiple copies of a call!");
213 }
214#endif
215
216 updateAffectedValues(CI);
217}
218
219AssumptionCache AssumptionAnalysis::run(Function &F,
220 FunctionAnalysisManager &FAM) {
221 auto &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
222 return AssumptionCache(F, &TTI);
223}
224
225AnalysisKey AssumptionAnalysis::Key;
226
227PreservedAnalyses AssumptionPrinterPass::run(Function &F,
228 FunctionAnalysisManager &AM) {
229 AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
230
231 OS << "Cached assumptions for function: " << F.getName() << "\n";
232 for (auto &VH : AC.assumptions())
233 if (VH)
234 OS << " " << *cast<CallInst>(Val&: VH)->getArgOperand(i: 0) << "\n";
235
236 return PreservedAnalyses::all();
237}
238
239void AssumptionCacheTracker::FunctionCallbackVH::deleted() {
240 auto I = ACT->AssumptionCaches.find_as(Val: cast<Function>(Val: getValPtr()));
241 if (I != ACT->AssumptionCaches.end())
242 ACT->AssumptionCaches.erase(I);
243 // 'this' now dangles!
244}
245
246AssumptionCache &AssumptionCacheTracker::getAssumptionCache(Function &F) {
247 // We probe the function map twice to try and avoid creating a value handle
248 // around the function in common cases. This makes insertion a bit slower,
249 // but if we have to insert we're going to scan the whole function so that
250 // shouldn't matter.
251 auto I = AssumptionCaches.find_as(Val: &F);
252 if (I != AssumptionCaches.end())
253 return *I->second;
254
255 auto *TTIWP = getAnalysisIfAvailable<TargetTransformInfoWrapperPass>();
256 auto *TTI = TTIWP ? &TTIWP->getTTI(F) : nullptr;
257
258 // Ok, build a new cache by scanning the function, insert it and the value
259 // handle into our map, and return the newly populated cache.
260 auto IP = AssumptionCaches.insert(KV: std::make_pair(
261 x: FunctionCallbackVH(&F, this), y: std::make_unique<AssumptionCache>(args&: F, args&: TTI)));
262 assert(IP.second && "Scanning function already in the map?");
263 return *IP.first->second;
264}
265
266AssumptionCache *AssumptionCacheTracker::lookupAssumptionCache(Function &F) {
267 auto I = AssumptionCaches.find_as(Val: &F);
268 if (I != AssumptionCaches.end())
269 return I->second.get();
270 return nullptr;
271}
272
273void AssumptionCacheTracker::verifyAnalysis() const {
274 // FIXME: In the long term the verifier should not be controllable with a
275 // flag. We should either fix all passes to correctly update the assumption
276 // cache and enable the verifier unconditionally or somehow arrange for the
277 // assumption list to be updated automatically by passes.
278 if (!VerifyAssumptionCache)
279 return;
280
281 SmallPtrSet<const CallInst *, 4> AssumptionSet;
282 for (const auto &I : AssumptionCaches) {
283 for (auto &VH : I.second->assumptions())
284 if (VH)
285 AssumptionSet.insert(Ptr: cast<CallInst>(Val&: VH));
286
287 for (const BasicBlock &B : cast<Function>(Val&: *I.first))
288 for (const Instruction &II : B)
289 if (match(V: &II, P: m_Intrinsic<Intrinsic::assume>()) &&
290 !AssumptionSet.count(Ptr: cast<CallInst>(Val: &II)))
291 report_fatal_error(reason: "Assumption in scanned function not in cache");
292 }
293}
294
295AssumptionCacheTracker::AssumptionCacheTracker() : ImmutablePass(ID) {}
296
297AssumptionCacheTracker::~AssumptionCacheTracker() = default;
298
299char AssumptionCacheTracker::ID = 0;
300
301INITIALIZE_PASS(AssumptionCacheTracker, "assumption-cache-tracker",
302 "Assumption Cache Tracker", false, true)
303