1//===- SSAUpdaterBulk.cpp - Unstructured SSA Update Tool ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SSAUpdaterBulk class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/Transforms/Utils/SSAUpdaterBulk.h"
14#include "llvm/Analysis/InstructionSimplify.h"
15#include "llvm/Analysis/IteratedDominanceFrontier.h"
16#include "llvm/IR/BasicBlock.h"
17#include "llvm/IR/Dominators.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Use.h"
20#include "llvm/IR/Value.h"
21
22using namespace llvm;
23
24#define DEBUG_TYPE "ssaupdaterbulk"
25
26/// Helper function for finding a block which should have a value for the given
27/// user. For PHI-nodes this block is the corresponding predecessor, for other
28/// instructions it's their parent block.
29static BasicBlock *getUserBB(Use *U) {
30 auto *User = cast<Instruction>(Val: U->getUser());
31
32 if (auto *UserPN = dyn_cast<PHINode>(Val: User))
33 return UserPN->getIncomingBlock(U: *U);
34 else
35 return User->getParent();
36}
37
38/// Add a new variable to the SSA rewriter. This needs to be called before
39/// AddAvailableValue or AddUse calls.
40unsigned SSAUpdaterBulk::AddVariable(StringRef Name, Type *Ty) {
41 unsigned Var = Rewrites.size();
42 LLVM_DEBUG(dbgs() << "SSAUpdater: Var=" << Var << ": initialized with Ty = "
43 << *Ty << ", Name = " << Name << "\n");
44 RewriteInfo RI(Name, Ty);
45 Rewrites.push_back(Elt: RI);
46 return Var;
47}
48
49/// Indicate that a rewritten value is available in the specified block with the
50/// specified value.
51void SSAUpdaterBulk::AddAvailableValue(unsigned Var, BasicBlock *BB, Value *V) {
52 assert(Var < Rewrites.size() && "Variable not found!");
53 LLVM_DEBUG(dbgs() << "SSAUpdater: Var=" << Var
54 << ": added new available value " << *V << " in "
55 << BB->getName() << "\n");
56 Rewrites[Var].Defines.emplace_back(Args&: BB, Args&: V);
57}
58
59/// Record a use of the symbolic value. This use will be updated with a
60/// rewritten value when RewriteAllUses is called.
61void SSAUpdaterBulk::AddUse(unsigned Var, Use *U) {
62 assert(Var < Rewrites.size() && "Variable not found!");
63 LLVM_DEBUG(dbgs() << "SSAUpdater: Var=" << Var << ": added a use" << *U->get()
64 << " in " << getUserBB(U)->getName() << "\n");
65 Rewrites[Var].Uses.push_back(Elt: U);
66}
67
68/// Given sets of UsingBlocks and DefBlocks, compute the set of LiveInBlocks.
69/// This is basically a subgraph limited by DefBlocks and UsingBlocks.
70static void
71ComputeLiveInBlocks(const SmallPtrSetImpl<BasicBlock *> &UsingBlocks,
72 const SmallPtrSetImpl<BasicBlock *> &DefBlocks,
73 SmallPtrSetImpl<BasicBlock *> &LiveInBlocks,
74 PredIteratorCache &PredCache) {
75 // To determine liveness, we must iterate through the predecessors of blocks
76 // where the def is live. Blocks are added to the worklist if we need to
77 // check their predecessors. Start with all the using blocks.
78 SmallVector<BasicBlock *, 64> LiveInBlockWorklist(UsingBlocks.begin(),
79 UsingBlocks.end());
80
81 // Now that we have a set of blocks where the phi is live-in, recursively add
82 // their predecessors until we find the full region the value is live.
83 while (!LiveInBlockWorklist.empty()) {
84 BasicBlock *BB = LiveInBlockWorklist.pop_back_val();
85
86 // The block really is live in here, insert it into the set. If already in
87 // the set, then it has already been processed.
88 if (!LiveInBlocks.insert(Ptr: BB).second)
89 continue;
90
91 // Since the value is live into BB, it is either defined in a predecessor or
92 // live into it to. Add the preds to the worklist unless they are a
93 // defining block.
94 for (BasicBlock *P : PredCache.get(BB)) {
95 // The value is not live into a predecessor if it defines the value.
96 if (DefBlocks.count(Ptr: P))
97 continue;
98
99 // Otherwise it is, add to the worklist.
100 LiveInBlockWorklist.push_back(Elt: P);
101 }
102 }
103}
104
105struct BBValueInfo {
106 Value *LiveInValue = nullptr;
107 Value *LiveOutValue = nullptr;
108};
109
110/// Perform all the necessary updates, including new PHI-nodes insertion and the
111/// requested uses update.
112void SSAUpdaterBulk::RewriteAllUses(DominatorTree *DT,
113 SmallVectorImpl<PHINode *> *InsertedPHIs) {
114 DenseMap<BasicBlock *, BBValueInfo> BBInfos;
115 for (RewriteInfo &R : Rewrites) {
116 BBInfos.clear();
117
118 // Compute locations for new phi-nodes.
119 // For that we need to initialize DefBlocks from definitions in R.Defines,
120 // UsingBlocks from uses in R.Uses, then compute LiveInBlocks, and then use
121 // this set for computing iterated dominance frontier (IDF).
122 // The IDF blocks are the blocks where we need to insert new phi-nodes.
123 ForwardIDFCalculator IDF(*DT);
124 LLVM_DEBUG(dbgs() << "SSAUpdater: rewriting " << R.Uses.size()
125 << " use(s)\n");
126
127 SmallPtrSet<BasicBlock *, 2> DefBlocks(llvm::from_range,
128 llvm::make_first_range(c&: R.Defines));
129 IDF.setDefiningBlocks(DefBlocks);
130
131 SmallPtrSet<BasicBlock *, 2> UsingBlocks;
132 for (Use *U : R.Uses)
133 UsingBlocks.insert(Ptr: getUserBB(U));
134
135 SmallVector<BasicBlock *, 32> IDFBlocks;
136 SmallPtrSet<BasicBlock *, 32> LiveInBlocks;
137 ComputeLiveInBlocks(UsingBlocks, DefBlocks, LiveInBlocks, PredCache);
138 IDF.setLiveInBlocks(LiveInBlocks);
139 IDF.calculate(IDFBlocks);
140
141 // Reserve sufficient buckets to prevent map growth. [1]
142 BBInfos.reserve(NumEntries: LiveInBlocks.size() + DefBlocks.size());
143
144 for (auto [BB, V] : R.Defines)
145 BBInfos[BB].LiveOutValue = V;
146
147 // We've computed IDF, now insert new phi-nodes there.
148 for (BasicBlock *FrontierBB : IDFBlocks) {
149 IRBuilder<> B(FrontierBB, FrontierBB->begin());
150 PHINode *PN = B.CreatePHI(Ty: R.Ty, NumReservedValues: 0, Name: R.Name);
151 BBInfos[FrontierBB].LiveInValue = PN;
152 if (InsertedPHIs)
153 InsertedPHIs->push_back(Elt: PN);
154 }
155
156 // IsLiveOut indicates whether we are computing live-out values (true) or
157 // live-in values (false).
158 auto ComputeValue = [&](BasicBlock *BB, bool IsLiveOut) -> Value * {
159 BBValueInfo *BBInfo = &BBInfos[BB];
160
161 if (IsLiveOut && BBInfo->LiveOutValue)
162 return BBInfo->LiveOutValue;
163
164 if (BBInfo->LiveInValue)
165 return BBInfo->LiveInValue;
166
167 SmallVector<BBValueInfo *, 4> Stack = {BBInfo};
168 Value *V = nullptr;
169
170 while (DT->isReachableFromEntry(A: BB) && !PredCache.get(BB).empty() &&
171 (BB = DT->getNode(BB)->getIDom()->getBlock())) {
172 BBInfo = &BBInfos[BB];
173
174 if (BBInfo->LiveOutValue) {
175 V = BBInfo->LiveOutValue;
176 break;
177 }
178
179 if (BBInfo->LiveInValue) {
180 V = BBInfo->LiveInValue;
181 break;
182 }
183
184 Stack.emplace_back(Args&: BBInfo);
185 }
186
187 if (!V)
188 V = UndefValue::get(T: R.Ty);
189
190 for (BBValueInfo *BBInfo : Stack)
191 // Loop above can insert new entries into the BBInfos map: assume the
192 // map shouldn't grow due to [1] and BBInfo references are valid.
193 BBInfo->LiveInValue = V;
194
195 return V;
196 };
197
198 // Fill in arguments of the inserted PHIs.
199 for (BasicBlock *BB : IDFBlocks) {
200 auto *PHI = cast<PHINode>(Val: &BB->front());
201 for (BasicBlock *Pred : PredCache.get(BB))
202 PHI->addIncoming(V: ComputeValue(Pred, /*IsLiveOut=*/true), BB: Pred);
203 }
204
205 // Rewrite actual uses with the inserted definitions.
206 SmallPtrSet<Use *, 4> ProcessedUses;
207 for (Use *U : R.Uses) {
208 if (!ProcessedUses.insert(Ptr: U).second)
209 continue;
210
211 auto *User = cast<Instruction>(Val: U->getUser());
212 BasicBlock *BB = getUserBB(U);
213 Value *V = ComputeValue(BB, /*IsLiveOut=*/BB != User->getParent());
214 Value *OldVal = U->get();
215 assert(OldVal && "Invalid use!");
216 // Notify that users of the existing value that it is being replaced.
217 if (OldVal != V && OldVal->hasValueHandle())
218 ValueHandleBase::ValueIsRAUWd(Old: OldVal, New: V);
219 LLVM_DEBUG(dbgs() << "SSAUpdater: replacing " << *OldVal << " with " << *V
220 << "\n");
221 U->set(V);
222 }
223 }
224}
225
226// Perform a single pass of simplification over the worklist of PHIs.
227// This should be called after RewriteAllUses() because simplifying PHIs
228// immediately after creation would require updating all references to those
229// PHIs in the BBValueInfo structures, which would necessitate additional
230// reference tracking overhead.
231static void simplifyPass(MutableArrayRef<PHINode *> Worklist,
232 const DataLayout &DL) {
233 for (PHINode *&PHI : Worklist) {
234 if (Value *Simplified = simplifyInstruction(I: PHI, Q: DL)) {
235 PHI->replaceAllUsesWith(V: Simplified);
236 PHI->eraseFromParent();
237 PHI = nullptr; // Mark as removed.
238 }
239 }
240}
241
242#ifndef NDEBUG // Should this be under EXPENSIVE_CHECKS?
243// New PHI nodes should not reference one another but they may reference
244// themselves or existing PHI nodes, and existing PHI nodes may reference new
245// PHI nodes.
246static bool
247PHIAreRefEachOther(const iterator_range<BasicBlock::phi_iterator> NewPHIs) {
248 SmallPtrSet<PHINode *, 8> NewPHISet;
249 for (PHINode &PN : NewPHIs)
250 NewPHISet.insert(&PN);
251 for (PHINode &PHI : NewPHIs) {
252 for (Value *V : PHI.incoming_values()) {
253 PHINode *IncPHI = dyn_cast<PHINode>(V);
254 if (IncPHI && IncPHI != &PHI && NewPHISet.contains(IncPHI))
255 return true;
256 }
257 }
258 return false;
259}
260#endif
261
262static bool replaceIfIdentical(PHINode &PHI, PHINode &ReplPHI) {
263 if (!PHI.isIdenticalToWhenDefined(I: &ReplPHI))
264 return false;
265 PHI.replaceAllUsesWith(V: &ReplPHI);
266 PHI.eraseFromParent();
267 return true;
268}
269
270namespace llvm {
271
272bool EliminateNewDuplicatePHINodes(BasicBlock *BB,
273 BasicBlock::phi_iterator FirstExistingPN) {
274 assert(!PHIAreRefEachOther(make_range(BB->phis().begin(), FirstExistingPN)));
275
276 // Deduplicate new PHIs first to reduce the number of comparisons on the
277 // following new -> existing pass.
278 bool Changed = false;
279 for (auto I = BB->phis().begin(); I != FirstExistingPN; ++I) {
280 for (auto J = std::next(x: I); J != FirstExistingPN;) {
281 Changed |= replaceIfIdentical(PHI&: *J++, ReplPHI&: *I);
282 }
283 }
284
285 // Iterate over existing PHIs and replace identical new PHIs.
286 for (PHINode &ExistingPHI : make_range(x: FirstExistingPN, y: BB->phis().end())) {
287 auto I = BB->phis().begin();
288 assert(I != FirstExistingPN); // Should be at least one new PHI.
289 do {
290 Changed |= replaceIfIdentical(PHI&: *I++, ReplPHI&: ExistingPHI);
291 } while (I != FirstExistingPN);
292 if (BB->phis().begin() == FirstExistingPN)
293 return Changed;
294 }
295 return Changed;
296}
297
298} // end namespace llvm
299
300static void deduplicatePass(ArrayRef<PHINode *> Worklist) {
301 SmallDenseMap<BasicBlock *, unsigned> BBs;
302 for (PHINode *PHI : Worklist) {
303 if (PHI)
304 ++BBs[PHI->getParent()];
305 }
306
307 for (auto [BB, NumNewPHIs] : BBs) {
308 auto FirstExistingPN = std::next(x: BB->phis().begin(), n: NumNewPHIs);
309 EliminateNewDuplicatePHINodes(BB, FirstExistingPN);
310 }
311}
312
313void SSAUpdaterBulk::RewriteAndOptimizeAllUses(DominatorTree &DT) {
314 SmallVector<PHINode *, 4> PHIs;
315 RewriteAllUses(DT: &DT, InsertedPHIs: &PHIs);
316 if (PHIs.empty())
317 return;
318
319 simplifyPass(Worklist: PHIs, DL: PHIs.front()->getParent()->getDataLayout());
320 deduplicatePass(Worklist: PHIs);
321}
322