1//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions perform manipulations on basic blocks, and
10// instructions contained within basic blocks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/BasicBlockUtils.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallPtrSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/CFG.h"
20#include "llvm/Analysis/DomTreeUpdater.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/Analysis/MemoryDependenceAnalysis.h"
23#include "llvm/Analysis/MemorySSAUpdater.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/CFG.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/Dominators.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstrTypes.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/LLVMContext.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/User.h"
38#include "llvm/IR/Value.h"
39#include "llvm/IR/ValueHandle.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/raw_ostream.h"
44#include "llvm/Transforms/Utils/Local.h"
45#include <cassert>
46#include <cstdint>
47#include <string>
48#include <utility>
49#include <vector>
50
51using namespace llvm;
52
53#define DEBUG_TYPE "basicblock-utils"
54
55static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
56 "max-deopt-or-unreachable-succ-check-depth", cl::init(Val: 8), cl::Hidden,
57 cl::desc("Set the maximum path length when checking whether a basic block "
58 "is followed by a block that either has a terminating "
59 "deoptimizing call or is terminated with an unreachable"));
60
61void llvm::detachDeadBlocks(
62 ArrayRef<BasicBlock *> BBs,
63 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
64 bool KeepOneInputPHIs) {
65 for (auto *BB : BBs) {
66 // Loop through all of our successors and make sure they know that one
67 // of their predecessors is going away.
68 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
69 for (BasicBlock *Succ : successors(BB)) {
70 Succ->removePredecessor(Pred: BB, KeepOneInputPHIs);
71 if (Updates && UniqueSuccessors.insert(Ptr: Succ).second)
72 Updates->push_back(Elt: {DominatorTree::Delete, BB, Succ});
73 }
74
75 // Zap all the instructions in the block.
76 while (!BB->empty()) {
77 Instruction &I = BB->back();
78 // If this instruction is used, replace uses with an arbitrary value.
79 // Because control flow can't get here, we don't care what we replace the
80 // value with. Note that since this block is unreachable, and all values
81 // contained within it must dominate their uses, that all uses will
82 // eventually be removed (they are themselves dead).
83 if (!I.use_empty())
84 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
85 BB->back().eraseFromParent();
86 }
87 new UnreachableInst(BB->getContext(), BB);
88 assert(BB->size() == 1 &&
89 isa<UnreachableInst>(BB->getTerminator()) &&
90 "The successor list of BB isn't empty before "
91 "applying corresponding DTU updates.");
92 }
93}
94
95void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
96 bool KeepOneInputPHIs) {
97 DeleteDeadBlocks(BBs: {BB}, DTU, KeepOneInputPHIs);
98}
99
100void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
101 bool KeepOneInputPHIs) {
102#ifndef NDEBUG
103 // Make sure that all predecessors of each dead block is also dead.
104 SmallPtrSet<BasicBlock *, 4> Dead(llvm::from_range, BBs);
105 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
106 for (auto *BB : Dead)
107 for (BasicBlock *Pred : predecessors(BB))
108 assert(Dead.count(Pred) && "All predecessors must be dead!");
109#endif
110
111 SmallVector<DominatorTree::UpdateType, 4> Updates;
112 detachDeadBlocks(BBs, Updates: DTU ? &Updates : nullptr, KeepOneInputPHIs);
113
114 if (DTU)
115 DTU->applyUpdates(Updates);
116
117 for (BasicBlock *BB : BBs)
118 if (DTU)
119 DTU->deleteBB(DelBB: BB);
120 else
121 BB->eraseFromParent();
122}
123
124bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
125 bool KeepOneInputPHIs) {
126 df_iterator_default_set<BasicBlock*> Reachable;
127
128 // Mark all reachable blocks.
129 for (BasicBlock *BB : depth_first_ext(G: &F, S&: Reachable))
130 (void)BB/* Mark all reachable blocks */;
131
132 // Collect all dead blocks.
133 std::vector<BasicBlock*> DeadBlocks;
134 for (BasicBlock &BB : F)
135 if (!Reachable.count(Ptr: &BB))
136 DeadBlocks.push_back(x: &BB);
137
138 // Delete the dead blocks.
139 DeleteDeadBlocks(BBs: DeadBlocks, DTU, KeepOneInputPHIs);
140
141 return !DeadBlocks.empty();
142}
143
144bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
145 MemoryDependenceResults *MemDep) {
146 if (!isa<PHINode>(Val: BB->begin()))
147 return false;
148
149 while (PHINode *PN = dyn_cast<PHINode>(Val: BB->begin())) {
150 if (PN->getIncomingValue(i: 0) != PN)
151 PN->replaceAllUsesWith(V: PN->getIncomingValue(i: 0));
152 else
153 PN->replaceAllUsesWith(V: PoisonValue::get(T: PN->getType()));
154
155 if (MemDep)
156 MemDep->removeInstruction(InstToRemove: PN); // Memdep updates AA itself.
157
158 PN->eraseFromParent();
159 }
160 return true;
161}
162
163bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
164 MemorySSAUpdater *MSSAU) {
165 // Recursively deleting a PHI may cause multiple PHIs to be deleted
166 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
167 SmallVector<WeakTrackingVH, 8> PHIs(llvm::make_pointer_range(Range: BB->phis()));
168
169 bool Changed = false;
170 for (const auto &PHI : PHIs)
171 if (PHINode *PN = dyn_cast_or_null<PHINode>(Val: PHI.operator Value *()))
172 Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
173
174 return Changed;
175}
176
177bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
178 LoopInfo *LI, MemorySSAUpdater *MSSAU,
179 MemoryDependenceResults *MemDep,
180 bool PredecessorWithTwoSuccessors,
181 DominatorTree *DT) {
182 if (BB->hasAddressTaken())
183 return false;
184
185 // Can't merge if there are multiple predecessors, or no predecessors.
186 BasicBlock *PredBB = BB->getUniquePredecessor();
187 if (!PredBB) return false;
188
189 // Don't break self-loops.
190 if (PredBB == BB) return false;
191
192 // Don't break unwinding instructions or terminators with other side-effects.
193 Instruction *PTI = PredBB->getTerminator();
194 if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
195 return false;
196
197 // Can't merge if there are multiple distinct successors.
198 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
199 return false;
200
201 // Currently only allow PredBB to have two predecessors, one being BB.
202 // Update BI to branch to BB's only successor instead of BB.
203 BranchInst *PredBB_BI;
204 BasicBlock *NewSucc = nullptr;
205 unsigned FallThruPath;
206 if (PredecessorWithTwoSuccessors) {
207 if (!(PredBB_BI = dyn_cast<BranchInst>(Val: PTI)))
208 return false;
209 BranchInst *BB_JmpI = dyn_cast<BranchInst>(Val: BB->getTerminator());
210 if (!BB_JmpI || !BB_JmpI->isUnconditional())
211 return false;
212 NewSucc = BB_JmpI->getSuccessor(i: 0);
213 FallThruPath = PredBB_BI->getSuccessor(i: 0) == BB ? 0 : 1;
214 }
215
216 // Can't merge if there is PHI loop.
217 for (PHINode &PN : BB->phis())
218 if (llvm::is_contained(Range: PN.incoming_values(), Element: &PN))
219 return false;
220
221 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
222 << PredBB->getName() << "\n");
223
224 // Begin by getting rid of unneeded PHIs.
225 SmallVector<AssertingVH<Value>, 4> IncomingValues;
226 if (isa<PHINode>(Val: BB->front())) {
227 for (PHINode &PN : BB->phis())
228 if (!isa<PHINode>(Val: PN.getIncomingValue(i: 0)) ||
229 cast<PHINode>(Val: PN.getIncomingValue(i: 0))->getParent() != BB)
230 IncomingValues.push_back(Elt: PN.getIncomingValue(i: 0));
231 FoldSingleEntryPHINodes(BB, MemDep);
232 }
233
234 if (DT) {
235 assert(!DTU && "cannot use both DT and DTU for updates");
236 DomTreeNode *PredNode = DT->getNode(BB: PredBB);
237 DomTreeNode *BBNode = DT->getNode(BB);
238 if (PredNode) {
239 assert(BBNode && "PredNode unreachable but BBNode reachable?");
240 for (DomTreeNode *C : to_vector(Range: BBNode->children()))
241 C->setIDom(PredNode);
242 }
243 }
244 // DTU update: Collect all the edges that exit BB.
245 // These dominator edges will be redirected from Pred.
246 std::vector<DominatorTree::UpdateType> Updates;
247 if (DTU) {
248 assert(!DT && "cannot use both DT and DTU for updates");
249 // To avoid processing the same predecessor more than once.
250 SmallPtrSet<BasicBlock *, 8> SeenSuccs;
251 SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(llvm::from_range,
252 successors(BB: PredBB));
253 Updates.reserve(n: Updates.size() + 2 * succ_size(BB) + 1);
254 // Add insert edges first. Experimentally, for the particular case of two
255 // blocks that can be merged, with a single successor and single predecessor
256 // respectively, it is beneficial to have all insert updates first. Deleting
257 // edges first may lead to unreachable blocks, followed by inserting edges
258 // making the blocks reachable again. Such DT updates lead to high compile
259 // times. We add inserts before deletes here to reduce compile time.
260 for (BasicBlock *SuccOfBB : successors(BB))
261 // This successor of BB may already be a PredBB's successor.
262 if (!SuccsOfPredBB.contains(Ptr: SuccOfBB))
263 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
264 Updates.push_back(x: {DominatorTree::Insert, PredBB, SuccOfBB});
265 SeenSuccs.clear();
266 for (BasicBlock *SuccOfBB : successors(BB))
267 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
268 Updates.push_back(x: {DominatorTree::Delete, BB, SuccOfBB});
269 Updates.push_back(x: {DominatorTree::Delete, PredBB, BB});
270 }
271
272 Instruction *STI = BB->getTerminator();
273 Instruction *Start = &*BB->begin();
274 // If there's nothing to move, mark the starting instruction as the last
275 // instruction in the block. Terminator instruction is handled separately.
276 if (Start == STI)
277 Start = PTI;
278
279 // Move all definitions in the successor to the predecessor...
280 PredBB->splice(ToIt: PTI->getIterator(), FromBB: BB, FromBeginIt: BB->begin(), FromEndIt: STI->getIterator());
281
282 if (MSSAU)
283 MSSAU->moveAllAfterMergeBlocks(From: BB, To: PredBB, Start);
284
285 // Make all PHI nodes that referred to BB now refer to Pred as their
286 // source...
287 BB->replaceAllUsesWith(V: PredBB);
288
289 if (PredecessorWithTwoSuccessors) {
290 // Delete the unconditional branch from BB.
291 BB->back().eraseFromParent();
292
293 // Update branch in the predecessor.
294 PredBB_BI->setSuccessor(idx: FallThruPath, NewSucc);
295 } else {
296 // Delete the unconditional branch from the predecessor.
297 PredBB->back().eraseFromParent();
298
299 // Move terminator instruction.
300 BB->back().moveBeforePreserving(BB&: *PredBB, I: PredBB->end());
301
302 // Terminator may be a memory accessing instruction too.
303 if (MSSAU)
304 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
305 Val: MSSAU->getMemorySSA()->getMemoryAccess(I: PredBB->getTerminator())))
306 MSSAU->moveToPlace(What: MUD, BB: PredBB, Where: MemorySSA::End);
307 }
308 // Add unreachable to now empty BB.
309 new UnreachableInst(BB->getContext(), BB);
310
311 // Inherit predecessors name if it exists.
312 if (!PredBB->hasName())
313 PredBB->takeName(V: BB);
314
315 if (LI)
316 LI->removeBlock(BB);
317
318 if (MemDep)
319 MemDep->invalidateCachedPredecessors();
320
321 if (DTU)
322 DTU->applyUpdates(Updates);
323
324 if (DT) {
325 assert(succ_empty(BB) &&
326 "successors should have been transferred to PredBB");
327 DT->eraseNode(BB);
328 }
329
330 // Finally, erase the old block and update dominator info.
331 DeleteDeadBlock(BB, DTU);
332
333 return true;
334}
335
336bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
337 SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
338 LoopInfo *LI) {
339 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
340
341 bool BlocksHaveBeenMerged = false;
342 while (!MergeBlocks.empty()) {
343 BasicBlock *BB = *MergeBlocks.begin();
344 BasicBlock *Dest = BB->getSingleSuccessor();
345 if (Dest && (!L || L->contains(BB: Dest))) {
346 BasicBlock *Fold = Dest->getUniquePredecessor();
347 (void)Fold;
348 if (MergeBlockIntoPredecessor(BB: Dest, DTU, LI)) {
349 assert(Fold == BB &&
350 "Expecting BB to be unique predecessor of the Dest block");
351 MergeBlocks.erase(Ptr: Dest);
352 BlocksHaveBeenMerged = true;
353 } else
354 MergeBlocks.erase(Ptr: BB);
355 } else
356 MergeBlocks.erase(Ptr: BB);
357 }
358 return BlocksHaveBeenMerged;
359}
360
361/// Remove redundant instructions within sequences of consecutive dbg.value
362/// instructions. This is done using a backward scan to keep the last dbg.value
363/// describing a specific variable/fragment.
364///
365/// BackwardScan strategy:
366/// ----------------------
367/// Given a sequence of consecutive DbgValueInst like this
368///
369/// dbg.value ..., "x", FragmentX1 (*)
370/// dbg.value ..., "y", FragmentY1
371/// dbg.value ..., "x", FragmentX2
372/// dbg.value ..., "x", FragmentX1 (**)
373///
374/// then the instruction marked with (*) can be removed (it is guaranteed to be
375/// obsoleted by the instruction marked with (**) as the latter instruction is
376/// describing the same variable using the same fragment info).
377///
378/// Possible improvements:
379/// - Check fully overlapping fragments and not only identical fragments.
380static bool
381DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
382 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
383 SmallDenseSet<DebugVariable> VariableSet;
384 for (auto &I : reverse(C&: *BB)) {
385 for (DbgVariableRecord &DR :
386 reverse(C: filterDbgVars(R: I.getDbgRecordRange()))) {
387 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
388
389 DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
390 DVR.getDebugLoc()->getInlinedAt());
391 auto R = VariableSet.insert(V: Key);
392 // If the same variable fragment is described more than once it is enough
393 // to keep the last one (i.e. the first found since we for reverse
394 // iteration).
395 if (R.second)
396 continue;
397
398 if (DVR.isDbgAssign()) {
399 // Don't delete dbg.assign intrinsics that are linked to instructions.
400 if (!at::getAssignmentInsts(DVR: &DVR).empty())
401 continue;
402 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
403 }
404
405 ToBeRemoved.push_back(Elt: &DVR);
406 }
407 // Sequence with consecutive dbg.value instrs ended. Clear the map to
408 // restart identifying redundant instructions if case we find another
409 // dbg.value sequence.
410 VariableSet.clear();
411 }
412
413 for (auto &DVR : ToBeRemoved)
414 DVR->eraseFromParent();
415
416 return !ToBeRemoved.empty();
417}
418
419static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
420 return DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BB);
421}
422
423/// Remove redundant dbg.value instructions using a forward scan. This can
424/// remove a dbg.value instruction that is redundant due to indicating that a
425/// variable has the same value as already being indicated by an earlier
426/// dbg.value.
427///
428/// ForwardScan strategy:
429/// ---------------------
430/// Given two identical dbg.value instructions, separated by a block of
431/// instructions that isn't describing the same variable, like this
432///
433/// dbg.value X1, "x", FragmentX1 (**)
434/// <block of instructions, none being "dbg.value ..., "x", ...">
435/// dbg.value X1, "x", FragmentX1 (*)
436///
437/// then the instruction marked with (*) can be removed. Variable "x" is already
438/// described as being mapped to the SSA value X1.
439///
440/// Possible improvements:
441/// - Keep track of non-overlapping fragments.
442static bool
443DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
444 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
445 SmallDenseMap<DebugVariable,
446 std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
447 VariableMap;
448 for (auto &I : *BB) {
449 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
450 if (DVR.getType() == DbgVariableRecord::LocationType::Declare)
451 continue;
452 DebugVariable Key(DVR.getVariable(), std::nullopt,
453 DVR.getDebugLoc()->getInlinedAt());
454 auto [VMI, Inserted] = VariableMap.try_emplace(Key);
455 // A dbg.assign with no linked instructions can be treated like a
456 // dbg.value (i.e. can be deleted).
457 bool IsDbgValueKind =
458 (!DVR.isDbgAssign() || at::getAssignmentInsts(DVR: &DVR).empty());
459
460 // Update the map if we found a new value/expression describing the
461 // variable, or if the variable wasn't mapped already.
462 SmallVector<Value *, 4> Values(DVR.location_ops());
463 if (Inserted || VMI->second.first != Values ||
464 VMI->second.second != DVR.getExpression()) {
465 if (IsDbgValueKind)
466 VMI->second = {Values, DVR.getExpression()};
467 else
468 VMI->second = {Values, nullptr};
469 continue;
470 }
471 // Don't delete dbg.assign intrinsics that are linked to instructions.
472 if (!IsDbgValueKind)
473 continue;
474 // Found an identical mapping. Remember the instruction for later removal.
475 ToBeRemoved.push_back(Elt: &DVR);
476 }
477 }
478
479 for (auto *DVR : ToBeRemoved)
480 DVR->eraseFromParent();
481
482 return !ToBeRemoved.empty();
483}
484
485static bool
486DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
487 assert(BB->isEntryBlock() && "expected entry block");
488 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
489 DenseSet<DebugVariable> SeenDefForAggregate;
490 // Returns the DebugVariable for DVI with no fragment info.
491 auto GetAggregateVariable = [](const DbgVariableRecord &DVR) {
492 return DebugVariable(DVR.getVariable(), std::nullopt,
493 DVR.getDebugLoc().getInlinedAt());
494 };
495
496 // Remove undef dbg.assign intrinsics that are encountered before
497 // any non-undef intrinsics from the entry block.
498 for (auto &I : *BB) {
499 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
500 if (!DVR.isDbgValue() && !DVR.isDbgAssign())
501 continue;
502 bool IsDbgValueKind =
503 (DVR.isDbgValue() || at::getAssignmentInsts(DVR: &DVR).empty());
504 DebugVariable Aggregate = GetAggregateVariable(DVR);
505 if (!SeenDefForAggregate.contains(V: Aggregate)) {
506 bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
507 if (!IsKill) {
508 SeenDefForAggregate.insert(V: Aggregate);
509 } else if (DVR.isDbgAssign()) {
510 ToBeRemoved.push_back(Elt: &DVR);
511 }
512 }
513 }
514 }
515
516 for (DbgVariableRecord *DVR : ToBeRemoved)
517 DVR->eraseFromParent();
518
519 return !ToBeRemoved.empty();
520}
521
522static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
523 return DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BB);
524}
525
526/// Remove redundant undef dbg.assign intrinsic from an entry block using a
527/// forward scan.
528/// Strategy:
529/// ---------------------
530/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
531/// linked to an intrinsic, and don't share an aggregate variable with a debug
532/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
533/// that come before non-undef debug intrinsics for the variable are
534/// deleted. Given:
535///
536/// dbg.assign undef, "x", FragmentX1 (*)
537/// <block of instructions, none being "dbg.value ..., "x", ...">
538/// dbg.value %V, "x", FragmentX2
539/// <block of instructions, none being "dbg.value ..., "x", ...">
540/// dbg.assign undef, "x", FragmentX1
541///
542/// then (only) the instruction marked with (*) can be removed.
543/// Possible improvements:
544/// - Keep track of non-overlapping fragments.
545static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
546 return DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BB);
547}
548
549bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
550 bool MadeChanges = false;
551 // By using the "backward scan" strategy before the "forward scan" strategy we
552 // can remove both dbg.value (2) and (3) in a situation like this:
553 //
554 // (1) dbg.value V1, "x", DIExpression()
555 // ...
556 // (2) dbg.value V2, "x", DIExpression()
557 // (3) dbg.value V1, "x", DIExpression()
558 //
559 // The backward scan will remove (2), it is made obsolete by (3). After
560 // getting (2) out of the way, the foward scan will remove (3) since "x"
561 // already is described as having the value V1 at (1).
562 MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
563 if (BB->isEntryBlock() &&
564 isAssignmentTrackingEnabled(M: *BB->getParent()->getParent()))
565 MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
566 MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
567
568 if (MadeChanges)
569 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
570 << BB->getName() << "\n");
571 return MadeChanges;
572}
573
574void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
575 Instruction &I = *BI;
576 // Replaces all of the uses of the instruction with uses of the value
577 I.replaceAllUsesWith(V);
578
579 // Make sure to propagate a name if there is one already.
580 if (I.hasName() && !V->hasName())
581 V->takeName(V: &I);
582
583 // Delete the unnecessary instruction now...
584 BI = BI->eraseFromParent();
585}
586
587void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
588 Instruction *I) {
589 assert(I->getParent() == nullptr &&
590 "ReplaceInstWithInst: Instruction already inserted into basic block!");
591
592 // Copy debug location to newly added instruction, if it wasn't already set
593 // by the caller.
594 if (!I->getDebugLoc())
595 I->setDebugLoc(BI->getDebugLoc());
596
597 // Insert the new instruction into the basic block...
598 BasicBlock::iterator New = I->insertInto(ParentBB: BB, It: BI);
599
600 // Replace all uses of the old instruction, and delete it.
601 ReplaceInstWithValue(BI, V: I);
602
603 // Move BI back to point to the newly inserted instruction
604 BI = New;
605}
606
607bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
608 // Remember visited blocks to avoid infinite loop
609 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
610 unsigned Depth = 0;
611 while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
612 VisitedBlocks.insert(Ptr: BB).second) {
613 if (isa<UnreachableInst>(Val: BB->getTerminator()) ||
614 BB->getTerminatingDeoptimizeCall())
615 return true;
616 BB = BB->getUniqueSuccessor();
617 }
618 return false;
619}
620
621void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
622 BasicBlock::iterator BI(From);
623 ReplaceInstWithInst(BB: From->getParent(), BI, I: To);
624}
625
626BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
627 LoopInfo *LI, MemorySSAUpdater *MSSAU,
628 const Twine &BBName) {
629 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
630
631 Instruction *LatchTerm = BB->getTerminator();
632
633 CriticalEdgeSplittingOptions Options =
634 CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
635
636 if ((isCriticalEdge(TI: LatchTerm, SuccNum, AllowIdenticalEdges: Options.MergeIdenticalEdges))) {
637 // If this is a critical edge, let SplitKnownCriticalEdge do it.
638 return SplitKnownCriticalEdge(TI: LatchTerm, SuccNum, Options, BBName);
639 }
640
641 // If the edge isn't critical, then BB has a single successor or Succ has a
642 // single pred. Split the block.
643 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
644 // If the successor only has a single pred, split the top of the successor
645 // block.
646 assert(SP == BB && "CFG broken");
647 (void)SP;
648 return SplitBlock(Old: Succ, SplitPt: &Succ->front(), DT, LI, MSSAU, BBName,
649 /*Before=*/true);
650 }
651
652 // Otherwise, if BB has a single successor, split it at the bottom of the
653 // block.
654 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
655 "Should have a single succ!");
656 return SplitBlock(Old: BB, SplitPt: BB->getTerminator(), DT, LI, MSSAU, BBName);
657}
658
659void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
660 if (auto *II = dyn_cast<InvokeInst>(Val: TI))
661 II->setUnwindDest(Succ);
662 else if (auto *CS = dyn_cast<CatchSwitchInst>(Val: TI))
663 CS->setUnwindDest(Succ);
664 else if (auto *CR = dyn_cast<CleanupReturnInst>(Val: TI))
665 CR->setUnwindDest(Succ);
666 else
667 llvm_unreachable("unexpected terminator instruction");
668}
669
670void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
671 BasicBlock *NewPred, PHINode *Until) {
672 int BBIdx = 0;
673 for (PHINode &PN : DestBB->phis()) {
674 // We manually update the LandingPadReplacement PHINode and it is the last
675 // PHI Node. So, if we find it, we are done.
676 if (Until == &PN)
677 break;
678
679 // Reuse the previous value of BBIdx if it lines up. In cases where we
680 // have multiple phi nodes with *lots* of predecessors, this is a speed
681 // win because we don't have to scan the PHI looking for TIBB. This
682 // happens because the BB list of PHI nodes are usually in the same
683 // order.
684 if (PN.getIncomingBlock(i: BBIdx) != OldPred)
685 BBIdx = PN.getBasicBlockIndex(BB: OldPred);
686
687 assert(BBIdx != -1 && "Invalid PHI Index!");
688 PN.setIncomingBlock(i: BBIdx, BB: NewPred);
689 }
690}
691
692BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
693 LandingPadInst *OriginalPad,
694 PHINode *LandingPadReplacement,
695 const CriticalEdgeSplittingOptions &Options,
696 const Twine &BBName) {
697
698 auto PadInst = Succ->getFirstNonPHIIt();
699 if (!LandingPadReplacement && !PadInst->isEHPad())
700 return SplitEdge(BB, Succ, DT: Options.DT, LI: Options.LI, MSSAU: Options.MSSAU, BBName);
701
702 auto *LI = Options.LI;
703 SmallVector<BasicBlock *, 4> LoopPreds;
704 // Check if extra modifications will be required to preserve loop-simplify
705 // form after splitting. If it would require splitting blocks with IndirectBr
706 // terminators, bail out if preserving loop-simplify form is requested.
707 if (Options.PreserveLoopSimplify && LI) {
708 if (Loop *BBLoop = LI->getLoopFor(BB)) {
709
710 // The only way that we can break LoopSimplify form by splitting a
711 // critical edge is when there exists some edge from BBLoop to Succ *and*
712 // the only edge into Succ from outside of BBLoop is that of NewBB after
713 // the split. If the first isn't true, then LoopSimplify still holds,
714 // NewBB is the new exit block and it has no non-loop predecessors. If the
715 // second isn't true, then Succ was not in LoopSimplify form prior to
716 // the split as it had a non-loop predecessor. In both of these cases,
717 // the predecessor must be directly in BBLoop, not in a subloop, or again
718 // LoopSimplify doesn't hold.
719 for (BasicBlock *P : predecessors(BB: Succ)) {
720 if (P == BB)
721 continue; // The new block is known.
722 if (LI->getLoopFor(BB: P) != BBLoop) {
723 // Loop is not in LoopSimplify form, no need to re simplify after
724 // splitting edge.
725 LoopPreds.clear();
726 break;
727 }
728 LoopPreds.push_back(Elt: P);
729 }
730 // Loop-simplify form can be preserved, if we can split all in-loop
731 // predecessors.
732 if (any_of(Range&: LoopPreds, P: [](BasicBlock *Pred) {
733 return isa<IndirectBrInst>(Val: Pred->getTerminator());
734 })) {
735 return nullptr;
736 }
737 }
738 }
739
740 auto *NewBB =
741 BasicBlock::Create(Context&: BB->getContext(), Name: BBName, Parent: BB->getParent(), InsertBefore: Succ);
742 setUnwindEdgeTo(TI: BB->getTerminator(), Succ: NewBB);
743 updatePhiNodes(DestBB: Succ, OldPred: BB, NewPred: NewBB, Until: LandingPadReplacement);
744
745 if (LandingPadReplacement) {
746 auto *NewLP = OriginalPad->clone();
747 auto *Terminator = BranchInst::Create(IfTrue: Succ, InsertBefore: NewBB);
748 NewLP->insertBefore(InsertPos: Terminator->getIterator());
749 LandingPadReplacement->addIncoming(V: NewLP, BB: NewBB);
750 } else {
751 Value *ParentPad = nullptr;
752 if (auto *FuncletPad = dyn_cast<FuncletPadInst>(Val&: PadInst))
753 ParentPad = FuncletPad->getParentPad();
754 else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: PadInst))
755 ParentPad = CatchSwitch->getParentPad();
756 else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(Val&: PadInst))
757 ParentPad = CleanupPad->getParentPad();
758 else if (auto *LandingPad = dyn_cast<LandingPadInst>(Val&: PadInst))
759 ParentPad = LandingPad->getParent();
760 else
761 llvm_unreachable("handling for other EHPads not implemented yet");
762
763 auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, Args: {}, NameStr: BBName, InsertBefore: NewBB);
764 CleanupReturnInst::Create(CleanupPad: NewCleanupPad, UnwindBB: Succ, InsertBefore: NewBB);
765 }
766
767 auto *DT = Options.DT;
768 auto *MSSAU = Options.MSSAU;
769 if (!DT && !LI)
770 return NewBB;
771
772 if (DT) {
773 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
774 SmallVector<DominatorTree::UpdateType, 3> Updates;
775
776 Updates.push_back(Elt: {DominatorTree::Insert, BB, NewBB});
777 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, Succ});
778 Updates.push_back(Elt: {DominatorTree::Delete, BB, Succ});
779
780 DTU.applyUpdates(Updates);
781 DTU.flush();
782
783 if (MSSAU) {
784 MSSAU->applyUpdates(Updates, DT&: *DT);
785 if (VerifyMemorySSA)
786 MSSAU->getMemorySSA()->verifyMemorySSA();
787 }
788 }
789
790 if (LI) {
791 if (Loop *BBLoop = LI->getLoopFor(BB)) {
792 // If one or the other blocks were not in a loop, the new block is not
793 // either, and thus LI doesn't need to be updated.
794 if (Loop *SuccLoop = LI->getLoopFor(BB: Succ)) {
795 if (BBLoop == SuccLoop) {
796 // Both in the same loop, the NewBB joins loop.
797 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
798 } else if (BBLoop->contains(L: SuccLoop)) {
799 // Edge from an outer loop to an inner loop. Add to the outer loop.
800 BBLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
801 } else if (SuccLoop->contains(L: BBLoop)) {
802 // Edge from an inner loop to an outer loop. Add to the outer loop.
803 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
804 } else {
805 // Edge from two loops with no containment relation. Because these
806 // are natural loops, we know that the destination block must be the
807 // header of its loop (adding a branch into a loop elsewhere would
808 // create an irreducible loop).
809 assert(SuccLoop->getHeader() == Succ &&
810 "Should not create irreducible loops!");
811 if (Loop *P = SuccLoop->getParentLoop())
812 P->addBasicBlockToLoop(NewBB, LI&: *LI);
813 }
814 }
815
816 // If BB is in a loop and Succ is outside of that loop, we may need to
817 // update LoopSimplify form and LCSSA form.
818 if (!BBLoop->contains(BB: Succ)) {
819 assert(!BBLoop->contains(NewBB) &&
820 "Split point for loop exit is contained in loop!");
821
822 // Update LCSSA form in the newly created exit block.
823 if (Options.PreserveLCSSA) {
824 createPHIsForSplitLoopExit(Preds: BB, SplitBB: NewBB, DestBB: Succ);
825 }
826
827 if (!LoopPreds.empty()) {
828 BasicBlock *NewExitBB = SplitBlockPredecessors(
829 BB: Succ, Preds: LoopPreds, Suffix: "split", DT, LI, MSSAU, PreserveLCSSA: Options.PreserveLCSSA);
830 if (Options.PreserveLCSSA)
831 createPHIsForSplitLoopExit(Preds: LoopPreds, SplitBB: NewExitBB, DestBB: Succ);
832 }
833 }
834 }
835 }
836
837 return NewBB;
838}
839
840void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
841 BasicBlock *SplitBB, BasicBlock *DestBB) {
842 // SplitBB shouldn't have anything non-trivial in it yet.
843 assert((&*SplitBB->getFirstNonPHIIt() == SplitBB->getTerminator() ||
844 SplitBB->isLandingPad()) &&
845 "SplitBB has non-PHI nodes!");
846
847 // For each PHI in the destination block.
848 for (PHINode &PN : DestBB->phis()) {
849 int Idx = PN.getBasicBlockIndex(BB: SplitBB);
850 assert(Idx >= 0 && "Invalid Block Index");
851 Value *V = PN.getIncomingValue(i: Idx);
852
853 // If the input is a PHI which already satisfies LCSSA, don't create
854 // a new one.
855 if (const PHINode *VP = dyn_cast<PHINode>(Val: V))
856 if (VP->getParent() == SplitBB)
857 continue;
858
859 // Otherwise a new PHI is needed. Create one and populate it.
860 PHINode *NewPN = PHINode::Create(Ty: PN.getType(), NumReservedValues: Preds.size(), NameStr: "split");
861 BasicBlock::iterator InsertPos =
862 SplitBB->isLandingPad() ? SplitBB->begin()
863 : SplitBB->getTerminator()->getIterator();
864 NewPN->insertBefore(InsertPos);
865 for (BasicBlock *BB : Preds)
866 NewPN->addIncoming(V, BB);
867
868 // Update the original PHI.
869 PN.setIncomingValue(i: Idx, V: NewPN);
870 }
871}
872
873unsigned
874llvm::SplitAllCriticalEdges(Function &F,
875 const CriticalEdgeSplittingOptions &Options) {
876 unsigned NumBroken = 0;
877 for (BasicBlock &BB : F) {
878 Instruction *TI = BB.getTerminator();
879 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(Val: TI))
880 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
881 if (SplitCriticalEdge(TI, SuccNum: i, Options))
882 ++NumBroken;
883 }
884 return NumBroken;
885}
886
887static BasicBlock *SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt,
888 DomTreeUpdater *DTU, DominatorTree *DT,
889 LoopInfo *LI, MemorySSAUpdater *MSSAU,
890 const Twine &BBName, bool Before) {
891 if (Before) {
892 DomTreeUpdater LocalDTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
893 return splitBlockBefore(Old, SplitPt,
894 DTU: DTU ? DTU : (DT ? &LocalDTU : nullptr), LI, MSSAU,
895 BBName);
896 }
897 BasicBlock::iterator SplitIt = SplitPt;
898 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad()) {
899 ++SplitIt;
900 assert(SplitIt != SplitPt->getParent()->end());
901 }
902 std::string Name = BBName.str();
903 BasicBlock *New = Old->splitBasicBlock(
904 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name);
905
906 // The new block lives in whichever loop the old one did. This preserves
907 // LCSSA as well, because we force the split point to be after any PHI nodes.
908 if (LI)
909 if (Loop *L = LI->getLoopFor(BB: Old))
910 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
911
912 if (DTU) {
913 SmallVector<DominatorTree::UpdateType, 8> Updates;
914 // Old dominates New. New node dominates all other nodes dominated by Old.
915 SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
916 Updates.push_back(Elt: {DominatorTree::Insert, Old, New});
917 Updates.reserve(N: Updates.size() + 2 * succ_size(BB: New));
918 for (BasicBlock *SuccessorOfOld : successors(BB: New))
919 if (UniqueSuccessorsOfOld.insert(Ptr: SuccessorOfOld).second) {
920 Updates.push_back(Elt: {DominatorTree::Insert, New, SuccessorOfOld});
921 Updates.push_back(Elt: {DominatorTree::Delete, Old, SuccessorOfOld});
922 }
923
924 DTU->applyUpdates(Updates);
925 } else if (DT)
926 // Old dominates New. New node dominates all other nodes dominated by Old.
927 if (DomTreeNode *OldNode = DT->getNode(BB: Old)) {
928 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
929
930 DomTreeNode *NewNode = DT->addNewBlock(BB: New, DomBB: Old);
931 for (DomTreeNode *I : Children)
932 DT->changeImmediateDominator(N: I, NewIDom: NewNode);
933 }
934
935 // Move MemoryAccesses still tracked in Old, but part of New now.
936 // Update accesses in successor blocks accordingly.
937 if (MSSAU)
938 MSSAU->moveAllAfterSpliceBlocks(From: Old, To: New, Start: &*(New->begin()));
939
940 return New;
941}
942
943BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
944 DominatorTree *DT, LoopInfo *LI,
945 MemorySSAUpdater *MSSAU, const Twine &BBName,
946 bool Before) {
947 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName,
948 Before);
949}
950BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
951 DomTreeUpdater *DTU, LoopInfo *LI,
952 MemorySSAUpdater *MSSAU, const Twine &BBName,
953 bool Before) {
954 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName,
955 Before);
956}
957
958BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt,
959 DomTreeUpdater *DTU, LoopInfo *LI,
960 MemorySSAUpdater *MSSAU,
961 const Twine &BBName) {
962
963 BasicBlock::iterator SplitIt = SplitPt;
964 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad())
965 ++SplitIt;
966 std::string Name = BBName.str();
967 BasicBlock *New = Old->splitBasicBlock(
968 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name,
969 /* Before=*/true);
970
971 // The new block lives in whichever loop the old one did. This preserves
972 // LCSSA as well, because we force the split point to be after any PHI nodes.
973 if (LI)
974 if (Loop *L = LI->getLoopFor(BB: Old))
975 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
976
977 if (DTU) {
978 SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
979 // New dominates Old. The predecessor nodes of the Old node dominate
980 // New node.
981 SmallPtrSet<BasicBlock *, 8> UniquePredecessorsOfOld;
982 DTUpdates.push_back(Elt: {DominatorTree::Insert, New, Old});
983 DTUpdates.reserve(N: DTUpdates.size() + 2 * pred_size(BB: New));
984 for (BasicBlock *PredecessorOfOld : predecessors(BB: New))
985 if (UniquePredecessorsOfOld.insert(Ptr: PredecessorOfOld).second) {
986 DTUpdates.push_back(Elt: {DominatorTree::Insert, PredecessorOfOld, New});
987 DTUpdates.push_back(Elt: {DominatorTree::Delete, PredecessorOfOld, Old});
988 }
989
990 DTU->applyUpdates(Updates: DTUpdates);
991
992 // Move MemoryAccesses still tracked in Old, but part of New now.
993 // Update accesses in successor blocks accordingly.
994 if (MSSAU) {
995 MSSAU->applyUpdates(Updates: DTUpdates, DT&: DTU->getDomTree());
996 if (VerifyMemorySSA)
997 MSSAU->getMemorySSA()->verifyMemorySSA();
998 }
999 }
1000 return New;
1001}
1002
1003/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
1004/// Invalidates DFS Numbering when DTU or DT is provided.
1005static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
1006 ArrayRef<BasicBlock *> Preds,
1007 DomTreeUpdater *DTU, DominatorTree *DT,
1008 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1009 bool PreserveLCSSA, bool &HasLoopExit) {
1010 // Update dominator tree if available.
1011 if (DTU) {
1012 // Recalculation of DomTree is needed when updating a forward DomTree and
1013 // the Entry BB is replaced.
1014 if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1015 // The entry block was removed and there is no external interface for
1016 // the dominator tree to be notified of this change. In this corner-case
1017 // we recalculate the entire tree.
1018 DTU->recalculate(F&: *NewBB->getParent());
1019 } else {
1020 // Split block expects NewBB to have a non-empty set of predecessors.
1021 SmallVector<DominatorTree::UpdateType, 8> Updates;
1022 SmallPtrSet<BasicBlock *, 8> UniquePreds;
1023 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, OldBB});
1024 Updates.reserve(N: Updates.size() + 2 * Preds.size());
1025 for (auto *Pred : Preds)
1026 if (UniquePreds.insert(Ptr: Pred).second) {
1027 Updates.push_back(Elt: {DominatorTree::Insert, Pred, NewBB});
1028 Updates.push_back(Elt: {DominatorTree::Delete, Pred, OldBB});
1029 }
1030 DTU->applyUpdates(Updates);
1031 }
1032 } else if (DT) {
1033 if (OldBB == DT->getRootNode()->getBlock()) {
1034 assert(NewBB->isEntryBlock());
1035 DT->setNewRoot(NewBB);
1036 } else {
1037 // Split block expects NewBB to have a non-empty set of predecessors.
1038 DT->splitBlock(NewBB);
1039 }
1040 }
1041
1042 // Update MemoryPhis after split if MemorySSA is available
1043 if (MSSAU)
1044 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(Old: OldBB, New: NewBB, Preds);
1045
1046 // The rest of the logic is only relevant for updating the loop structures.
1047 if (!LI)
1048 return;
1049
1050 if (DTU && DTU->hasDomTree())
1051 DT = &DTU->getDomTree();
1052 assert(DT && "DT should be available to update LoopInfo!");
1053 Loop *L = LI->getLoopFor(BB: OldBB);
1054
1055 // If we need to preserve loop analyses, collect some information about how
1056 // this split will affect loops.
1057 bool IsLoopEntry = !!L;
1058 bool SplitMakesNewLoopHeader = false;
1059 for (BasicBlock *Pred : Preds) {
1060 // Preds that are not reachable from entry should not be used to identify if
1061 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1062 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1063 // as true and make the NewBB the header of some loop. This breaks LI.
1064 if (!DT->isReachableFromEntry(A: Pred))
1065 continue;
1066 // If we need to preserve LCSSA, determine if any of the preds is a loop
1067 // exit.
1068 if (PreserveLCSSA)
1069 if (Loop *PL = LI->getLoopFor(BB: Pred))
1070 if (!PL->contains(BB: OldBB))
1071 HasLoopExit = true;
1072
1073 // If we need to preserve LoopInfo, note whether any of the preds crosses
1074 // an interesting loop boundary.
1075 if (!L)
1076 continue;
1077 if (L->contains(BB: Pred))
1078 IsLoopEntry = false;
1079 else
1080 SplitMakesNewLoopHeader = true;
1081 }
1082
1083 // Unless we have a loop for OldBB, nothing else to do here.
1084 if (!L)
1085 return;
1086
1087 if (IsLoopEntry) {
1088 // Add the new block to the nearest enclosing loop (and not an adjacent
1089 // loop). To find this, examine each of the predecessors and determine which
1090 // loops enclose them, and select the most-nested loop which contains the
1091 // loop containing the block being split.
1092 Loop *InnermostPredLoop = nullptr;
1093 for (BasicBlock *Pred : Preds) {
1094 if (Loop *PredLoop = LI->getLoopFor(BB: Pred)) {
1095 // Seek a loop which actually contains the block being split (to avoid
1096 // adjacent loops).
1097 while (PredLoop && !PredLoop->contains(BB: OldBB))
1098 PredLoop = PredLoop->getParentLoop();
1099
1100 // Select the most-nested of these loops which contains the block.
1101 if (PredLoop && PredLoop->contains(BB: OldBB) &&
1102 (!InnermostPredLoop ||
1103 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1104 InnermostPredLoop = PredLoop;
1105 }
1106 }
1107
1108 if (InnermostPredLoop)
1109 InnermostPredLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1110 } else {
1111 L->addBasicBlockToLoop(NewBB, LI&: *LI);
1112 if (SplitMakesNewLoopHeader)
1113 L->moveToHeader(BB: NewBB);
1114 }
1115}
1116
1117/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1118/// This also updates AliasAnalysis, if available.
1119static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1120 ArrayRef<BasicBlock *> Preds, BranchInst *BI,
1121 bool HasLoopExit) {
1122 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1123 SmallPtrSet<BasicBlock *, 16> PredSet(llvm::from_range, Preds);
1124 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(Val: I); ) {
1125 PHINode *PN = cast<PHINode>(Val: I++);
1126
1127 // Check to see if all of the values coming in are the same. If so, we
1128 // don't need to create a new PHI node, unless it's needed for LCSSA.
1129 Value *InVal = nullptr;
1130 if (!HasLoopExit) {
1131 InVal = PN->getIncomingValueForBlock(BB: Preds[0]);
1132 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1133 if (!PredSet.count(Ptr: PN->getIncomingBlock(i)))
1134 continue;
1135 if (!InVal)
1136 InVal = PN->getIncomingValue(i);
1137 else if (InVal != PN->getIncomingValue(i)) {
1138 InVal = nullptr;
1139 break;
1140 }
1141 }
1142 }
1143
1144 if (InVal) {
1145 // If all incoming values for the new PHI would be the same, just don't
1146 // make a new PHI. Instead, just remove the incoming values from the old
1147 // PHI.
1148 PN->removeIncomingValueIf(
1149 Predicate: [&](unsigned Idx) {
1150 return PredSet.contains(Ptr: PN->getIncomingBlock(i: Idx));
1151 },
1152 /* DeletePHIIfEmpty */ false);
1153
1154 // Add an incoming value to the PHI node in the loop for the preheader
1155 // edge.
1156 PN->addIncoming(V: InVal, BB: NewBB);
1157 continue;
1158 }
1159
1160 // If the values coming into the block are not the same, we need a new
1161 // PHI.
1162 // Create the new PHI node, insert it into NewBB at the end of the block
1163 PHINode *NewPHI =
1164 PHINode::Create(Ty: PN->getType(), NumReservedValues: Preds.size(), NameStr: PN->getName() + ".ph", InsertBefore: BI->getIterator());
1165
1166 // NOTE! This loop walks backwards for a reason! First off, this minimizes
1167 // the cost of removal if we end up removing a large number of values, and
1168 // second off, this ensures that the indices for the incoming values aren't
1169 // invalidated when we remove one.
1170 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1171 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1172 if (PredSet.count(Ptr: IncomingBB)) {
1173 Value *V = PN->removeIncomingValue(Idx: i, DeletePHIIfEmpty: false);
1174 NewPHI->addIncoming(V, BB: IncomingBB);
1175 }
1176 }
1177
1178 PN->addIncoming(V: NewPHI, BB: NewBB);
1179 }
1180}
1181
1182static void SplitLandingPadPredecessorsImpl(
1183 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1184 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1185 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1186 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1187
1188static BasicBlock *
1189SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
1190 const char *Suffix, DomTreeUpdater *DTU,
1191 DominatorTree *DT, LoopInfo *LI,
1192 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1193 // Do not attempt to split that which cannot be split.
1194 if (!BB->canSplitPredecessors())
1195 return nullptr;
1196
1197 // For the landingpads we need to act a bit differently.
1198 // Delegate this work to the SplitLandingPadPredecessors.
1199 if (BB->isLandingPad()) {
1200 SmallVector<BasicBlock*, 2> NewBBs;
1201 std::string NewName = std::string(Suffix) + ".split-lp";
1202
1203 SplitLandingPadPredecessorsImpl(OrigBB: BB, Preds, Suffix1: Suffix, Suffix2: NewName.c_str(), NewBBs,
1204 DTU, DT, LI, MSSAU, PreserveLCSSA);
1205 return NewBBs[0];
1206 }
1207
1208 // Create new basic block, insert right before the original block.
1209 BasicBlock *NewBB = BasicBlock::Create(
1210 Context&: BB->getContext(), Name: BB->getName() + Suffix, Parent: BB->getParent(), InsertBefore: BB);
1211
1212 // The new block unconditionally branches to the old block.
1213 BranchInst *BI = BranchInst::Create(IfTrue: BB, InsertBefore: NewBB);
1214
1215 Loop *L = nullptr;
1216 BasicBlock *OldLatch = nullptr;
1217 // Splitting the predecessors of a loop header creates a preheader block.
1218 if (LI && LI->isLoopHeader(BB)) {
1219 L = LI->getLoopFor(BB);
1220 // Using the loop start line number prevents debuggers stepping into the
1221 // loop body for this instruction.
1222 BI->setDebugLoc(L->getStartLoc());
1223
1224 // If BB is the header of the Loop, it is possible that the loop is
1225 // modified, such that the current latch does not remain the latch of the
1226 // loop. If that is the case, the loop metadata from the current latch needs
1227 // to be applied to the new latch.
1228 OldLatch = L->getLoopLatch();
1229 } else
1230 BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
1231
1232 // Move the edges from Preds to point to NewBB instead of BB.
1233 for (BasicBlock *Pred : Preds) {
1234 // This is slightly more strict than necessary; the minimum requirement
1235 // is that there be no more than one indirectbr branching to BB. And
1236 // all BlockAddress uses would need to be updated.
1237 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1238 "Cannot split an edge from an IndirectBrInst");
1239 Pred->getTerminator()->replaceSuccessorWith(OldBB: BB, NewBB);
1240 }
1241
1242 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1243 // node becomes an incoming value for BB's phi node. However, if the Preds
1244 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1245 // account for the newly created predecessor.
1246 if (Preds.empty()) {
1247 // Insert dummy values as the incoming value.
1248 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(Val: I); ++I)
1249 cast<PHINode>(Val&: I)->addIncoming(V: PoisonValue::get(T: I->getType()), BB: NewBB);
1250 }
1251
1252 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1253 bool HasLoopExit = false;
1254 UpdateAnalysisInformation(OldBB: BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1255 HasLoopExit);
1256
1257 if (!Preds.empty()) {
1258 // Update the PHI nodes in BB with the values coming from NewBB.
1259 UpdatePHINodes(OrigBB: BB, NewBB, Preds, BI, HasLoopExit);
1260 }
1261
1262 if (OldLatch) {
1263 BasicBlock *NewLatch = L->getLoopLatch();
1264 if (NewLatch != OldLatch) {
1265 MDNode *MD = OldLatch->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
1266 NewLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: MD);
1267 // It's still possible that OldLatch is the latch of another inner loop,
1268 // in which case we do not remove the metadata.
1269 Loop *IL = LI->getLoopFor(BB: OldLatch);
1270 if (IL && IL->getLoopLatch() != OldLatch)
1271 OldLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: nullptr);
1272 }
1273 }
1274
1275 return NewBB;
1276}
1277
1278BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1279 ArrayRef<BasicBlock *> Preds,
1280 const char *Suffix, DominatorTree *DT,
1281 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1282 bool PreserveLCSSA) {
1283 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1284 MSSAU, PreserveLCSSA);
1285}
1286BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1287 ArrayRef<BasicBlock *> Preds,
1288 const char *Suffix,
1289 DomTreeUpdater *DTU, LoopInfo *LI,
1290 MemorySSAUpdater *MSSAU,
1291 bool PreserveLCSSA) {
1292 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1293 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1294}
1295
1296static void SplitLandingPadPredecessorsImpl(
1297 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1298 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1299 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1300 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1301 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1302
1303 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1304 // it right before the original block.
1305 BasicBlock *NewBB1 = BasicBlock::Create(Context&: OrigBB->getContext(),
1306 Name: OrigBB->getName() + Suffix1,
1307 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1308 NewBBs.push_back(Elt: NewBB1);
1309
1310 // The new block unconditionally branches to the old block.
1311 BranchInst *BI1 = BranchInst::Create(IfTrue: OrigBB, InsertBefore: NewBB1);
1312 BI1->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1313
1314 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1315 for (BasicBlock *Pred : Preds) {
1316 // This is slightly more strict than necessary; the minimum requirement
1317 // is that there be no more than one indirectbr branching to BB. And
1318 // all BlockAddress uses would need to be updated.
1319 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1320 "Cannot split an edge from an IndirectBrInst");
1321 Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB1);
1322 }
1323
1324 bool HasLoopExit = false;
1325 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB1, Preds, DTU, DT, LI, MSSAU,
1326 PreserveLCSSA, HasLoopExit);
1327
1328 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1329 UpdatePHINodes(OrigBB, NewBB: NewBB1, Preds, BI: BI1, HasLoopExit);
1330
1331 // Move the remaining edges from OrigBB to point to NewBB2.
1332 SmallVector<BasicBlock*, 8> NewBB2Preds;
1333 for (pred_iterator i = pred_begin(BB: OrigBB), e = pred_end(BB: OrigBB);
1334 i != e; ) {
1335 BasicBlock *Pred = *i++;
1336 if (Pred == NewBB1) continue;
1337 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1338 "Cannot split an edge from an IndirectBrInst");
1339 NewBB2Preds.push_back(Elt: Pred);
1340 e = pred_end(BB: OrigBB);
1341 }
1342
1343 BasicBlock *NewBB2 = nullptr;
1344 if (!NewBB2Preds.empty()) {
1345 // Create another basic block for the rest of OrigBB's predecessors.
1346 NewBB2 = BasicBlock::Create(Context&: OrigBB->getContext(),
1347 Name: OrigBB->getName() + Suffix2,
1348 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1349 NewBBs.push_back(Elt: NewBB2);
1350
1351 // The new block unconditionally branches to the old block.
1352 BranchInst *BI2 = BranchInst::Create(IfTrue: OrigBB, InsertBefore: NewBB2);
1353 BI2->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1354
1355 // Move the remaining edges from OrigBB to point to NewBB2.
1356 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1357 NewBB2Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB2);
1358
1359 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1360 HasLoopExit = false;
1361 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, DTU, DT, LI, MSSAU,
1362 PreserveLCSSA, HasLoopExit);
1363
1364 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1365 UpdatePHINodes(OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, BI: BI2, HasLoopExit);
1366 }
1367
1368 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1369 Instruction *Clone1 = LPad->clone();
1370 Clone1->setName(Twine("lpad") + Suffix1);
1371 Clone1->insertInto(ParentBB: NewBB1, It: NewBB1->getFirstInsertionPt());
1372
1373 if (NewBB2) {
1374 Instruction *Clone2 = LPad->clone();
1375 Clone2->setName(Twine("lpad") + Suffix2);
1376 Clone2->insertInto(ParentBB: NewBB2, It: NewBB2->getFirstInsertionPt());
1377
1378 // Create a PHI node for the two cloned landingpad instructions only
1379 // if the original landingpad instruction has some uses.
1380 if (!LPad->use_empty()) {
1381 assert(!LPad->getType()->isTokenTy() &&
1382 "Split cannot be applied if LPad is token type. Otherwise an "
1383 "invalid PHINode of token type would be created.");
1384 PHINode *PN = PHINode::Create(Ty: LPad->getType(), NumReservedValues: 2, NameStr: "lpad.phi", InsertBefore: LPad->getIterator());
1385 PN->addIncoming(V: Clone1, BB: NewBB1);
1386 PN->addIncoming(V: Clone2, BB: NewBB2);
1387 LPad->replaceAllUsesWith(V: PN);
1388 }
1389 LPad->eraseFromParent();
1390 } else {
1391 // There is no second clone. Just replace the landing pad with the first
1392 // clone.
1393 LPad->replaceAllUsesWith(V: Clone1);
1394 LPad->eraseFromParent();
1395 }
1396}
1397
1398void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1399 ArrayRef<BasicBlock *> Preds,
1400 const char *Suffix1, const char *Suffix2,
1401 SmallVectorImpl<BasicBlock *> &NewBBs,
1402 DomTreeUpdater *DTU, LoopInfo *LI,
1403 MemorySSAUpdater *MSSAU,
1404 bool PreserveLCSSA) {
1405 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1406 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1407 PreserveLCSSA);
1408}
1409
1410ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1411 BasicBlock *Pred,
1412 DomTreeUpdater *DTU) {
1413 Instruction *UncondBranch = Pred->getTerminator();
1414 // Clone the return and add it to the end of the predecessor.
1415 Instruction *NewRet = RI->clone();
1416 NewRet->insertInto(ParentBB: Pred, It: Pred->end());
1417
1418 // If the return instruction returns a value, and if the value was a
1419 // PHI node in "BB", propagate the right value into the return.
1420 for (Use &Op : NewRet->operands()) {
1421 Value *V = Op;
1422 Instruction *NewBC = nullptr;
1423 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Val: V)) {
1424 // Return value might be bitcasted. Clone and insert it before the
1425 // return instruction.
1426 V = BCI->getOperand(i_nocapture: 0);
1427 NewBC = BCI->clone();
1428 NewBC->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1429 Op = NewBC;
1430 }
1431
1432 Instruction *NewEV = nullptr;
1433 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: V)) {
1434 V = EVI->getOperand(i_nocapture: 0);
1435 NewEV = EVI->clone();
1436 if (NewBC) {
1437 NewBC->setOperand(i: 0, Val: NewEV);
1438 NewEV->insertInto(ParentBB: Pred, It: NewBC->getIterator());
1439 } else {
1440 NewEV->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1441 Op = NewEV;
1442 }
1443 }
1444
1445 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
1446 if (PN->getParent() == BB) {
1447 if (NewEV) {
1448 NewEV->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1449 } else if (NewBC)
1450 NewBC->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1451 else
1452 Op = PN->getIncomingValueForBlock(BB: Pred);
1453 }
1454 }
1455 }
1456
1457 // Update any PHI nodes in the returning block to realize that we no
1458 // longer branch to them.
1459 BB->removePredecessor(Pred);
1460 UncondBranch->eraseFromParent();
1461
1462 if (DTU)
1463 DTU->applyUpdates(Updates: {{DominatorTree::Delete, Pred, BB}});
1464
1465 return cast<ReturnInst>(Val: NewRet);
1466}
1467
1468Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1469 BasicBlock::iterator SplitBefore,
1470 bool Unreachable,
1471 MDNode *BranchWeights,
1472 DomTreeUpdater *DTU, LoopInfo *LI,
1473 BasicBlock *ThenBlock) {
1474 SplitBlockAndInsertIfThenElse(
1475 Cond, SplitBefore, ThenBlock: &ThenBlock, /* ElseBlock */ nullptr,
1476 /* UnreachableThen */ Unreachable,
1477 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1478 return ThenBlock->getTerminator();
1479}
1480
1481Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
1482 BasicBlock::iterator SplitBefore,
1483 bool Unreachable,
1484 MDNode *BranchWeights,
1485 DomTreeUpdater *DTU, LoopInfo *LI,
1486 BasicBlock *ElseBlock) {
1487 SplitBlockAndInsertIfThenElse(
1488 Cond, SplitBefore, /* ThenBlock */ nullptr, ElseBlock: &ElseBlock,
1489 /* UnreachableThen */ false,
1490 /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1491 return ElseBlock->getTerminator();
1492}
1493
1494void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore,
1495 Instruction **ThenTerm,
1496 Instruction **ElseTerm,
1497 MDNode *BranchWeights,
1498 DomTreeUpdater *DTU, LoopInfo *LI) {
1499 BasicBlock *ThenBlock = nullptr;
1500 BasicBlock *ElseBlock = nullptr;
1501 SplitBlockAndInsertIfThenElse(
1502 Cond, SplitBefore, ThenBlock: &ThenBlock, ElseBlock: &ElseBlock, /* UnreachableThen */ false,
1503 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1504
1505 *ThenTerm = ThenBlock->getTerminator();
1506 *ElseTerm = ElseBlock->getTerminator();
1507}
1508
1509void llvm::SplitBlockAndInsertIfThenElse(
1510 Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
1511 BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1512 MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1513 assert((ThenBlock || ElseBlock) &&
1514 "At least one branch block must be created");
1515 assert((!UnreachableThen || !UnreachableElse) &&
1516 "Split block tail must be reachable");
1517
1518 SmallVector<DominatorTree::UpdateType, 8> Updates;
1519 SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1520 BasicBlock *Head = SplitBefore->getParent();
1521 if (DTU) {
1522 UniqueOrigSuccessors.insert_range(R: successors(BB: Head));
1523 Updates.reserve(N: 4 + 2 * UniqueOrigSuccessors.size());
1524 }
1525
1526 LLVMContext &C = Head->getContext();
1527 BasicBlock *Tail = Head->splitBasicBlock(I: SplitBefore);
1528 BasicBlock *TrueBlock = Tail;
1529 BasicBlock *FalseBlock = Tail;
1530 bool ThenToTailEdge = false;
1531 bool ElseToTailEdge = false;
1532
1533 // Encapsulate the logic around creation/insertion/etc of a new block.
1534 auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1535 bool &ToTailEdge) {
1536 if (PBB == nullptr)
1537 return; // Do not create/insert a block.
1538
1539 if (*PBB)
1540 BB = *PBB; // Caller supplied block, use it.
1541 else {
1542 // Create a new block.
1543 BB = BasicBlock::Create(Context&: C, Name: "", Parent: Head->getParent(), InsertBefore: Tail);
1544 if (Unreachable)
1545 (void)new UnreachableInst(C, BB);
1546 else {
1547 (void)BranchInst::Create(IfTrue: Tail, InsertBefore: BB);
1548 ToTailEdge = true;
1549 }
1550 BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1551 // Pass the new block back to the caller.
1552 *PBB = BB;
1553 }
1554 };
1555
1556 handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1557 handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1558
1559 Instruction *HeadOldTerm = Head->getTerminator();
1560 BranchInst *HeadNewTerm =
1561 BranchInst::Create(/*ifTrue*/ IfTrue: TrueBlock, /*ifFalse*/ IfFalse: FalseBlock, Cond);
1562 HeadNewTerm->setMetadata(KindID: LLVMContext::MD_prof, Node: BranchWeights);
1563 ReplaceInstWithInst(From: HeadOldTerm, To: HeadNewTerm);
1564
1565 if (DTU) {
1566 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: TrueBlock);
1567 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: FalseBlock);
1568 if (ThenToTailEdge)
1569 Updates.emplace_back(Args: DominatorTree::Insert, Args&: TrueBlock, Args&: Tail);
1570 if (ElseToTailEdge)
1571 Updates.emplace_back(Args: DominatorTree::Insert, Args&: FalseBlock, Args&: Tail);
1572 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1573 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Tail, Args&: UniqueOrigSuccessor);
1574 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1575 Updates.emplace_back(Args: DominatorTree::Delete, Args&: Head, Args&: UniqueOrigSuccessor);
1576 DTU->applyUpdates(Updates);
1577 }
1578
1579 if (LI) {
1580 if (Loop *L = LI->getLoopFor(BB: Head); L) {
1581 if (ThenToTailEdge)
1582 L->addBasicBlockToLoop(NewBB: TrueBlock, LI&: *LI);
1583 if (ElseToTailEdge)
1584 L->addBasicBlockToLoop(NewBB: FalseBlock, LI&: *LI);
1585 L->addBasicBlockToLoop(NewBB: Tail, LI&: *LI);
1586 }
1587 }
1588}
1589
1590std::pair<Instruction *, Value *>
1591llvm::SplitBlockAndInsertSimpleForLoop(Value *End,
1592 BasicBlock::iterator SplitBefore) {
1593 BasicBlock *LoopPred = SplitBefore->getParent();
1594 BasicBlock *LoopBody = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1595 BasicBlock *LoopExit = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1596
1597 auto *Ty = End->getType();
1598 auto &DL = SplitBefore->getDataLayout();
1599 const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1600
1601 IRBuilder<> Builder(LoopBody->getTerminator());
1602 auto *IV = Builder.CreatePHI(Ty, NumReservedValues: 2, Name: "iv");
1603 auto *IVNext =
1604 Builder.CreateAdd(LHS: IV, RHS: ConstantInt::get(Ty, V: 1), Name: IV->getName() + ".next",
1605 /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1606 auto *IVCheck = Builder.CreateICmpEQ(LHS: IVNext, RHS: End,
1607 Name: IV->getName() + ".check");
1608 Builder.CreateCondBr(Cond: IVCheck, True: LoopExit, False: LoopBody);
1609 LoopBody->getTerminator()->eraseFromParent();
1610
1611 // Populate the IV PHI.
1612 IV->addIncoming(V: ConstantInt::get(Ty, V: 0), BB: LoopPred);
1613 IV->addIncoming(V: IVNext, BB: LoopBody);
1614
1615 return std::make_pair(x: &*LoopBody->getFirstNonPHIIt(), y&: IV);
1616}
1617
1618void llvm::SplitBlockAndInsertForEachLane(
1619 ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
1620 std::function<void(IRBuilderBase &, Value *)> Func) {
1621
1622 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1623
1624 if (EC.isScalable()) {
1625 Value *NumElements = IRB.CreateElementCount(Ty: IndexTy, EC);
1626
1627 auto [BodyIP, Index] =
1628 SplitBlockAndInsertSimpleForLoop(End: NumElements, SplitBefore: InsertBefore);
1629
1630 IRB.SetInsertPoint(BodyIP);
1631 Func(IRB, Index);
1632 return;
1633 }
1634
1635 unsigned Num = EC.getFixedValue();
1636 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1637 IRB.SetInsertPoint(InsertBefore);
1638 Func(IRB, ConstantInt::get(Ty: IndexTy, V: Idx));
1639 }
1640}
1641
1642void llvm::SplitBlockAndInsertForEachLane(
1643 Value *EVL, BasicBlock::iterator InsertBefore,
1644 std::function<void(IRBuilderBase &, Value *)> Func) {
1645
1646 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1647 Type *Ty = EVL->getType();
1648
1649 if (!isa<ConstantInt>(Val: EVL)) {
1650 auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(End: EVL, SplitBefore: InsertBefore);
1651 IRB.SetInsertPoint(BodyIP);
1652 Func(IRB, Index);
1653 return;
1654 }
1655
1656 unsigned Num = cast<ConstantInt>(Val: EVL)->getZExtValue();
1657 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1658 IRB.SetInsertPoint(InsertBefore);
1659 Func(IRB, ConstantInt::get(Ty, V: Idx));
1660 }
1661}
1662
1663BranchInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1664 BasicBlock *&IfFalse) {
1665 PHINode *SomePHI = dyn_cast<PHINode>(Val: BB->begin());
1666 BasicBlock *Pred1 = nullptr;
1667 BasicBlock *Pred2 = nullptr;
1668
1669 if (SomePHI) {
1670 if (SomePHI->getNumIncomingValues() != 2)
1671 return nullptr;
1672 Pred1 = SomePHI->getIncomingBlock(i: 0);
1673 Pred2 = SomePHI->getIncomingBlock(i: 1);
1674 } else {
1675 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1676 if (PI == PE) // No predecessor
1677 return nullptr;
1678 Pred1 = *PI++;
1679 if (PI == PE) // Only one predecessor
1680 return nullptr;
1681 Pred2 = *PI++;
1682 if (PI != PE) // More than two predecessors
1683 return nullptr;
1684 }
1685
1686 // We can only handle branches. Other control flow will be lowered to
1687 // branches if possible anyway.
1688 BranchInst *Pred1Br = dyn_cast<BranchInst>(Val: Pred1->getTerminator());
1689 BranchInst *Pred2Br = dyn_cast<BranchInst>(Val: Pred2->getTerminator());
1690 if (!Pred1Br || !Pred2Br)
1691 return nullptr;
1692
1693 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1694 // either are.
1695 if (Pred2Br->isConditional()) {
1696 // If both branches are conditional, we don't have an "if statement". In
1697 // reality, we could transform this case, but since the condition will be
1698 // required anyway, we stand no chance of eliminating it, so the xform is
1699 // probably not profitable.
1700 if (Pred1Br->isConditional())
1701 return nullptr;
1702
1703 std::swap(a&: Pred1, b&: Pred2);
1704 std::swap(a&: Pred1Br, b&: Pred2Br);
1705 }
1706
1707 if (Pred1Br->isConditional()) {
1708 // The only thing we have to watch out for here is to make sure that Pred2
1709 // doesn't have incoming edges from other blocks. If it does, the condition
1710 // doesn't dominate BB.
1711 if (!Pred2->getSinglePredecessor())
1712 return nullptr;
1713
1714 // If we found a conditional branch predecessor, make sure that it branches
1715 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1716 if (Pred1Br->getSuccessor(i: 0) == BB &&
1717 Pred1Br->getSuccessor(i: 1) == Pred2) {
1718 IfTrue = Pred1;
1719 IfFalse = Pred2;
1720 } else if (Pred1Br->getSuccessor(i: 0) == Pred2 &&
1721 Pred1Br->getSuccessor(i: 1) == BB) {
1722 IfTrue = Pred2;
1723 IfFalse = Pred1;
1724 } else {
1725 // We know that one arm of the conditional goes to BB, so the other must
1726 // go somewhere unrelated, and this must not be an "if statement".
1727 return nullptr;
1728 }
1729
1730 return Pred1Br;
1731 }
1732
1733 // Ok, if we got here, both predecessors end with an unconditional branch to
1734 // BB. Don't panic! If both blocks only have a single (identical)
1735 // predecessor, and THAT is a conditional branch, then we're all ok!
1736 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1737 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1738 return nullptr;
1739
1740 // Otherwise, if this is a conditional branch, then we can use it!
1741 BranchInst *BI = dyn_cast<BranchInst>(Val: CommonPred->getTerminator());
1742 if (!BI) return nullptr;
1743
1744 assert(BI->isConditional() && "Two successors but not conditional?");
1745 if (BI->getSuccessor(i: 0) == Pred1) {
1746 IfTrue = Pred1;
1747 IfFalse = Pred2;
1748 } else {
1749 IfTrue = Pred2;
1750 IfFalse = Pred1;
1751 }
1752 return BI;
1753}
1754
1755void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
1756 Value *NewCond = PBI->getCondition();
1757 // If this is a "cmp" instruction, only used for branching (and nowhere
1758 // else), then we can simply invert the predicate.
1759 if (NewCond->hasOneUse() && isa<CmpInst>(Val: NewCond)) {
1760 CmpInst *CI = cast<CmpInst>(Val: NewCond);
1761 CI->setPredicate(CI->getInversePredicate());
1762 } else
1763 NewCond = Builder.CreateNot(V: NewCond, Name: NewCond->getName() + ".not");
1764
1765 PBI->setCondition(NewCond);
1766 PBI->swapSuccessors();
1767}
1768
1769bool llvm::hasOnlySimpleTerminator(const Function &F) {
1770 for (auto &BB : F) {
1771 auto *Term = BB.getTerminator();
1772 if (!(isa<ReturnInst>(Val: Term) || isa<UnreachableInst>(Val: Term) ||
1773 isa<BranchInst>(Val: Term)))
1774 return false;
1775 }
1776 return true;
1777}
1778