1//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions perform manipulations on basic blocks, and
10// instructions contained within basic blocks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/BasicBlockUtils.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallPtrSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/CFG.h"
20#include "llvm/Analysis/DomTreeUpdater.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/Analysis/MemoryDependenceAnalysis.h"
23#include "llvm/Analysis/MemorySSAUpdater.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/CFG.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/Dominators.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/InstrTypes.h"
33#include "llvm/IR/Instruction.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/LLVMContext.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/User.h"
38#include "llvm/IR/Value.h"
39#include "llvm/IR/ValueHandle.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/raw_ostream.h"
44#include "llvm/Transforms/Utils/Local.h"
45#include <cassert>
46#include <cstdint>
47#include <string>
48#include <utility>
49#include <vector>
50
51using namespace llvm;
52
53#define DEBUG_TYPE "basicblock-utils"
54
55static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
56 "max-deopt-or-unreachable-succ-check-depth", cl::init(Val: 8), cl::Hidden,
57 cl::desc("Set the maximum path length when checking whether a basic block "
58 "is followed by a block that either has a terminating "
59 "deoptimizing call or is terminated with an unreachable"));
60
61/// Zap all the instructions in the block and replace them with an unreachable
62/// instruction and notify the basic block's successors that one of their
63/// predecessors is going away.
64static void
65emptyAndDetachBlock(BasicBlock *BB,
66 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
67 bool KeepOneInputPHIs) {
68 // Loop through all of our successors and make sure they know that one
69 // of their predecessors is going away.
70 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
71 for (BasicBlock *Succ : successors(BB)) {
72 Succ->removePredecessor(Pred: BB, KeepOneInputPHIs);
73 if (Updates && UniqueSuccessors.insert(Ptr: Succ).second)
74 Updates->push_back(Elt: {DominatorTree::Delete, BB, Succ});
75 }
76
77 // Zap all the instructions in the block.
78 while (!BB->empty()) {
79 Instruction &I = BB->back();
80 // If this instruction is used, replace uses with an arbitrary value.
81 // Because control flow can't get here, we don't care what we replace the
82 // value with. Note that since this block is unreachable, and all values
83 // contained within it must dominate their uses, that all uses will
84 // eventually be removed (they are themselves dead).
85 if (!I.use_empty())
86 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
87 BB->back().eraseFromParent();
88 }
89 new UnreachableInst(BB->getContext(), BB);
90 assert(BB->size() == 1 && isa<UnreachableInst>(BB->getTerminator()) &&
91 "The successor list of BB isn't empty before "
92 "applying corresponding DTU updates.");
93}
94
95bool llvm::HasLoopOrEntryConvergenceToken(const BasicBlock *BB) {
96 for (const Instruction &I : *BB) {
97 const ConvergenceControlInst *CCI = dyn_cast<ConvergenceControlInst>(Val: &I);
98 if (CCI && (CCI->isLoop() || CCI->isEntry()))
99 return true;
100 }
101 return false;
102}
103
104void llvm::detachDeadBlocks(ArrayRef<BasicBlock *> BBs,
105 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
106 bool KeepOneInputPHIs) {
107 SmallPtrSet<BasicBlock *, 4> UniqueEHRetBlocksToDelete;
108 for (auto *BB : BBs) {
109 auto NonFirstPhiIt = BB->getFirstNonPHIIt();
110 if (NonFirstPhiIt != BB->end()) {
111 Instruction &I = *NonFirstPhiIt;
112 // Exception handling funclets need to be explicitly addressed.
113 // These funclets must begin with cleanuppad or catchpad and end with
114 // cleanupred or catchret. The return instructions can be in different
115 // basic blocks than the pad instruction. If we would only delete the
116 // first block, the we would have possible cleanupret and catchret
117 // instructions with poison arguments, which wouldn't be valid.
118 if (isa<FuncletPadInst>(Val: I)) {
119 UniqueEHRetBlocksToDelete.clear();
120
121 for (User *User : I.users()) {
122 Instruction *ReturnInstr = dyn_cast<Instruction>(Val: User);
123 // If we have a cleanupret or catchret block, replace it with just an
124 // unreachable. The other alternative, that may use a catchpad is a
125 // catchswitch. That does not need special handling for now.
126 if (isa<CatchReturnInst>(Val: ReturnInstr) ||
127 isa<CleanupReturnInst>(Val: ReturnInstr)) {
128 BasicBlock *ReturnInstrBB = ReturnInstr->getParent();
129 UniqueEHRetBlocksToDelete.insert(Ptr: ReturnInstrBB);
130 }
131 }
132
133 for (BasicBlock *EHRetBB : UniqueEHRetBlocksToDelete)
134 emptyAndDetachBlock(BB: EHRetBB, Updates, KeepOneInputPHIs);
135 }
136 }
137
138 UniqueEHRetBlocksToDelete.clear();
139
140 // Detaching and emptying the current basic block.
141 emptyAndDetachBlock(BB, Updates, KeepOneInputPHIs);
142 }
143}
144
145void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
146 bool KeepOneInputPHIs) {
147 DeleteDeadBlocks(BBs: {BB}, DTU, KeepOneInputPHIs);
148}
149
150void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
151 bool KeepOneInputPHIs) {
152#ifndef NDEBUG
153 // Make sure that all predecessors of each dead block is also dead.
154 SmallPtrSet<BasicBlock *, 4> Dead(llvm::from_range, BBs);
155 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
156 for (auto *BB : Dead)
157 for (BasicBlock *Pred : predecessors(BB))
158 assert(Dead.count(Pred) && "All predecessors must be dead!");
159#endif
160
161 SmallVector<DominatorTree::UpdateType, 4> Updates;
162 detachDeadBlocks(BBs, Updates: DTU ? &Updates : nullptr, KeepOneInputPHIs);
163
164 if (DTU)
165 DTU->applyUpdates(Updates);
166
167 for (BasicBlock *BB : BBs)
168 if (DTU)
169 DTU->deleteBB(DelBB: BB);
170 else
171 BB->eraseFromParent();
172}
173
174bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
175 bool KeepOneInputPHIs) {
176 df_iterator_default_set<BasicBlock*> Reachable;
177
178 // Mark all reachable blocks.
179 for (BasicBlock *BB : depth_first_ext(G: &F, S&: Reachable))
180 (void)BB/* Mark all reachable blocks */;
181
182 // Collect all dead blocks.
183 std::vector<BasicBlock*> DeadBlocks;
184 for (BasicBlock &BB : F)
185 if (!Reachable.count(Ptr: &BB))
186 DeadBlocks.push_back(x: &BB);
187
188 // Delete the dead blocks.
189 DeleteDeadBlocks(BBs: DeadBlocks, DTU, KeepOneInputPHIs);
190
191 return !DeadBlocks.empty();
192}
193
194bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
195 MemoryDependenceResults *MemDep) {
196 if (!isa<PHINode>(Val: BB->begin()))
197 return false;
198
199 while (PHINode *PN = dyn_cast<PHINode>(Val: BB->begin())) {
200 if (PN->getIncomingValue(i: 0) != PN)
201 PN->replaceAllUsesWith(V: PN->getIncomingValue(i: 0));
202 else
203 PN->replaceAllUsesWith(V: PoisonValue::get(T: PN->getType()));
204
205 if (MemDep)
206 MemDep->removeInstruction(InstToRemove: PN); // Memdep updates AA itself.
207
208 PN->eraseFromParent();
209 }
210 return true;
211}
212
213bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
214 MemorySSAUpdater *MSSAU) {
215 // Recursively deleting a PHI may cause multiple PHIs to be deleted
216 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
217 SmallVector<WeakTrackingVH, 8> PHIs(llvm::make_pointer_range(Range: BB->phis()));
218
219 bool Changed = false;
220 for (const auto &PHI : PHIs)
221 if (PHINode *PN = dyn_cast_or_null<PHINode>(Val: PHI.operator Value *()))
222 Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
223
224 return Changed;
225}
226
227bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
228 LoopInfo *LI, MemorySSAUpdater *MSSAU,
229 MemoryDependenceResults *MemDep,
230 bool PredecessorWithTwoSuccessors,
231 DominatorTree *DT) {
232 if (BB->hasAddressTaken())
233 return false;
234
235 // Can't merge if there are multiple predecessors, or no predecessors.
236 BasicBlock *PredBB = BB->getUniquePredecessor();
237 if (!PredBB) return false;
238
239 // Don't break self-loops.
240 if (PredBB == BB) return false;
241
242 // Don't break unwinding instructions or terminators with other side-effects.
243 Instruction *PTI = PredBB->getTerminator();
244 if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
245 return false;
246
247 // Can't merge if there are multiple distinct successors.
248 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
249 return false;
250
251 // Currently only allow PredBB to have two predecessors, one being BB.
252 // Update BI to branch to BB's only successor instead of BB.
253 BranchInst *PredBB_BI;
254 BasicBlock *NewSucc = nullptr;
255 unsigned FallThruPath;
256 if (PredecessorWithTwoSuccessors) {
257 if (!(PredBB_BI = dyn_cast<BranchInst>(Val: PTI)))
258 return false;
259 BranchInst *BB_JmpI = dyn_cast<BranchInst>(Val: BB->getTerminator());
260 if (!BB_JmpI || !BB_JmpI->isUnconditional())
261 return false;
262 NewSucc = BB_JmpI->getSuccessor(i: 0);
263 FallThruPath = PredBB_BI->getSuccessor(i: 0) == BB ? 0 : 1;
264 }
265
266 // Can't merge if there is PHI loop.
267 for (PHINode &PN : BB->phis())
268 if (llvm::is_contained(Range: PN.incoming_values(), Element: &PN))
269 return false;
270
271 // Don't break if both the basic block and the predecessor contain loop or
272 // entry convergent intrinsics, since there may only be one convergence token
273 // per block.
274 if (HasLoopOrEntryConvergenceToken(BB) &&
275 HasLoopOrEntryConvergenceToken(BB: PredBB))
276 return false;
277
278 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
279 << PredBB->getName() << "\n");
280
281 // Begin by getting rid of unneeded PHIs.
282 SmallVector<AssertingVH<Value>, 4> IncomingValues;
283 if (isa<PHINode>(Val: BB->front())) {
284 for (PHINode &PN : BB->phis())
285 if (!isa<PHINode>(Val: PN.getIncomingValue(i: 0)) ||
286 cast<PHINode>(Val: PN.getIncomingValue(i: 0))->getParent() != BB)
287 IncomingValues.push_back(Elt: PN.getIncomingValue(i: 0));
288 FoldSingleEntryPHINodes(BB, MemDep);
289 }
290
291 if (DT) {
292 assert(!DTU && "cannot use both DT and DTU for updates");
293 DomTreeNode *PredNode = DT->getNode(BB: PredBB);
294 DomTreeNode *BBNode = DT->getNode(BB);
295 if (PredNode) {
296 assert(BBNode && "PredNode unreachable but BBNode reachable?");
297 for (DomTreeNode *C : to_vector(Range: BBNode->children()))
298 C->setIDom(PredNode);
299 }
300 }
301 // DTU update: Collect all the edges that exit BB.
302 // These dominator edges will be redirected from Pred.
303 std::vector<DominatorTree::UpdateType> Updates;
304 if (DTU) {
305 assert(!DT && "cannot use both DT and DTU for updates");
306 // To avoid processing the same predecessor more than once.
307 SmallPtrSet<BasicBlock *, 8> SeenSuccs;
308 SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(llvm::from_range,
309 successors(BB: PredBB));
310 Updates.reserve(n: Updates.size() + 2 * succ_size(BB) + 1);
311 // Add insert edges first. Experimentally, for the particular case of two
312 // blocks that can be merged, with a single successor and single predecessor
313 // respectively, it is beneficial to have all insert updates first. Deleting
314 // edges first may lead to unreachable blocks, followed by inserting edges
315 // making the blocks reachable again. Such DT updates lead to high compile
316 // times. We add inserts before deletes here to reduce compile time.
317 for (BasicBlock *SuccOfBB : successors(BB))
318 // This successor of BB may already be a PredBB's successor.
319 if (!SuccsOfPredBB.contains(Ptr: SuccOfBB))
320 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
321 Updates.push_back(x: {DominatorTree::Insert, PredBB, SuccOfBB});
322 SeenSuccs.clear();
323 for (BasicBlock *SuccOfBB : successors(BB))
324 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
325 Updates.push_back(x: {DominatorTree::Delete, BB, SuccOfBB});
326 Updates.push_back(x: {DominatorTree::Delete, PredBB, BB});
327 }
328
329 Instruction *STI = BB->getTerminator();
330 Instruction *Start = &*BB->begin();
331 // If there's nothing to move, mark the starting instruction as the last
332 // instruction in the block. Terminator instruction is handled separately.
333 if (Start == STI)
334 Start = PTI;
335
336 // Move all definitions in the successor to the predecessor...
337 PredBB->splice(ToIt: PTI->getIterator(), FromBB: BB, FromBeginIt: BB->begin(), FromEndIt: STI->getIterator());
338
339 if (MSSAU)
340 MSSAU->moveAllAfterMergeBlocks(From: BB, To: PredBB, Start);
341
342 // Make all PHI nodes that referred to BB now refer to Pred as their
343 // source...
344 BB->replaceAllUsesWith(V: PredBB);
345
346 if (PredecessorWithTwoSuccessors) {
347 // Delete the unconditional branch from BB.
348 BB->back().eraseFromParent();
349
350 // Update branch in the predecessor.
351 PredBB_BI->setSuccessor(idx: FallThruPath, NewSucc);
352 } else {
353 // Delete the unconditional branch from the predecessor.
354 PredBB->back().eraseFromParent();
355
356 // Move terminator instruction.
357 BB->back().moveBeforePreserving(BB&: *PredBB, I: PredBB->end());
358
359 // Terminator may be a memory accessing instruction too.
360 if (MSSAU)
361 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
362 Val: MSSAU->getMemorySSA()->getMemoryAccess(I: PredBB->getTerminator())))
363 MSSAU->moveToPlace(What: MUD, BB: PredBB, Where: MemorySSA::End);
364 }
365 // Add unreachable to now empty BB.
366 new UnreachableInst(BB->getContext(), BB);
367
368 // Inherit predecessors name if it exists.
369 if (!PredBB->hasName())
370 PredBB->takeName(V: BB);
371
372 if (LI)
373 LI->removeBlock(BB);
374
375 if (MemDep)
376 MemDep->invalidateCachedPredecessors();
377
378 if (DTU)
379 DTU->applyUpdates(Updates);
380
381 if (DT) {
382 assert(succ_empty(BB) &&
383 "successors should have been transferred to PredBB");
384 DT->eraseNode(BB);
385 }
386
387 // Finally, erase the old block and update dominator info.
388 DeleteDeadBlock(BB, DTU);
389
390 return true;
391}
392
393bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
394 SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
395 LoopInfo *LI) {
396 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
397
398 bool BlocksHaveBeenMerged = false;
399 while (!MergeBlocks.empty()) {
400 BasicBlock *BB = *MergeBlocks.begin();
401 BasicBlock *Dest = BB->getSingleSuccessor();
402 if (Dest && (!L || L->contains(BB: Dest))) {
403 BasicBlock *Fold = Dest->getUniquePredecessor();
404 (void)Fold;
405 if (MergeBlockIntoPredecessor(BB: Dest, DTU, LI)) {
406 assert(Fold == BB &&
407 "Expecting BB to be unique predecessor of the Dest block");
408 MergeBlocks.erase(Ptr: Dest);
409 BlocksHaveBeenMerged = true;
410 } else
411 MergeBlocks.erase(Ptr: BB);
412 } else
413 MergeBlocks.erase(Ptr: BB);
414 }
415 return BlocksHaveBeenMerged;
416}
417
418/// Remove redundant instructions within sequences of consecutive dbg.value
419/// instructions. This is done using a backward scan to keep the last dbg.value
420/// describing a specific variable/fragment.
421///
422/// BackwardScan strategy:
423/// ----------------------
424/// Given a sequence of consecutive DbgValueInst like this
425///
426/// dbg.value ..., "x", FragmentX1 (*)
427/// dbg.value ..., "y", FragmentY1
428/// dbg.value ..., "x", FragmentX2
429/// dbg.value ..., "x", FragmentX1 (**)
430///
431/// then the instruction marked with (*) can be removed (it is guaranteed to be
432/// obsoleted by the instruction marked with (**) as the latter instruction is
433/// describing the same variable using the same fragment info).
434///
435/// Possible improvements:
436/// - Check fully overlapping fragments and not only identical fragments.
437static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
438 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
439 SmallDenseSet<DebugVariable> VariableSet;
440 for (auto &I : reverse(C&: *BB)) {
441 for (DbgVariableRecord &DVR :
442 reverse(C: filterDbgVars(R: I.getDbgRecordRange()))) {
443 DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
444 DVR.getDebugLoc()->getInlinedAt());
445 auto R = VariableSet.insert(V: Key);
446 // If the same variable fragment is described more than once it is enough
447 // to keep the last one (i.e. the first found since we for reverse
448 // iteration).
449 if (R.second)
450 continue;
451
452 if (DVR.isDbgAssign()) {
453 // Don't delete dbg.assign intrinsics that are linked to instructions.
454 if (!at::getAssignmentInsts(DVR: &DVR).empty())
455 continue;
456 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
457 }
458
459 ToBeRemoved.push_back(Elt: &DVR);
460 }
461 // Sequence with consecutive dbg.value instrs ended. Clear the map to
462 // restart identifying redundant instructions if case we find another
463 // dbg.value sequence.
464 VariableSet.clear();
465 }
466
467 for (auto &DVR : ToBeRemoved)
468 DVR->eraseFromParent();
469
470 return !ToBeRemoved.empty();
471}
472
473/// Remove redundant dbg.value instructions using a forward scan. This can
474/// remove a dbg.value instruction that is redundant due to indicating that a
475/// variable has the same value as already being indicated by an earlier
476/// dbg.value.
477///
478/// ForwardScan strategy:
479/// ---------------------
480/// Given two identical dbg.value instructions, separated by a block of
481/// instructions that isn't describing the same variable, like this
482///
483/// dbg.value X1, "x", FragmentX1 (**)
484/// <block of instructions, none being "dbg.value ..., "x", ...">
485/// dbg.value X1, "x", FragmentX1 (*)
486///
487/// then the instruction marked with (*) can be removed. Variable "x" is already
488/// described as being mapped to the SSA value X1.
489///
490/// Possible improvements:
491/// - Keep track of non-overlapping fragments.
492static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
493 bool RemovedAny = false;
494 SmallDenseMap<DebugVariable,
495 std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
496 VariableMap;
497 for (auto &I : *BB) {
498 for (DbgVariableRecord &DVR :
499 make_early_inc_range(Range: filterDbgVars(R: I.getDbgRecordRange()))) {
500 if (DVR.getType() == DbgVariableRecord::LocationType::Declare)
501 continue;
502 DebugVariable Key(DVR.getVariable(), std::nullopt,
503 DVR.getDebugLoc()->getInlinedAt());
504 auto [VMI, Inserted] = VariableMap.try_emplace(Key);
505 // A dbg.assign with no linked instructions can be treated like a
506 // dbg.value (i.e. can be deleted).
507 bool IsDbgValueKind =
508 (!DVR.isDbgAssign() || at::getAssignmentInsts(DVR: &DVR).empty());
509
510 // Update the map if we found a new value/expression describing the
511 // variable, or if the variable wasn't mapped already.
512 SmallVector<Value *, 4> Values(DVR.location_ops());
513 if (Inserted || VMI->second.first != Values ||
514 VMI->second.second != DVR.getExpression()) {
515 if (IsDbgValueKind)
516 VMI->second = {Values, DVR.getExpression()};
517 else
518 VMI->second = {Values, nullptr};
519 continue;
520 }
521 // Don't delete dbg.assign intrinsics that are linked to instructions.
522 if (!IsDbgValueKind)
523 continue;
524 // Found an identical mapping. Remember the instruction for later removal.
525 DVR.eraseFromParent();
526 RemovedAny = true;
527 }
528 }
529
530 return RemovedAny;
531}
532
533/// Remove redundant undef dbg.assign intrinsic from an entry block using a
534/// forward scan.
535/// Strategy:
536/// ---------------------
537/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
538/// linked to an intrinsic, and don't share an aggregate variable with a debug
539/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
540/// that come before non-undef debug intrinsics for the variable are
541/// deleted. Given:
542///
543/// dbg.assign undef, "x", FragmentX1 (*)
544/// <block of instructions, none being "dbg.value ..., "x", ...">
545/// dbg.value %V, "x", FragmentX2
546/// <block of instructions, none being "dbg.value ..., "x", ...">
547/// dbg.assign undef, "x", FragmentX1
548///
549/// then (only) the instruction marked with (*) can be removed.
550/// Possible improvements:
551/// - Keep track of non-overlapping fragments.
552static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
553 assert(BB->isEntryBlock() && "expected entry block");
554 bool RemovedAny = false;
555 DenseSet<DebugVariableAggregate> SeenDefForAggregate;
556
557 // Remove undef dbg.assign intrinsics that are encountered before
558 // any non-undef intrinsics from the entry block.
559 for (auto &I : *BB) {
560 for (DbgVariableRecord &DVR :
561 make_early_inc_range(Range: filterDbgVars(R: I.getDbgRecordRange()))) {
562 if (!DVR.isDbgValue() && !DVR.isDbgAssign())
563 continue;
564 bool IsDbgValueKind =
565 (DVR.isDbgValue() || at::getAssignmentInsts(DVR: &DVR).empty());
566
567 DebugVariableAggregate Aggregate(&DVR);
568 if (!SeenDefForAggregate.contains(V: Aggregate)) {
569 bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
570 if (!IsKill) {
571 SeenDefForAggregate.insert(V: Aggregate);
572 } else if (DVR.isDbgAssign()) {
573 DVR.eraseFromParent();
574 RemovedAny = true;
575 }
576 }
577 }
578 }
579
580 return RemovedAny;
581}
582
583bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
584 bool MadeChanges = false;
585 // By using the "backward scan" strategy before the "forward scan" strategy we
586 // can remove both dbg.value (2) and (3) in a situation like this:
587 //
588 // (1) dbg.value V1, "x", DIExpression()
589 // ...
590 // (2) dbg.value V2, "x", DIExpression()
591 // (3) dbg.value V1, "x", DIExpression()
592 //
593 // The backward scan will remove (2), it is made obsolete by (3). After
594 // getting (2) out of the way, the foward scan will remove (3) since "x"
595 // already is described as having the value V1 at (1).
596 MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
597 if (BB->isEntryBlock() &&
598 isAssignmentTrackingEnabled(M: *BB->getParent()->getParent()))
599 MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
600 MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
601
602 if (MadeChanges)
603 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
604 << BB->getName() << "\n");
605 return MadeChanges;
606}
607
608void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
609 Instruction &I = *BI;
610 // Replaces all of the uses of the instruction with uses of the value
611 I.replaceAllUsesWith(V);
612
613 // Make sure to propagate a name if there is one already.
614 if (I.hasName() && !V->hasName())
615 V->takeName(V: &I);
616
617 // Delete the unnecessary instruction now...
618 BI = BI->eraseFromParent();
619}
620
621void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
622 Instruction *I) {
623 assert(I->getParent() == nullptr &&
624 "ReplaceInstWithInst: Instruction already inserted into basic block!");
625
626 // Copy debug location to newly added instruction, if it wasn't already set
627 // by the caller.
628 if (!I->getDebugLoc())
629 I->setDebugLoc(BI->getDebugLoc());
630
631 // Insert the new instruction into the basic block...
632 BasicBlock::iterator New = I->insertInto(ParentBB: BB, It: BI);
633
634 // Replace all uses of the old instruction, and delete it.
635 ReplaceInstWithValue(BI, V: I);
636
637 // Move BI back to point to the newly inserted instruction
638 BI = New;
639}
640
641bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
642 // Remember visited blocks to avoid infinite loop
643 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
644 unsigned Depth = 0;
645 while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
646 VisitedBlocks.insert(Ptr: BB).second) {
647 if (isa<UnreachableInst>(Val: BB->getTerminator()) ||
648 BB->getTerminatingDeoptimizeCall())
649 return true;
650 BB = BB->getUniqueSuccessor();
651 }
652 return false;
653}
654
655void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
656 BasicBlock::iterator BI(From);
657 ReplaceInstWithInst(BB: From->getParent(), BI, I: To);
658}
659
660BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
661 LoopInfo *LI, MemorySSAUpdater *MSSAU,
662 const Twine &BBName) {
663 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
664
665 Instruction *LatchTerm = BB->getTerminator();
666
667 CriticalEdgeSplittingOptions Options =
668 CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
669
670 if ((isCriticalEdge(TI: LatchTerm, SuccNum, AllowIdenticalEdges: Options.MergeIdenticalEdges))) {
671 // If this is a critical edge, let SplitKnownCriticalEdge do it.
672 return SplitKnownCriticalEdge(TI: LatchTerm, SuccNum, Options, BBName);
673 }
674
675 // If the edge isn't critical, then BB has a single successor or Succ has a
676 // single pred. Split the block.
677 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
678 // If the successor only has a single pred, split the top of the successor
679 // block.
680 assert(SP == BB && "CFG broken");
681 (void)SP;
682 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
683 return splitBlockBefore(Old: Succ, SplitPt: &Succ->front(), DTU: &DTU, LI, MSSAU, BBName);
684 }
685
686 // Otherwise, if BB has a single successor, split it at the bottom of the
687 // block.
688 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
689 "Should have a single succ!");
690 return SplitBlock(Old: BB, SplitPt: BB->getTerminator(), DT, LI, MSSAU, BBName);
691}
692
693/// Helper function to update the cycle or loop information after inserting a
694/// new block between a callbr instruction and one of its target blocks. Adds
695/// the new block to the innermost cycle or loop that the callbr instruction and
696/// the original target block share.
697/// \p LCI cycle or loop information to update
698/// \p CallBrBlock block containing the callbr instruction
699/// \p CallBrTarget new target block of the callbr instruction
700/// \p Succ original target block of the callbr instruction
701template <typename TI, typename T>
702static bool updateCycleLoopInfo(TI *LCI, BasicBlock *CallBrBlock,
703 BasicBlock *CallBrTarget, BasicBlock *Succ) {
704 static_assert(std::is_same_v<TI, CycleInfo> || std::is_same_v<TI, LoopInfo>,
705 "type must be CycleInfo or LoopInfo");
706 if (!LCI)
707 return false;
708
709 T *LC;
710 if constexpr (std::is_same_v<TI, CycleInfo>)
711 LC = LCI->getSmallestCommonCycle(CallBrBlock, Succ);
712 else
713 LC = LCI->getSmallestCommonLoop(CallBrBlock, Succ);
714 if (!LC)
715 return false;
716
717 if constexpr (std::is_same_v<TI, CycleInfo>)
718 LCI->addBlockToCycle(CallBrTarget, LC);
719 else
720 LC->addBasicBlockToLoop(CallBrTarget, *LCI);
721
722 return true;
723}
724
725BasicBlock *llvm::SplitCallBrEdge(BasicBlock *CallBrBlock, BasicBlock *Succ,
726 unsigned SuccIdx, DomTreeUpdater *DTU,
727 CycleInfo *CI, LoopInfo *LI,
728 bool *UpdatedLI) {
729 CallBrInst *CallBr = dyn_cast<CallBrInst>(Val: CallBrBlock->getTerminator());
730 assert(CallBr && "expected callbr terminator");
731 assert(SuccIdx < CallBr->getNumSuccessors() &&
732 Succ == CallBr->getSuccessor(SuccIdx) && "invalid successor index");
733
734 // Create a new block between callbr and the specified successor.
735 // splitBlockBefore cannot be re-used here since it cannot split if the split
736 // point is a PHI node (because BasicBlock::splitBasicBlockBefore cannot
737 // handle that). But we don't need to rewire every part of a potential PHI
738 // node. We only care about the edge between CallBrBlock and the original
739 // successor.
740 BasicBlock *CallBrTarget =
741 BasicBlock::Create(Context&: CallBrBlock->getContext(),
742 Name: CallBrBlock->getName() + ".target." + Succ->getName(),
743 Parent: CallBrBlock->getParent());
744 // Rewire control flow from the new target block to the original successor.
745 Succ->replacePhiUsesWith(Old: CallBrBlock, New: CallBrTarget);
746 // Rewire control flow from callbr to the new target block.
747 CallBr->setSuccessor(i: SuccIdx, NewSucc: CallBrTarget);
748 // Jump from the new target block to the original successor.
749 BranchInst::Create(IfTrue: Succ, InsertBefore: CallBrTarget);
750
751 bool Updated =
752 updateCycleLoopInfo<LoopInfo, Loop>(LCI: LI, CallBrBlock, CallBrTarget, Succ);
753 if (UpdatedLI)
754 *UpdatedLI = Updated;
755 updateCycleLoopInfo<CycleInfo, Cycle>(LCI: CI, CallBrBlock, CallBrTarget, Succ);
756 if (DTU) {
757 DTU->applyUpdates(Updates: {{DominatorTree::Insert, CallBrBlock, CallBrTarget}});
758 if (DTU->getDomTree().dominates(A: CallBrBlock, B: Succ)) {
759 if (!is_contained(Range: successors(BB: CallBrBlock), Element: Succ))
760 DTU->applyUpdates(Updates: {{DominatorTree::Delete, CallBrBlock, Succ}});
761 DTU->applyUpdates(Updates: {{DominatorTree::Insert, CallBrTarget, Succ}});
762 }
763 }
764
765 return CallBrTarget;
766}
767
768void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
769 if (auto *II = dyn_cast<InvokeInst>(Val: TI))
770 II->setUnwindDest(Succ);
771 else if (auto *CS = dyn_cast<CatchSwitchInst>(Val: TI))
772 CS->setUnwindDest(Succ);
773 else if (auto *CR = dyn_cast<CleanupReturnInst>(Val: TI))
774 CR->setUnwindDest(Succ);
775 else
776 llvm_unreachable("unexpected terminator instruction");
777}
778
779void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
780 BasicBlock *NewPred, PHINode *Until) {
781 int BBIdx = 0;
782 for (PHINode &PN : DestBB->phis()) {
783 // We manually update the LandingPadReplacement PHINode and it is the last
784 // PHI Node. So, if we find it, we are done.
785 if (Until == &PN)
786 break;
787
788 // Reuse the previous value of BBIdx if it lines up. In cases where we
789 // have multiple phi nodes with *lots* of predecessors, this is a speed
790 // win because we don't have to scan the PHI looking for TIBB. This
791 // happens because the BB list of PHI nodes are usually in the same
792 // order.
793 if (PN.getIncomingBlock(i: BBIdx) != OldPred)
794 BBIdx = PN.getBasicBlockIndex(BB: OldPred);
795
796 assert(BBIdx != -1 && "Invalid PHI Index!");
797 PN.setIncomingBlock(i: BBIdx, BB: NewPred);
798 }
799}
800
801BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
802 LandingPadInst *OriginalPad,
803 PHINode *LandingPadReplacement,
804 const CriticalEdgeSplittingOptions &Options,
805 const Twine &BBName) {
806
807 auto PadInst = Succ->getFirstNonPHIIt();
808 if (!LandingPadReplacement && !PadInst->isEHPad())
809 return SplitEdge(BB, Succ, DT: Options.DT, LI: Options.LI, MSSAU: Options.MSSAU, BBName);
810
811 auto *LI = Options.LI;
812 SmallVector<BasicBlock *, 4> LoopPreds;
813 // Check if extra modifications will be required to preserve loop-simplify
814 // form after splitting. If it would require splitting blocks with IndirectBr
815 // terminators, bail out if preserving loop-simplify form is requested.
816 if (Options.PreserveLoopSimplify && LI) {
817 if (Loop *BBLoop = LI->getLoopFor(BB)) {
818
819 // The only way that we can break LoopSimplify form by splitting a
820 // critical edge is when there exists some edge from BBLoop to Succ *and*
821 // the only edge into Succ from outside of BBLoop is that of NewBB after
822 // the split. If the first isn't true, then LoopSimplify still holds,
823 // NewBB is the new exit block and it has no non-loop predecessors. If the
824 // second isn't true, then Succ was not in LoopSimplify form prior to
825 // the split as it had a non-loop predecessor. In both of these cases,
826 // the predecessor must be directly in BBLoop, not in a subloop, or again
827 // LoopSimplify doesn't hold.
828 for (BasicBlock *P : predecessors(BB: Succ)) {
829 if (P == BB)
830 continue; // The new block is known.
831 if (LI->getLoopFor(BB: P) != BBLoop) {
832 // Loop is not in LoopSimplify form, no need to re simplify after
833 // splitting edge.
834 LoopPreds.clear();
835 break;
836 }
837 LoopPreds.push_back(Elt: P);
838 }
839 // Loop-simplify form can be preserved, if we can split all in-loop
840 // predecessors.
841 if (any_of(Range&: LoopPreds, P: [](BasicBlock *Pred) {
842 return isa<IndirectBrInst>(Val: Pred->getTerminator());
843 })) {
844 return nullptr;
845 }
846 }
847 }
848
849 auto *NewBB =
850 BasicBlock::Create(Context&: BB->getContext(), Name: BBName, Parent: BB->getParent(), InsertBefore: Succ);
851 setUnwindEdgeTo(TI: BB->getTerminator(), Succ: NewBB);
852 updatePhiNodes(DestBB: Succ, OldPred: BB, NewPred: NewBB, Until: LandingPadReplacement);
853
854 if (LandingPadReplacement) {
855 auto *NewLP = OriginalPad->clone();
856 auto *Terminator = BranchInst::Create(IfTrue: Succ, InsertBefore: NewBB);
857 NewLP->insertBefore(InsertPos: Terminator->getIterator());
858 LandingPadReplacement->addIncoming(V: NewLP, BB: NewBB);
859 } else {
860 Value *ParentPad = nullptr;
861 if (auto *FuncletPad = dyn_cast<FuncletPadInst>(Val&: PadInst))
862 ParentPad = FuncletPad->getParentPad();
863 else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: PadInst))
864 ParentPad = CatchSwitch->getParentPad();
865 else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(Val&: PadInst))
866 ParentPad = CleanupPad->getParentPad();
867 else if (auto *LandingPad = dyn_cast<LandingPadInst>(Val&: PadInst))
868 ParentPad = LandingPad->getParent();
869 else
870 llvm_unreachable("handling for other EHPads not implemented yet");
871
872 auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, Args: {}, NameStr: BBName, InsertBefore: NewBB);
873 CleanupReturnInst::Create(CleanupPad: NewCleanupPad, UnwindBB: Succ, InsertBefore: NewBB);
874 }
875
876 auto *DT = Options.DT;
877 auto *MSSAU = Options.MSSAU;
878 if (!DT && !LI)
879 return NewBB;
880
881 if (DT) {
882 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
883 SmallVector<DominatorTree::UpdateType, 3> Updates;
884
885 Updates.push_back(Elt: {DominatorTree::Insert, BB, NewBB});
886 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, Succ});
887 Updates.push_back(Elt: {DominatorTree::Delete, BB, Succ});
888
889 DTU.applyUpdates(Updates);
890 DTU.flush();
891
892 if (MSSAU) {
893 MSSAU->applyUpdates(Updates, DT&: *DT);
894 if (VerifyMemorySSA)
895 MSSAU->getMemorySSA()->verifyMemorySSA();
896 }
897 }
898
899 if (LI) {
900 if (Loop *BBLoop = LI->getLoopFor(BB)) {
901 // If one or the other blocks were not in a loop, the new block is not
902 // either, and thus LI doesn't need to be updated.
903 if (Loop *SuccLoop = LI->getLoopFor(BB: Succ)) {
904 if (BBLoop == SuccLoop) {
905 // Both in the same loop, the NewBB joins loop.
906 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
907 } else if (BBLoop->contains(L: SuccLoop)) {
908 // Edge from an outer loop to an inner loop. Add to the outer loop.
909 BBLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
910 } else if (SuccLoop->contains(L: BBLoop)) {
911 // Edge from an inner loop to an outer loop. Add to the outer loop.
912 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
913 } else {
914 // Edge from two loops with no containment relation. Because these
915 // are natural loops, we know that the destination block must be the
916 // header of its loop (adding a branch into a loop elsewhere would
917 // create an irreducible loop).
918 assert(SuccLoop->getHeader() == Succ &&
919 "Should not create irreducible loops!");
920 if (Loop *P = SuccLoop->getParentLoop())
921 P->addBasicBlockToLoop(NewBB, LI&: *LI);
922 }
923 }
924
925 // If BB is in a loop and Succ is outside of that loop, we may need to
926 // update LoopSimplify form and LCSSA form.
927 if (!BBLoop->contains(BB: Succ)) {
928 assert(!BBLoop->contains(NewBB) &&
929 "Split point for loop exit is contained in loop!");
930
931 // Update LCSSA form in the newly created exit block.
932 if (Options.PreserveLCSSA) {
933 createPHIsForSplitLoopExit(Preds: BB, SplitBB: NewBB, DestBB: Succ);
934 }
935
936 if (!LoopPreds.empty()) {
937 BasicBlock *NewExitBB = SplitBlockPredecessors(
938 BB: Succ, Preds: LoopPreds, Suffix: "split", DT, LI, MSSAU, PreserveLCSSA: Options.PreserveLCSSA);
939 if (Options.PreserveLCSSA)
940 createPHIsForSplitLoopExit(Preds: LoopPreds, SplitBB: NewExitBB, DestBB: Succ);
941 }
942 }
943 }
944 }
945
946 return NewBB;
947}
948
949void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
950 BasicBlock *SplitBB, BasicBlock *DestBB) {
951 // SplitBB shouldn't have anything non-trivial in it yet.
952 assert((&*SplitBB->getFirstNonPHIIt() == SplitBB->getTerminator() ||
953 SplitBB->isLandingPad()) &&
954 "SplitBB has non-PHI nodes!");
955
956 // For each PHI in the destination block.
957 for (PHINode &PN : DestBB->phis()) {
958 int Idx = PN.getBasicBlockIndex(BB: SplitBB);
959 assert(Idx >= 0 && "Invalid Block Index");
960 Value *V = PN.getIncomingValue(i: Idx);
961
962 // If the input is a PHI which already satisfies LCSSA, don't create
963 // a new one.
964 if (const PHINode *VP = dyn_cast<PHINode>(Val: V))
965 if (VP->getParent() == SplitBB)
966 continue;
967
968 // Otherwise a new PHI is needed. Create one and populate it.
969 PHINode *NewPN = PHINode::Create(Ty: PN.getType(), NumReservedValues: Preds.size(), NameStr: "split");
970 BasicBlock::iterator InsertPos =
971 SplitBB->isLandingPad() ? SplitBB->begin()
972 : SplitBB->getTerminator()->getIterator();
973 NewPN->insertBefore(InsertPos);
974 for (BasicBlock *BB : Preds)
975 NewPN->addIncoming(V, BB);
976
977 // Update the original PHI.
978 PN.setIncomingValue(i: Idx, V: NewPN);
979 }
980}
981
982unsigned
983llvm::SplitAllCriticalEdges(Function &F,
984 const CriticalEdgeSplittingOptions &Options) {
985 unsigned NumBroken = 0;
986 for (BasicBlock &BB : F) {
987 Instruction *TI = BB.getTerminator();
988 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(Val: TI))
989 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
990 if (SplitCriticalEdge(TI, SuccNum: i, Options))
991 ++NumBroken;
992 }
993 return NumBroken;
994}
995
996static BasicBlock *SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt,
997 DomTreeUpdater *DTU, DominatorTree *DT,
998 LoopInfo *LI, MemorySSAUpdater *MSSAU,
999 const Twine &BBName) {
1000 BasicBlock::iterator SplitIt = SplitPt;
1001 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad()) {
1002 ++SplitIt;
1003 assert(SplitIt != SplitPt->getParent()->end());
1004 }
1005 std::string Name = BBName.str();
1006 BasicBlock *New = Old->splitBasicBlock(
1007 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name);
1008
1009 // The new block lives in whichever loop the old one did. This preserves
1010 // LCSSA as well, because we force the split point to be after any PHI nodes.
1011 if (LI)
1012 if (Loop *L = LI->getLoopFor(BB: Old))
1013 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
1014
1015 if (DTU) {
1016 SmallVector<DominatorTree::UpdateType, 8> Updates;
1017 // Old dominates New. New node dominates all other nodes dominated by Old.
1018 SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
1019 Updates.push_back(Elt: {DominatorTree::Insert, Old, New});
1020 Updates.reserve(N: Updates.size() + 2 * succ_size(BB: New));
1021 for (BasicBlock *SuccessorOfOld : successors(BB: New))
1022 if (UniqueSuccessorsOfOld.insert(Ptr: SuccessorOfOld).second) {
1023 Updates.push_back(Elt: {DominatorTree::Insert, New, SuccessorOfOld});
1024 Updates.push_back(Elt: {DominatorTree::Delete, Old, SuccessorOfOld});
1025 }
1026
1027 DTU->applyUpdates(Updates);
1028 } else if (DT)
1029 // Old dominates New. New node dominates all other nodes dominated by Old.
1030 if (DomTreeNode *OldNode = DT->getNode(BB: Old)) {
1031 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1032
1033 DomTreeNode *NewNode = DT->addNewBlock(BB: New, DomBB: Old);
1034 for (DomTreeNode *I : Children)
1035 DT->changeImmediateDominator(N: I, NewIDom: NewNode);
1036 }
1037
1038 // Move MemoryAccesses still tracked in Old, but part of New now.
1039 // Update accesses in successor blocks accordingly.
1040 if (MSSAU)
1041 MSSAU->moveAllAfterSpliceBlocks(From: Old, To: New, Start: &*(New->begin()));
1042
1043 return New;
1044}
1045
1046BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1047 DominatorTree *DT, LoopInfo *LI,
1048 MemorySSAUpdater *MSSAU, const Twine &BBName) {
1049 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName);
1050}
1051BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1052 DomTreeUpdater *DTU, LoopInfo *LI,
1053 MemorySSAUpdater *MSSAU, const Twine &BBName) {
1054 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName);
1055}
1056
1057/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
1058/// Invalidates DFS Numbering when DTU or DT is provided.
1059static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
1060 ArrayRef<BasicBlock *> Preds,
1061 DomTreeUpdater *DTU, DominatorTree *DT,
1062 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1063 bool PreserveLCSSA, bool &HasLoopExit) {
1064 // Update dominator tree if available.
1065 if (DTU) {
1066 // Recalculation of DomTree is needed when updating a forward DomTree and
1067 // the Entry BB is replaced.
1068 if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1069 // The entry block was removed and there is no external interface for
1070 // the dominator tree to be notified of this change. In this corner-case
1071 // we recalculate the entire tree.
1072 DTU->recalculate(F&: *NewBB->getParent());
1073 } else {
1074 // Split block expects NewBB to have a non-empty set of predecessors.
1075 SmallVector<DominatorTree::UpdateType, 8> Updates;
1076 SmallPtrSet<BasicBlock *, 8> UniquePreds;
1077 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, OldBB});
1078 Updates.reserve(N: Updates.size() + 2 * Preds.size());
1079 for (auto *Pred : Preds)
1080 if (UniquePreds.insert(Ptr: Pred).second) {
1081 Updates.push_back(Elt: {DominatorTree::Insert, Pred, NewBB});
1082 Updates.push_back(Elt: {DominatorTree::Delete, Pred, OldBB});
1083 }
1084 DTU->applyUpdates(Updates);
1085 }
1086 } else if (DT) {
1087 if (OldBB == DT->getRootNode()->getBlock()) {
1088 assert(NewBB->isEntryBlock());
1089 DT->setNewRoot(NewBB);
1090 } else {
1091 // Split block expects NewBB to have a non-empty set of predecessors.
1092 DT->splitBlock(NewBB);
1093 }
1094 }
1095
1096 // Update MemoryPhis after split if MemorySSA is available
1097 if (MSSAU)
1098 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(Old: OldBB, New: NewBB, Preds);
1099
1100 // The rest of the logic is only relevant for updating the loop structures.
1101 if (!LI)
1102 return;
1103
1104 if (DTU && DTU->hasDomTree())
1105 DT = &DTU->getDomTree();
1106 assert(DT && "DT should be available to update LoopInfo!");
1107 Loop *L = LI->getLoopFor(BB: OldBB);
1108
1109 // If we need to preserve loop analyses, collect some information about how
1110 // this split will affect loops.
1111 bool IsLoopEntry = !!L;
1112 bool SplitMakesNewLoopHeader = false;
1113 for (BasicBlock *Pred : Preds) {
1114 // Preds that are not reachable from entry should not be used to identify if
1115 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1116 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1117 // as true and make the NewBB the header of some loop. This breaks LI.
1118 if (!DT->isReachableFromEntry(A: Pred))
1119 continue;
1120 // If we need to preserve LCSSA, determine if any of the preds is a loop
1121 // exit.
1122 if (PreserveLCSSA)
1123 if (Loop *PL = LI->getLoopFor(BB: Pred))
1124 if (!PL->contains(BB: OldBB))
1125 HasLoopExit = true;
1126
1127 // If we need to preserve LoopInfo, note whether any of the preds crosses
1128 // an interesting loop boundary.
1129 if (!L)
1130 continue;
1131 if (L->contains(BB: Pred))
1132 IsLoopEntry = false;
1133 else
1134 SplitMakesNewLoopHeader = true;
1135 }
1136
1137 // Unless we have a loop for OldBB, nothing else to do here.
1138 if (!L)
1139 return;
1140
1141 if (IsLoopEntry) {
1142 // Add the new block to the nearest enclosing loop (and not an adjacent
1143 // loop). To find this, examine each of the predecessors and determine which
1144 // loops enclose them, and select the most-nested loop which contains the
1145 // loop containing the block being split.
1146 Loop *InnermostPredLoop = nullptr;
1147 for (BasicBlock *Pred : Preds) {
1148 if (Loop *PredLoop = LI->getLoopFor(BB: Pred)) {
1149 // Seek a loop which actually contains the block being split (to avoid
1150 // adjacent loops).
1151 while (PredLoop && !PredLoop->contains(BB: OldBB))
1152 PredLoop = PredLoop->getParentLoop();
1153
1154 // Select the most-nested of these loops which contains the block.
1155 if (PredLoop && PredLoop->contains(BB: OldBB) &&
1156 (!InnermostPredLoop ||
1157 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1158 InnermostPredLoop = PredLoop;
1159 }
1160 }
1161
1162 if (InnermostPredLoop)
1163 InnermostPredLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1164 } else {
1165 L->addBasicBlockToLoop(NewBB, LI&: *LI);
1166 if (SplitMakesNewLoopHeader)
1167 L->moveToHeader(BB: NewBB);
1168 }
1169}
1170
1171BasicBlock *llvm::splitBlockBefore(BasicBlock *Old,
1172 BasicBlock::iterator SplitPt,
1173 DomTreeUpdater *DTU, LoopInfo *LI,
1174 MemorySSAUpdater *MSSAU,
1175 const Twine &BBName) {
1176 BasicBlock::iterator SplitIt = SplitPt;
1177 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad())
1178 ++SplitIt;
1179 SmallVector<BasicBlock *, 4> Preds(predecessors(BB: Old));
1180 BasicBlock *New = Old->splitBasicBlockBefore(
1181 I: SplitIt, BBName: BBName.isTriviallyEmpty() ? Old->getName() + ".split" : BBName);
1182
1183 bool HasLoopExit = false;
1184 UpdateAnalysisInformation(OldBB: Old, NewBB: New, Preds, DTU, DT: nullptr, LI, MSSAU, PreserveLCSSA: false,
1185 HasLoopExit);
1186
1187 return New;
1188}
1189
1190/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1191/// This also updates AliasAnalysis, if available.
1192static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1193 ArrayRef<BasicBlock *> Preds, BranchInst *BI,
1194 bool HasLoopExit) {
1195 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1196 SmallPtrSet<BasicBlock *, 16> PredSet(llvm::from_range, Preds);
1197 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(Val: I); ) {
1198 PHINode *PN = cast<PHINode>(Val: I++);
1199
1200 // Check to see if all of the values coming in are the same. If so, we
1201 // don't need to create a new PHI node, unless it's needed for LCSSA.
1202 Value *InVal = nullptr;
1203 if (!HasLoopExit) {
1204 InVal = PN->getIncomingValueForBlock(BB: Preds[0]);
1205 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1206 if (!PredSet.count(Ptr: PN->getIncomingBlock(i)))
1207 continue;
1208 if (!InVal)
1209 InVal = PN->getIncomingValue(i);
1210 else if (InVal != PN->getIncomingValue(i)) {
1211 InVal = nullptr;
1212 break;
1213 }
1214 }
1215 }
1216
1217 if (InVal) {
1218 // If all incoming values for the new PHI would be the same, just don't
1219 // make a new PHI. Instead, just remove the incoming values from the old
1220 // PHI.
1221 PN->removeIncomingValueIf(
1222 Predicate: [&](unsigned Idx) {
1223 return PredSet.contains(Ptr: PN->getIncomingBlock(i: Idx));
1224 },
1225 /* DeletePHIIfEmpty */ false);
1226
1227 // Add an incoming value to the PHI node in the loop for the preheader
1228 // edge.
1229 PN->addIncoming(V: InVal, BB: NewBB);
1230 continue;
1231 }
1232
1233 // If the values coming into the block are not the same, we need a new
1234 // PHI.
1235 // Create the new PHI node, insert it into NewBB at the end of the block
1236 PHINode *NewPHI =
1237 PHINode::Create(Ty: PN->getType(), NumReservedValues: Preds.size(), NameStr: PN->getName() + ".ph", InsertBefore: BI->getIterator());
1238
1239 // NOTE! This loop walks backwards for a reason! First off, this minimizes
1240 // the cost of removal if we end up removing a large number of values, and
1241 // second off, this ensures that the indices for the incoming values aren't
1242 // invalidated when we remove one.
1243 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1244 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1245 if (PredSet.count(Ptr: IncomingBB)) {
1246 Value *V = PN->removeIncomingValue(Idx: i, DeletePHIIfEmpty: false);
1247 NewPHI->addIncoming(V, BB: IncomingBB);
1248 }
1249 }
1250
1251 PN->addIncoming(V: NewPHI, BB: NewBB);
1252 }
1253}
1254
1255static void SplitLandingPadPredecessorsImpl(
1256 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1257 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1258 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1259 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1260
1261static BasicBlock *
1262SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
1263 const char *Suffix, DomTreeUpdater *DTU,
1264 DominatorTree *DT, LoopInfo *LI,
1265 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1266 // Do not attempt to split that which cannot be split.
1267 if (!BB->canSplitPredecessors())
1268 return nullptr;
1269
1270 // For the landingpads we need to act a bit differently.
1271 // Delegate this work to the SplitLandingPadPredecessors.
1272 if (BB->isLandingPad()) {
1273 SmallVector<BasicBlock*, 2> NewBBs;
1274 std::string NewName = std::string(Suffix) + ".split-lp";
1275
1276 SplitLandingPadPredecessorsImpl(OrigBB: BB, Preds, Suffix1: Suffix, Suffix2: NewName.c_str(), NewBBs,
1277 DTU, DT, LI, MSSAU, PreserveLCSSA);
1278 return NewBBs[0];
1279 }
1280
1281 // Create new basic block, insert right before the original block.
1282 BasicBlock *NewBB = BasicBlock::Create(
1283 Context&: BB->getContext(), Name: BB->getName() + Suffix, Parent: BB->getParent(), InsertBefore: BB);
1284
1285 // The new block unconditionally branches to the old block.
1286 BranchInst *BI = BranchInst::Create(IfTrue: BB, InsertBefore: NewBB);
1287
1288 Loop *L = nullptr;
1289 BasicBlock *OldLatch = nullptr;
1290 // Splitting the predecessors of a loop header creates a preheader block.
1291 if (LI && LI->isLoopHeader(BB)) {
1292 L = LI->getLoopFor(BB);
1293 // Using the loop start line number prevents debuggers stepping into the
1294 // loop body for this instruction.
1295 BI->setDebugLoc(L->getStartLoc());
1296
1297 // If BB is the header of the Loop, it is possible that the loop is
1298 // modified, such that the current latch does not remain the latch of the
1299 // loop. If that is the case, the loop metadata from the current latch needs
1300 // to be applied to the new latch.
1301 OldLatch = L->getLoopLatch();
1302 } else
1303 BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
1304
1305 // Move the edges from Preds to point to NewBB instead of BB.
1306 for (BasicBlock *Pred : Preds) {
1307 // This is slightly more strict than necessary; the minimum requirement
1308 // is that there be no more than one indirectbr branching to BB. And
1309 // all BlockAddress uses would need to be updated.
1310 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1311 "Cannot split an edge from an IndirectBrInst");
1312 Pred->getTerminator()->replaceSuccessorWith(OldBB: BB, NewBB);
1313 }
1314
1315 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1316 // node becomes an incoming value for BB's phi node. However, if the Preds
1317 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1318 // account for the newly created predecessor.
1319 if (Preds.empty()) {
1320 // Insert dummy values as the incoming value.
1321 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(Val: I); ++I)
1322 cast<PHINode>(Val&: I)->addIncoming(V: PoisonValue::get(T: I->getType()), BB: NewBB);
1323 }
1324
1325 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1326 bool HasLoopExit = false;
1327 UpdateAnalysisInformation(OldBB: BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1328 HasLoopExit);
1329
1330 if (!Preds.empty()) {
1331 // Update the PHI nodes in BB with the values coming from NewBB.
1332 UpdatePHINodes(OrigBB: BB, NewBB, Preds, BI, HasLoopExit);
1333 }
1334
1335 if (OldLatch) {
1336 BasicBlock *NewLatch = L->getLoopLatch();
1337 if (NewLatch != OldLatch) {
1338 MDNode *MD = OldLatch->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
1339 NewLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: MD);
1340 // It's still possible that OldLatch is the latch of another inner loop,
1341 // in which case we do not remove the metadata.
1342 Loop *IL = LI->getLoopFor(BB: OldLatch);
1343 if (IL && IL->getLoopLatch() != OldLatch)
1344 OldLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: nullptr);
1345 }
1346 }
1347
1348 return NewBB;
1349}
1350
1351BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1352 ArrayRef<BasicBlock *> Preds,
1353 const char *Suffix, DominatorTree *DT,
1354 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1355 bool PreserveLCSSA) {
1356 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1357 MSSAU, PreserveLCSSA);
1358}
1359BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1360 ArrayRef<BasicBlock *> Preds,
1361 const char *Suffix,
1362 DomTreeUpdater *DTU, LoopInfo *LI,
1363 MemorySSAUpdater *MSSAU,
1364 bool PreserveLCSSA) {
1365 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1366 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1367}
1368
1369static void SplitLandingPadPredecessorsImpl(
1370 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1371 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1372 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1373 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1374 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1375
1376 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1377 // it right before the original block.
1378 BasicBlock *NewBB1 = BasicBlock::Create(Context&: OrigBB->getContext(),
1379 Name: OrigBB->getName() + Suffix1,
1380 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1381 NewBBs.push_back(Elt: NewBB1);
1382
1383 // The new block unconditionally branches to the old block.
1384 BranchInst *BI1 = BranchInst::Create(IfTrue: OrigBB, InsertBefore: NewBB1);
1385 BI1->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1386
1387 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1388 for (BasicBlock *Pred : Preds) {
1389 // This is slightly more strict than necessary; the minimum requirement
1390 // is that there be no more than one indirectbr branching to BB. And
1391 // all BlockAddress uses would need to be updated.
1392 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1393 "Cannot split an edge from an IndirectBrInst");
1394 Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB1);
1395 }
1396
1397 bool HasLoopExit = false;
1398 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB1, Preds, DTU, DT, LI, MSSAU,
1399 PreserveLCSSA, HasLoopExit);
1400
1401 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1402 UpdatePHINodes(OrigBB, NewBB: NewBB1, Preds, BI: BI1, HasLoopExit);
1403
1404 // Move the remaining edges from OrigBB to point to NewBB2.
1405 SmallVector<BasicBlock*, 8> NewBB2Preds;
1406 for (pred_iterator i = pred_begin(BB: OrigBB), e = pred_end(BB: OrigBB);
1407 i != e; ) {
1408 BasicBlock *Pred = *i++;
1409 if (Pred == NewBB1) continue;
1410 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1411 "Cannot split an edge from an IndirectBrInst");
1412 NewBB2Preds.push_back(Elt: Pred);
1413 e = pred_end(BB: OrigBB);
1414 }
1415
1416 BasicBlock *NewBB2 = nullptr;
1417 if (!NewBB2Preds.empty()) {
1418 // Create another basic block for the rest of OrigBB's predecessors.
1419 NewBB2 = BasicBlock::Create(Context&: OrigBB->getContext(),
1420 Name: OrigBB->getName() + Suffix2,
1421 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1422 NewBBs.push_back(Elt: NewBB2);
1423
1424 // The new block unconditionally branches to the old block.
1425 BranchInst *BI2 = BranchInst::Create(IfTrue: OrigBB, InsertBefore: NewBB2);
1426 BI2->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1427
1428 // Move the remaining edges from OrigBB to point to NewBB2.
1429 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1430 NewBB2Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB2);
1431
1432 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1433 HasLoopExit = false;
1434 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, DTU, DT, LI, MSSAU,
1435 PreserveLCSSA, HasLoopExit);
1436
1437 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1438 UpdatePHINodes(OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, BI: BI2, HasLoopExit);
1439 }
1440
1441 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1442 Instruction *Clone1 = LPad->clone();
1443 Clone1->setName(Twine("lpad") + Suffix1);
1444 Clone1->insertInto(ParentBB: NewBB1, It: NewBB1->getFirstInsertionPt());
1445
1446 if (NewBB2) {
1447 Instruction *Clone2 = LPad->clone();
1448 Clone2->setName(Twine("lpad") + Suffix2);
1449 Clone2->insertInto(ParentBB: NewBB2, It: NewBB2->getFirstInsertionPt());
1450
1451 // Create a PHI node for the two cloned landingpad instructions only
1452 // if the original landingpad instruction has some uses.
1453 if (!LPad->use_empty()) {
1454 assert(!LPad->getType()->isTokenTy() &&
1455 "Split cannot be applied if LPad is token type. Otherwise an "
1456 "invalid PHINode of token type would be created.");
1457 PHINode *PN = PHINode::Create(Ty: LPad->getType(), NumReservedValues: 2, NameStr: "lpad.phi", InsertBefore: LPad->getIterator());
1458 PN->addIncoming(V: Clone1, BB: NewBB1);
1459 PN->addIncoming(V: Clone2, BB: NewBB2);
1460 LPad->replaceAllUsesWith(V: PN);
1461 }
1462 LPad->eraseFromParent();
1463 } else {
1464 // There is no second clone. Just replace the landing pad with the first
1465 // clone.
1466 LPad->replaceAllUsesWith(V: Clone1);
1467 LPad->eraseFromParent();
1468 }
1469}
1470
1471void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1472 ArrayRef<BasicBlock *> Preds,
1473 const char *Suffix1, const char *Suffix2,
1474 SmallVectorImpl<BasicBlock *> &NewBBs,
1475 DomTreeUpdater *DTU, LoopInfo *LI,
1476 MemorySSAUpdater *MSSAU,
1477 bool PreserveLCSSA) {
1478 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1479 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1480 PreserveLCSSA);
1481}
1482
1483ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1484 BasicBlock *Pred,
1485 DomTreeUpdater *DTU) {
1486 Instruction *UncondBranch = Pred->getTerminator();
1487 // Clone the return and add it to the end of the predecessor.
1488 Instruction *NewRet = RI->clone();
1489 NewRet->insertInto(ParentBB: Pred, It: Pred->end());
1490
1491 // If the return instruction returns a value, and if the value was a
1492 // PHI node in "BB", propagate the right value into the return.
1493 for (Use &Op : NewRet->operands()) {
1494 Value *V = Op;
1495 Instruction *NewBC = nullptr;
1496 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Val: V)) {
1497 // Return value might be bitcasted. Clone and insert it before the
1498 // return instruction.
1499 V = BCI->getOperand(i_nocapture: 0);
1500 NewBC = BCI->clone();
1501 NewBC->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1502 Op = NewBC;
1503 }
1504
1505 Instruction *NewEV = nullptr;
1506 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: V)) {
1507 V = EVI->getOperand(i_nocapture: 0);
1508 NewEV = EVI->clone();
1509 if (NewBC) {
1510 NewBC->setOperand(i: 0, Val: NewEV);
1511 NewEV->insertInto(ParentBB: Pred, It: NewBC->getIterator());
1512 } else {
1513 NewEV->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1514 Op = NewEV;
1515 }
1516 }
1517
1518 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
1519 if (PN->getParent() == BB) {
1520 if (NewEV) {
1521 NewEV->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1522 } else if (NewBC)
1523 NewBC->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1524 else
1525 Op = PN->getIncomingValueForBlock(BB: Pred);
1526 }
1527 }
1528 }
1529
1530 // Update any PHI nodes in the returning block to realize that we no
1531 // longer branch to them.
1532 BB->removePredecessor(Pred);
1533 UncondBranch->eraseFromParent();
1534
1535 if (DTU)
1536 DTU->applyUpdates(Updates: {{DominatorTree::Delete, Pred, BB}});
1537
1538 return cast<ReturnInst>(Val: NewRet);
1539}
1540
1541Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1542 BasicBlock::iterator SplitBefore,
1543 bool Unreachable,
1544 MDNode *BranchWeights,
1545 DomTreeUpdater *DTU, LoopInfo *LI,
1546 BasicBlock *ThenBlock) {
1547 SplitBlockAndInsertIfThenElse(
1548 Cond, SplitBefore, ThenBlock: &ThenBlock, /* ElseBlock */ nullptr,
1549 /* UnreachableThen */ Unreachable,
1550 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1551 return ThenBlock->getTerminator();
1552}
1553
1554Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
1555 BasicBlock::iterator SplitBefore,
1556 bool Unreachable,
1557 MDNode *BranchWeights,
1558 DomTreeUpdater *DTU, LoopInfo *LI,
1559 BasicBlock *ElseBlock) {
1560 SplitBlockAndInsertIfThenElse(
1561 Cond, SplitBefore, /* ThenBlock */ nullptr, ElseBlock: &ElseBlock,
1562 /* UnreachableThen */ false,
1563 /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1564 return ElseBlock->getTerminator();
1565}
1566
1567void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore,
1568 Instruction **ThenTerm,
1569 Instruction **ElseTerm,
1570 MDNode *BranchWeights,
1571 DomTreeUpdater *DTU, LoopInfo *LI) {
1572 BasicBlock *ThenBlock = nullptr;
1573 BasicBlock *ElseBlock = nullptr;
1574 SplitBlockAndInsertIfThenElse(
1575 Cond, SplitBefore, ThenBlock: &ThenBlock, ElseBlock: &ElseBlock, /* UnreachableThen */ false,
1576 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1577
1578 *ThenTerm = ThenBlock->getTerminator();
1579 *ElseTerm = ElseBlock->getTerminator();
1580}
1581
1582void llvm::SplitBlockAndInsertIfThenElse(
1583 Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
1584 BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1585 MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1586 assert((ThenBlock || ElseBlock) &&
1587 "At least one branch block must be created");
1588 assert((!UnreachableThen || !UnreachableElse) &&
1589 "Split block tail must be reachable");
1590
1591 SmallVector<DominatorTree::UpdateType, 8> Updates;
1592 SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1593 BasicBlock *Head = SplitBefore->getParent();
1594 if (DTU) {
1595 UniqueOrigSuccessors.insert_range(R: successors(BB: Head));
1596 Updates.reserve(N: 4 + 2 * UniqueOrigSuccessors.size());
1597 }
1598
1599 LLVMContext &C = Head->getContext();
1600 BasicBlock *Tail = Head->splitBasicBlock(I: SplitBefore);
1601 BasicBlock *TrueBlock = Tail;
1602 BasicBlock *FalseBlock = Tail;
1603 bool ThenToTailEdge = false;
1604 bool ElseToTailEdge = false;
1605
1606 // Encapsulate the logic around creation/insertion/etc of a new block.
1607 auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1608 bool &ToTailEdge) {
1609 if (PBB == nullptr)
1610 return; // Do not create/insert a block.
1611
1612 if (*PBB)
1613 BB = *PBB; // Caller supplied block, use it.
1614 else {
1615 // Create a new block.
1616 BB = BasicBlock::Create(Context&: C, Name: "", Parent: Head->getParent(), InsertBefore: Tail);
1617 if (Unreachable)
1618 (void)new UnreachableInst(C, BB);
1619 else {
1620 (void)BranchInst::Create(IfTrue: Tail, InsertBefore: BB);
1621 ToTailEdge = true;
1622 }
1623 BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1624 // Pass the new block back to the caller.
1625 *PBB = BB;
1626 }
1627 };
1628
1629 handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1630 handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1631
1632 Instruction *HeadOldTerm = Head->getTerminator();
1633 BranchInst *HeadNewTerm =
1634 BranchInst::Create(/*ifTrue*/ IfTrue: TrueBlock, /*ifFalse*/ IfFalse: FalseBlock, Cond);
1635 HeadNewTerm->setMetadata(KindID: LLVMContext::MD_prof, Node: BranchWeights);
1636 ReplaceInstWithInst(From: HeadOldTerm, To: HeadNewTerm);
1637
1638 if (DTU) {
1639 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: TrueBlock);
1640 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: FalseBlock);
1641 if (ThenToTailEdge)
1642 Updates.emplace_back(Args: DominatorTree::Insert, Args&: TrueBlock, Args&: Tail);
1643 if (ElseToTailEdge)
1644 Updates.emplace_back(Args: DominatorTree::Insert, Args&: FalseBlock, Args&: Tail);
1645 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1646 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Tail, Args&: UniqueOrigSuccessor);
1647 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1648 Updates.emplace_back(Args: DominatorTree::Delete, Args&: Head, Args&: UniqueOrigSuccessor);
1649 DTU->applyUpdates(Updates);
1650 }
1651
1652 if (LI) {
1653 if (Loop *L = LI->getLoopFor(BB: Head); L) {
1654 if (ThenToTailEdge)
1655 L->addBasicBlockToLoop(NewBB: TrueBlock, LI&: *LI);
1656 if (ElseToTailEdge)
1657 L->addBasicBlockToLoop(NewBB: FalseBlock, LI&: *LI);
1658 L->addBasicBlockToLoop(NewBB: Tail, LI&: *LI);
1659 }
1660 }
1661}
1662
1663std::pair<Instruction *, Value *>
1664llvm::SplitBlockAndInsertSimpleForLoop(Value *End,
1665 BasicBlock::iterator SplitBefore) {
1666 BasicBlock *LoopPred = SplitBefore->getParent();
1667 BasicBlock *LoopBody = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1668 BasicBlock *LoopExit = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1669
1670 auto *Ty = End->getType();
1671 auto &DL = SplitBefore->getDataLayout();
1672 const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1673
1674 IRBuilder<> Builder(LoopBody->getTerminator());
1675 auto *IV = Builder.CreatePHI(Ty, NumReservedValues: 2, Name: "iv");
1676 auto *IVNext =
1677 Builder.CreateAdd(LHS: IV, RHS: ConstantInt::get(Ty, V: 1), Name: IV->getName() + ".next",
1678 /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1679 auto *IVCheck = Builder.CreateICmpEQ(LHS: IVNext, RHS: End,
1680 Name: IV->getName() + ".check");
1681 Builder.CreateCondBr(Cond: IVCheck, True: LoopExit, False: LoopBody);
1682 LoopBody->getTerminator()->eraseFromParent();
1683
1684 // Populate the IV PHI.
1685 IV->addIncoming(V: ConstantInt::get(Ty, V: 0), BB: LoopPred);
1686 IV->addIncoming(V: IVNext, BB: LoopBody);
1687
1688 return std::make_pair(x: &*LoopBody->getFirstNonPHIIt(), y&: IV);
1689}
1690
1691void llvm::SplitBlockAndInsertForEachLane(
1692 ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
1693 std::function<void(IRBuilderBase &, Value *)> Func) {
1694
1695 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1696
1697 if (EC.isScalable()) {
1698 Value *NumElements = IRB.CreateElementCount(Ty: IndexTy, EC);
1699
1700 auto [BodyIP, Index] =
1701 SplitBlockAndInsertSimpleForLoop(End: NumElements, SplitBefore: InsertBefore);
1702
1703 IRB.SetInsertPoint(BodyIP);
1704 Func(IRB, Index);
1705 return;
1706 }
1707
1708 unsigned Num = EC.getFixedValue();
1709 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1710 IRB.SetInsertPoint(InsertBefore);
1711 Func(IRB, ConstantInt::get(Ty: IndexTy, V: Idx));
1712 }
1713}
1714
1715void llvm::SplitBlockAndInsertForEachLane(
1716 Value *EVL, BasicBlock::iterator InsertBefore,
1717 std::function<void(IRBuilderBase &, Value *)> Func) {
1718
1719 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1720 Type *Ty = EVL->getType();
1721
1722 if (!isa<ConstantInt>(Val: EVL)) {
1723 auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(End: EVL, SplitBefore: InsertBefore);
1724 IRB.SetInsertPoint(BodyIP);
1725 Func(IRB, Index);
1726 return;
1727 }
1728
1729 unsigned Num = cast<ConstantInt>(Val: EVL)->getZExtValue();
1730 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1731 IRB.SetInsertPoint(InsertBefore);
1732 Func(IRB, ConstantInt::get(Ty, V: Idx));
1733 }
1734}
1735
1736BranchInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1737 BasicBlock *&IfFalse) {
1738 PHINode *SomePHI = dyn_cast<PHINode>(Val: BB->begin());
1739 BasicBlock *Pred1 = nullptr;
1740 BasicBlock *Pred2 = nullptr;
1741
1742 if (SomePHI) {
1743 if (SomePHI->getNumIncomingValues() != 2)
1744 return nullptr;
1745 Pred1 = SomePHI->getIncomingBlock(i: 0);
1746 Pred2 = SomePHI->getIncomingBlock(i: 1);
1747 } else {
1748 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1749 if (PI == PE) // No predecessor
1750 return nullptr;
1751 Pred1 = *PI++;
1752 if (PI == PE) // Only one predecessor
1753 return nullptr;
1754 Pred2 = *PI++;
1755 if (PI != PE) // More than two predecessors
1756 return nullptr;
1757 }
1758
1759 // We can only handle branches. Other control flow will be lowered to
1760 // branches if possible anyway.
1761 BranchInst *Pred1Br = dyn_cast<BranchInst>(Val: Pred1->getTerminator());
1762 BranchInst *Pred2Br = dyn_cast<BranchInst>(Val: Pred2->getTerminator());
1763 if (!Pred1Br || !Pred2Br)
1764 return nullptr;
1765
1766 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1767 // either are.
1768 if (Pred2Br->isConditional()) {
1769 // If both branches are conditional, we don't have an "if statement". In
1770 // reality, we could transform this case, but since the condition will be
1771 // required anyway, we stand no chance of eliminating it, so the xform is
1772 // probably not profitable.
1773 if (Pred1Br->isConditional())
1774 return nullptr;
1775
1776 std::swap(a&: Pred1, b&: Pred2);
1777 std::swap(a&: Pred1Br, b&: Pred2Br);
1778 }
1779
1780 if (Pred1Br->isConditional()) {
1781 // The only thing we have to watch out for here is to make sure that Pred2
1782 // doesn't have incoming edges from other blocks. If it does, the condition
1783 // doesn't dominate BB.
1784 if (!Pred2->getSinglePredecessor())
1785 return nullptr;
1786
1787 // If we found a conditional branch predecessor, make sure that it branches
1788 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1789 if (Pred1Br->getSuccessor(i: 0) == BB &&
1790 Pred1Br->getSuccessor(i: 1) == Pred2) {
1791 IfTrue = Pred1;
1792 IfFalse = Pred2;
1793 } else if (Pred1Br->getSuccessor(i: 0) == Pred2 &&
1794 Pred1Br->getSuccessor(i: 1) == BB) {
1795 IfTrue = Pred2;
1796 IfFalse = Pred1;
1797 } else {
1798 // We know that one arm of the conditional goes to BB, so the other must
1799 // go somewhere unrelated, and this must not be an "if statement".
1800 return nullptr;
1801 }
1802
1803 return Pred1Br;
1804 }
1805
1806 // Ok, if we got here, both predecessors end with an unconditional branch to
1807 // BB. Don't panic! If both blocks only have a single (identical)
1808 // predecessor, and THAT is a conditional branch, then we're all ok!
1809 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1810 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1811 return nullptr;
1812
1813 // Otherwise, if this is a conditional branch, then we can use it!
1814 BranchInst *BI = dyn_cast<BranchInst>(Val: CommonPred->getTerminator());
1815 if (!BI) return nullptr;
1816
1817 assert(BI->isConditional() && "Two successors but not conditional?");
1818 if (BI->getSuccessor(i: 0) == Pred1) {
1819 IfTrue = Pred1;
1820 IfFalse = Pred2;
1821 } else {
1822 IfTrue = Pred2;
1823 IfFalse = Pred1;
1824 }
1825 return BI;
1826}
1827
1828void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
1829 Value *NewCond = PBI->getCondition();
1830 // If this is a "cmp" instruction, only used for branching (and nowhere
1831 // else), then we can simply invert the predicate.
1832 if (NewCond->hasOneUse() && isa<CmpInst>(Val: NewCond)) {
1833 CmpInst *CI = cast<CmpInst>(Val: NewCond);
1834 CI->setPredicate(CI->getInversePredicate());
1835 } else
1836 NewCond = Builder.CreateNot(V: NewCond, Name: NewCond->getName() + ".not");
1837
1838 PBI->setCondition(NewCond);
1839 PBI->swapSuccessors();
1840}
1841
1842bool llvm::hasOnlySimpleTerminator(const Function &F) {
1843 for (auto &BB : F) {
1844 auto *Term = BB.getTerminator();
1845 if (!(isa<ReturnInst>(Val: Term) || isa<UnreachableInst>(Val: Term) ||
1846 isa<BranchInst>(Val: Term)))
1847 return false;
1848 }
1849 return true;
1850}
1851
1852Printable llvm::printBasicBlock(const BasicBlock *BB) {
1853 return Printable([BB](raw_ostream &OS) {
1854 if (!BB) {
1855 OS << "<nullptr>";
1856 return;
1857 }
1858 BB->printAsOperand(O&: OS);
1859 });
1860}
1861