1//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions perform manipulations on basic blocks, and
10// instructions contained within basic blocks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/BasicBlockUtils.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallPtrSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/CFG.h"
20#include "llvm/Analysis/DomTreeUpdater.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/Analysis/MemoryDependenceAnalysis.h"
23#include "llvm/Analysis/MemorySSAUpdater.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/CFG.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/CycleInfo.h"
28#include "llvm/IR/DebugInfo.h"
29#include "llvm/IR/DebugInfoMetadata.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/LLVMContext.h"
37#include "llvm/IR/Type.h"
38#include "llvm/IR/User.h"
39#include "llvm/IR/Value.h"
40#include "llvm/IR/ValueHandle.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/Utils/Local.h"
46#include <cassert>
47#include <cstdint>
48#include <string>
49#include <utility>
50#include <vector>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "basicblock-utils"
55
56static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
57 "max-deopt-or-unreachable-succ-check-depth", cl::init(Val: 8), cl::Hidden,
58 cl::desc("Set the maximum path length when checking whether a basic block "
59 "is followed by a block that either has a terminating "
60 "deoptimizing call or is terminated with an unreachable"));
61
62/// Zap all the instructions in the block and replace them with an unreachable
63/// instruction and notify the basic block's successors that one of their
64/// predecessors is going away.
65static void
66emptyAndDetachBlock(BasicBlock *BB,
67 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
68 bool KeepOneInputPHIs) {
69 // Loop through all of our successors and make sure they know that one
70 // of their predecessors is going away.
71 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
72 for (BasicBlock *Succ : successors(BB)) {
73 Succ->removePredecessor(Pred: BB, KeepOneInputPHIs);
74 if (Updates && UniqueSuccessors.insert(Ptr: Succ).second)
75 Updates->push_back(Elt: {DominatorTree::Delete, BB, Succ});
76 }
77
78 // Zap all the instructions in the block.
79 while (!BB->empty()) {
80 Instruction &I = BB->back();
81 // If this instruction is used, replace uses with an arbitrary value.
82 // Because control flow can't get here, we don't care what we replace the
83 // value with. Note that since this block is unreachable, and all values
84 // contained within it must dominate their uses, that all uses will
85 // eventually be removed (they are themselves dead).
86 if (!I.use_empty())
87 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
88 BB->back().eraseFromParent();
89 }
90 new UnreachableInst(BB->getContext(), BB);
91 assert(BB->size() == 1 && isa<UnreachableInst>(BB->getTerminator()) &&
92 "The successor list of BB isn't empty before "
93 "applying corresponding DTU updates.");
94}
95
96bool llvm::HasLoopOrEntryConvergenceToken(const BasicBlock *BB) {
97 for (const Instruction &I : *BB) {
98 const ConvergenceControlInst *CCI = dyn_cast<ConvergenceControlInst>(Val: &I);
99 if (CCI && (CCI->isLoop() || CCI->isEntry()))
100 return true;
101 }
102 return false;
103}
104
105void llvm::detachDeadBlocks(ArrayRef<BasicBlock *> BBs,
106 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
107 bool KeepOneInputPHIs) {
108 SmallPtrSet<BasicBlock *, 4> UniqueEHRetBlocksToDelete;
109 for (auto *BB : BBs) {
110 auto NonFirstPhiIt = BB->getFirstNonPHIIt();
111 if (NonFirstPhiIt != BB->end()) {
112 Instruction &I = *NonFirstPhiIt;
113 // Exception handling funclets need to be explicitly addressed.
114 // These funclets must begin with cleanuppad or catchpad and end with
115 // cleanupred or catchret. The return instructions can be in different
116 // basic blocks than the pad instruction. If we would only delete the
117 // first block, the we would have possible cleanupret and catchret
118 // instructions with poison arguments, which wouldn't be valid.
119 if (isa<FuncletPadInst>(Val: I)) {
120 UniqueEHRetBlocksToDelete.clear();
121
122 for (User *User : I.users()) {
123 Instruction *ReturnInstr = dyn_cast<Instruction>(Val: User);
124 // If we have a cleanupret or catchret block, replace it with just an
125 // unreachable. The other alternative, that may use a catchpad is a
126 // catchswitch. That does not need special handling for now.
127 if (isa<CatchReturnInst>(Val: ReturnInstr) ||
128 isa<CleanupReturnInst>(Val: ReturnInstr)) {
129 BasicBlock *ReturnInstrBB = ReturnInstr->getParent();
130 UniqueEHRetBlocksToDelete.insert(Ptr: ReturnInstrBB);
131 }
132 }
133
134 for (BasicBlock *EHRetBB : UniqueEHRetBlocksToDelete)
135 emptyAndDetachBlock(BB: EHRetBB, Updates, KeepOneInputPHIs);
136 }
137 }
138
139 UniqueEHRetBlocksToDelete.clear();
140
141 // Detaching and emptying the current basic block.
142 emptyAndDetachBlock(BB, Updates, KeepOneInputPHIs);
143 }
144}
145
146void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
147 bool KeepOneInputPHIs) {
148 DeleteDeadBlocks(BBs: {BB}, DTU, KeepOneInputPHIs);
149}
150
151void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
152 bool KeepOneInputPHIs) {
153#ifndef NDEBUG
154 // Make sure that all predecessors of each dead block is also dead.
155 SmallPtrSet<BasicBlock *, 4> Dead(llvm::from_range, BBs);
156 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
157 for (auto *BB : Dead)
158 for (BasicBlock *Pred : predecessors(BB))
159 assert(Dead.count(Pred) && "All predecessors must be dead!");
160#endif
161
162 SmallVector<DominatorTree::UpdateType, 4> Updates;
163 detachDeadBlocks(BBs, Updates: DTU ? &Updates : nullptr, KeepOneInputPHIs);
164
165 if (DTU)
166 DTU->applyUpdates(Updates);
167
168 for (BasicBlock *BB : BBs)
169 if (DTU)
170 DTU->deleteBB(DelBB: BB);
171 else
172 BB->eraseFromParent();
173}
174
175bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
176 bool KeepOneInputPHIs) {
177 df_iterator_default_set<BasicBlock*> Reachable;
178
179 // Mark all reachable blocks.
180 for (BasicBlock *BB : depth_first_ext(G: &F, S&: Reachable))
181 (void)BB/* Mark all reachable blocks */;
182
183 // Collect all dead blocks.
184 std::vector<BasicBlock*> DeadBlocks;
185 for (BasicBlock &BB : F)
186 if (!Reachable.count(Ptr: &BB))
187 DeadBlocks.push_back(x: &BB);
188
189 // Delete the dead blocks.
190 DeleteDeadBlocks(BBs: DeadBlocks, DTU, KeepOneInputPHIs);
191
192 return !DeadBlocks.empty();
193}
194
195bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
196 MemoryDependenceResults *MemDep) {
197 if (!isa<PHINode>(Val: BB->begin()))
198 return false;
199
200 while (PHINode *PN = dyn_cast<PHINode>(Val: BB->begin())) {
201 if (PN->getIncomingValue(i: 0) != PN)
202 PN->replaceAllUsesWith(V: PN->getIncomingValue(i: 0));
203 else
204 PN->replaceAllUsesWith(V: PoisonValue::get(T: PN->getType()));
205
206 if (MemDep)
207 MemDep->removeInstruction(InstToRemove: PN); // Memdep updates AA itself.
208
209 PN->eraseFromParent();
210 }
211 return true;
212}
213
214bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
215 MemorySSAUpdater *MSSAU,
216 SmallPtrSetImpl<PHINode *> *KnownNonDeadPHIs) {
217 // Recursively deleting a PHI may cause multiple PHIs to be deleted
218 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
219 SmallVector<WeakTrackingVH, 8> PHIs(llvm::make_pointer_range(Range: BB->phis()));
220
221 SmallPtrSet<PHINode *, 32> LocalKnownNonDeadPHIs;
222 if (!KnownNonDeadPHIs)
223 KnownNonDeadPHIs = &LocalKnownNonDeadPHIs;
224
225 bool Changed = false;
226 for (const auto &PHI : PHIs) {
227 if (PHINode *PN = dyn_cast_or_null<PHINode>(Val: PHI.operator Value *())) {
228 bool PHIChanged = RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU, KnownNonDeadPHIs);
229 Changed |= PHIChanged;
230 if (PHIChanged && KnownNonDeadPHIs)
231 KnownNonDeadPHIs->clear();
232 }
233 }
234 return Changed;
235}
236
237bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
238 LoopInfo *LI, MemorySSAUpdater *MSSAU,
239 MemoryDependenceResults *MemDep,
240 bool PredecessorWithTwoSuccessors,
241 DominatorTree *DT) {
242 if (BB->hasAddressTaken())
243 return false;
244
245 // Can't merge if there are multiple predecessors, or no predecessors.
246 BasicBlock *PredBB = BB->getUniquePredecessor();
247 if (!PredBB) return false;
248
249 // Don't break self-loops.
250 if (PredBB == BB) return false;
251
252 // Don't break unwinding instructions or terminators with other side-effects.
253 Instruction *PTI = PredBB->getTerminator();
254 if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
255 return false;
256
257 // Can't merge if there are multiple distinct successors.
258 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
259 return false;
260
261 // Currently only allow PredBB to have two predecessors, one being BB.
262 // Update BI to branch to BB's only successor instead of BB.
263 CondBrInst *PredBB_BI;
264 BasicBlock *NewSucc = nullptr;
265 unsigned FallThruPath;
266 if (PredecessorWithTwoSuccessors) {
267 if (!(PredBB_BI = dyn_cast<CondBrInst>(Val: PTI)))
268 return false;
269 UncondBrInst *BB_JmpI = dyn_cast<UncondBrInst>(Val: BB->getTerminator());
270 if (!BB_JmpI)
271 return false;
272 NewSucc = BB_JmpI->getSuccessor();
273 FallThruPath = PredBB_BI->getSuccessor(i: 0) == BB ? 0 : 1;
274 }
275
276 // Can't merge if there is PHI loop.
277 for (PHINode &PN : BB->phis())
278 if (llvm::is_contained(Range: PN.incoming_values(), Element: &PN))
279 return false;
280
281 // Don't break if both the basic block and the predecessor contain loop or
282 // entry convergent intrinsics, since there may only be one convergence token
283 // per block.
284 if (HasLoopOrEntryConvergenceToken(BB) &&
285 HasLoopOrEntryConvergenceToken(BB: PredBB))
286 return false;
287
288 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
289 << PredBB->getName() << "\n");
290
291 // Begin by getting rid of unneeded PHIs.
292 SmallVector<AssertingVH<Value>, 4> IncomingValues;
293 if (isa<PHINode>(Val: BB->front())) {
294 for (PHINode &PN : BB->phis())
295 if (!isa<PHINode>(Val: PN.getIncomingValue(i: 0)) ||
296 cast<PHINode>(Val: PN.getIncomingValue(i: 0))->getParent() != BB)
297 IncomingValues.push_back(Elt: PN.getIncomingValue(i: 0));
298 FoldSingleEntryPHINodes(BB, MemDep);
299 }
300
301 if (DT) {
302 assert(!DTU && "cannot use both DT and DTU for updates");
303 DomTreeNode *PredNode = DT->getNode(BB: PredBB);
304 DomTreeNode *BBNode = DT->getNode(BB);
305 if (PredNode) {
306 assert(BBNode && "PredNode unreachable but BBNode reachable?");
307 for (DomTreeNode *C : to_vector(Range: BBNode->children()))
308 C->setIDom(PredNode);
309 }
310 }
311 // DTU update: Collect all the edges that exit BB.
312 // These dominator edges will be redirected from Pred.
313 std::vector<DominatorTree::UpdateType> Updates;
314 if (DTU) {
315 assert(!DT && "cannot use both DT and DTU for updates");
316 // To avoid processing the same predecessor more than once.
317 SmallPtrSet<BasicBlock *, 8> SeenSuccs;
318 SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(llvm::from_range,
319 successors(BB: PredBB));
320 Updates.reserve(n: Updates.size() + 2 * succ_size(BB) + 1);
321 // Add insert edges first. Experimentally, for the particular case of two
322 // blocks that can be merged, with a single successor and single predecessor
323 // respectively, it is beneficial to have all insert updates first. Deleting
324 // edges first may lead to unreachable blocks, followed by inserting edges
325 // making the blocks reachable again. Such DT updates lead to high compile
326 // times. We add inserts before deletes here to reduce compile time.
327 for (BasicBlock *SuccOfBB : successors(BB))
328 // This successor of BB may already be a PredBB's successor.
329 if (!SuccsOfPredBB.contains(Ptr: SuccOfBB))
330 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
331 Updates.push_back(x: {DominatorTree::Insert, PredBB, SuccOfBB});
332 SeenSuccs.clear();
333 for (BasicBlock *SuccOfBB : successors(BB))
334 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
335 Updates.push_back(x: {DominatorTree::Delete, BB, SuccOfBB});
336 Updates.push_back(x: {DominatorTree::Delete, PredBB, BB});
337 }
338
339 Instruction *STI = BB->getTerminator();
340 Instruction *Start = &*BB->begin();
341 // If there's nothing to move, mark the starting instruction as the last
342 // instruction in the block. Terminator instruction is handled separately.
343 if (Start == STI)
344 Start = PTI;
345
346 // Move all definitions in the successor to the predecessor...
347 PredBB->splice(ToIt: PTI->getIterator(), FromBB: BB, FromBeginIt: BB->begin(), FromEndIt: STI->getIterator());
348
349 if (MSSAU)
350 MSSAU->moveAllAfterMergeBlocks(From: BB, To: PredBB, Start);
351
352 // Make all PHI nodes that referred to BB now refer to Pred as their
353 // source...
354 BB->replaceAllUsesWith(V: PredBB);
355
356 if (PredecessorWithTwoSuccessors) {
357 // Delete the unconditional branch from BB.
358 BB->back().eraseFromParent();
359 // Add unreachable to now empty BB.
360 new UnreachableInst(BB->getContext(), BB);
361
362 // Update branch in the predecessor.
363 PredBB_BI->setSuccessor(idx: FallThruPath, NewSucc);
364 } else {
365 // Delete the unconditional branch from the predecessor.
366 PredBB->back().eraseFromParent();
367
368 // Move terminator instruction.
369 BB->back().moveBeforePreserving(BB&: *PredBB, I: PredBB->end());
370 // Add unreachable to now empty BB.
371 new UnreachableInst(BB->getContext(), BB);
372
373 // Terminator may be a memory accessing instruction too.
374 if (MSSAU)
375 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
376 Val: MSSAU->getMemorySSA()->getMemoryAccess(I: PredBB->getTerminator())))
377 MSSAU->moveToPlace(What: MUD, BB: PredBB, Where: MemorySSA::End);
378 }
379
380 // Inherit predecessors name if it exists.
381 if (!PredBB->hasName())
382 PredBB->takeName(V: BB);
383
384 if (LI)
385 LI->removeBlock(BB);
386
387 if (MemDep)
388 MemDep->invalidateCachedPredecessors();
389
390 if (DTU)
391 DTU->applyUpdates(Updates);
392
393 if (DT) {
394 assert(succ_empty(BB) &&
395 "successors should have been transferred to PredBB");
396 DT->eraseNode(BB);
397 }
398
399 // Finally, erase the old block and update dominator info.
400 DeleteDeadBlock(BB, DTU);
401
402 return true;
403}
404
405bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
406 SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
407 LoopInfo *LI) {
408 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
409
410 bool BlocksHaveBeenMerged = false;
411 while (!MergeBlocks.empty()) {
412 BasicBlock *BB = *MergeBlocks.begin();
413 BasicBlock *Dest = BB->getSingleSuccessor();
414 if (Dest && (!L || L->contains(BB: Dest))) {
415 BasicBlock *Fold = Dest->getUniquePredecessor();
416 (void)Fold;
417 if (MergeBlockIntoPredecessor(BB: Dest, DTU, LI)) {
418 assert(Fold == BB &&
419 "Expecting BB to be unique predecessor of the Dest block");
420 MergeBlocks.erase(Ptr: Dest);
421 BlocksHaveBeenMerged = true;
422 } else
423 MergeBlocks.erase(Ptr: BB);
424 } else
425 MergeBlocks.erase(Ptr: BB);
426 }
427 return BlocksHaveBeenMerged;
428}
429
430/// Remove redundant instructions within sequences of consecutive dbg.value
431/// instructions. This is done using a backward scan to keep the last dbg.value
432/// describing a specific variable/fragment.
433///
434/// BackwardScan strategy:
435/// ----------------------
436/// Given a sequence of consecutive DbgValueInst like this
437///
438/// dbg.value ..., "x", FragmentX1 (*)
439/// dbg.value ..., "y", FragmentY1
440/// dbg.value ..., "x", FragmentX2
441/// dbg.value ..., "x", FragmentX1 (**)
442///
443/// then the instruction marked with (*) can be removed (it is guaranteed to be
444/// obsoleted by the instruction marked with (**) as the latter instruction is
445/// describing the same variable using the same fragment info).
446///
447/// Possible improvements:
448/// - Check fully overlapping fragments and not only identical fragments.
449static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
450 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
451 SmallDenseSet<DebugVariable> VariableSet;
452 for (auto &I : reverse(C&: *BB)) {
453 for (DbgVariableRecord &DVR :
454 reverse(C: filterDbgVars(R: I.getDbgRecordRange()))) {
455 DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
456 DVR.getDebugLoc()->getInlinedAt());
457 auto R = VariableSet.insert(V: Key);
458 // If the same variable fragment is described more than once it is enough
459 // to keep the last one (i.e. the first found since we for reverse
460 // iteration).
461 if (R.second)
462 continue;
463
464 if (DVR.isDbgAssign()) {
465 // Don't delete dbg.assign intrinsics that are linked to instructions.
466 if (!at::getAssignmentInsts(DVR: &DVR).empty())
467 continue;
468 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
469 }
470
471 ToBeRemoved.push_back(Elt: &DVR);
472 }
473 // Sequence with consecutive dbg.value instrs ended. Clear the map to
474 // restart identifying redundant instructions if case we find another
475 // dbg.value sequence.
476 VariableSet.clear();
477 }
478
479 for (auto &DVR : ToBeRemoved)
480 DVR->eraseFromParent();
481
482 return !ToBeRemoved.empty();
483}
484
485/// Remove redundant dbg.value instructions using a forward scan. This can
486/// remove a dbg.value instruction that is redundant due to indicating that a
487/// variable has the same value as already being indicated by an earlier
488/// dbg.value.
489///
490/// ForwardScan strategy:
491/// ---------------------
492/// Given two identical dbg.value instructions, separated by a block of
493/// instructions that isn't describing the same variable, like this
494///
495/// dbg.value X1, "x", FragmentX1 (**)
496/// <block of instructions, none being "dbg.value ..., "x", ...">
497/// dbg.value X1, "x", FragmentX1 (*)
498///
499/// then the instruction marked with (*) can be removed. Variable "x" is already
500/// described as being mapped to the SSA value X1.
501///
502/// Possible improvements:
503/// - Keep track of non-overlapping fragments.
504static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
505 bool RemovedAny = false;
506 SmallDenseMap<DebugVariable,
507 std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
508 VariableMap;
509 for (auto &I : *BB) {
510 for (DbgVariableRecord &DVR :
511 make_early_inc_range(Range: filterDbgVars(R: I.getDbgRecordRange()))) {
512 if (DVR.getType() == DbgVariableRecord::LocationType::Declare)
513 continue;
514 DebugVariable Key(DVR.getVariable(), std::nullopt,
515 DVR.getDebugLoc()->getInlinedAt());
516 auto [VMI, Inserted] = VariableMap.try_emplace(Key);
517 // A dbg.assign with no linked instructions can be treated like a
518 // dbg.value (i.e. can be deleted).
519 bool IsDbgValueKind =
520 (!DVR.isDbgAssign() || at::getAssignmentInsts(DVR: &DVR).empty());
521
522 // Update the map if we found a new value/expression describing the
523 // variable, or if the variable wasn't mapped already.
524 SmallVector<Value *, 4> Values(DVR.location_ops());
525 if (Inserted || VMI->second.first != Values ||
526 VMI->second.second != DVR.getExpression()) {
527 if (IsDbgValueKind)
528 VMI->second = {Values, DVR.getExpression()};
529 else
530 VMI->second = {Values, nullptr};
531 continue;
532 }
533 // Don't delete dbg.assign intrinsics that are linked to instructions.
534 if (!IsDbgValueKind)
535 continue;
536 // Found an identical mapping. Remember the instruction for later removal.
537 DVR.eraseFromParent();
538 RemovedAny = true;
539 }
540 }
541
542 return RemovedAny;
543}
544
545/// Remove redundant undef dbg.assign intrinsic from an entry block using a
546/// forward scan.
547/// Strategy:
548/// ---------------------
549/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
550/// linked to an intrinsic, and don't share an aggregate variable with a debug
551/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
552/// that come before non-undef debug intrinsics for the variable are
553/// deleted. Given:
554///
555/// dbg.assign undef, "x", FragmentX1 (*)
556/// <block of instructions, none being "dbg.value ..., "x", ...">
557/// dbg.value %V, "x", FragmentX2
558/// <block of instructions, none being "dbg.value ..., "x", ...">
559/// dbg.assign undef, "x", FragmentX1
560///
561/// then (only) the instruction marked with (*) can be removed.
562/// Possible improvements:
563/// - Keep track of non-overlapping fragments.
564static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
565 assert(BB->isEntryBlock() && "expected entry block");
566 bool RemovedAny = false;
567 DenseSet<DebugVariableAggregate> SeenDefForAggregate;
568
569 // Remove undef dbg.assign intrinsics that are encountered before
570 // any non-undef intrinsics from the entry block.
571 for (auto &I : *BB) {
572 for (DbgVariableRecord &DVR :
573 make_early_inc_range(Range: filterDbgVars(R: I.getDbgRecordRange()))) {
574 if (!DVR.isDbgValue() && !DVR.isDbgAssign())
575 continue;
576 bool IsDbgValueKind =
577 (DVR.isDbgValue() || at::getAssignmentInsts(DVR: &DVR).empty());
578
579 DebugVariableAggregate Aggregate(&DVR);
580 if (!SeenDefForAggregate.contains(V: Aggregate)) {
581 bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
582 if (!IsKill) {
583 SeenDefForAggregate.insert(V: Aggregate);
584 } else if (DVR.isDbgAssign()) {
585 DVR.eraseFromParent();
586 RemovedAny = true;
587 }
588 }
589 }
590 }
591
592 return RemovedAny;
593}
594
595bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
596 bool MadeChanges = false;
597 // By using the "backward scan" strategy before the "forward scan" strategy we
598 // can remove both dbg.value (2) and (3) in a situation like this:
599 //
600 // (1) dbg.value V1, "x", DIExpression()
601 // ...
602 // (2) dbg.value V2, "x", DIExpression()
603 // (3) dbg.value V1, "x", DIExpression()
604 //
605 // The backward scan will remove (2), it is made obsolete by (3). After
606 // getting (2) out of the way, the foward scan will remove (3) since "x"
607 // already is described as having the value V1 at (1).
608 MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
609 if (BB->isEntryBlock() &&
610 isAssignmentTrackingEnabled(M: *BB->getParent()->getParent()))
611 MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
612 MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
613
614 if (MadeChanges)
615 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
616 << BB->getName() << "\n");
617 return MadeChanges;
618}
619
620void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
621 Instruction &I = *BI;
622 // Replaces all of the uses of the instruction with uses of the value
623 I.replaceAllUsesWith(V);
624
625 // Make sure to propagate a name if there is one already.
626 if (I.hasName() && !V->hasName())
627 V->takeName(V: &I);
628
629 // Delete the unnecessary instruction now...
630 BI = BI->eraseFromParent();
631}
632
633void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
634 Instruction *I) {
635 assert(I->getParent() == nullptr &&
636 "ReplaceInstWithInst: Instruction already inserted into basic block!");
637
638 // Copy debug location to newly added instruction, if it wasn't already set
639 // by the caller.
640 if (!I->getDebugLoc())
641 I->setDebugLoc(BI->getDebugLoc());
642
643 // Insert the new instruction into the basic block...
644 BasicBlock::iterator New = I->insertInto(ParentBB: BB, It: BI);
645
646 // Replace all uses of the old instruction, and delete it.
647 ReplaceInstWithValue(BI, V: I);
648
649 // Move BI back to point to the newly inserted instruction
650 BI = New;
651}
652
653bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
654 // Remember visited blocks to avoid infinite loop
655 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
656 unsigned Depth = 0;
657 while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
658 VisitedBlocks.insert(Ptr: BB).second) {
659 if (isa<UnreachableInst>(Val: BB->getTerminator()) ||
660 BB->getTerminatingDeoptimizeCall())
661 return true;
662 BB = BB->getUniqueSuccessor();
663 }
664 return false;
665}
666
667void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
668 BasicBlock::iterator BI(From);
669 ReplaceInstWithInst(BB: From->getParent(), BI, I: To);
670}
671
672BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
673 LoopInfo *LI, MemorySSAUpdater *MSSAU,
674 const Twine &BBName) {
675 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
676
677 Instruction *LatchTerm = BB->getTerminator();
678
679 CriticalEdgeSplittingOptions Options =
680 CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
681
682 if ((isCriticalEdge(TI: LatchTerm, SuccNum, AllowIdenticalEdges: Options.MergeIdenticalEdges))) {
683 // If this is a critical edge, let SplitKnownCriticalEdge do it.
684 return SplitKnownCriticalEdge(TI: LatchTerm, SuccNum, Options, BBName);
685 }
686
687 // If the edge isn't critical, then BB has a single successor or Succ has a
688 // single pred. Split the block.
689 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
690 // If the successor only has a single pred, split the top of the successor
691 // block.
692 assert(SP == BB && "CFG broken");
693 (void)SP;
694 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
695 return splitBlockBefore(Old: Succ, SplitPt: &Succ->front(), DTU: &DTU, LI, MSSAU, BBName);
696 }
697
698 // Otherwise, if BB has a single successor, split it at the bottom of the
699 // block.
700 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
701 "Should have a single succ!");
702 return SplitBlock(Old: BB, SplitPt: BB->getTerminator(), DT, LI, MSSAU, BBName);
703}
704
705/// Helper function to update the cycle or loop information after inserting a
706/// new block between a callbr or switch instruction and one of its target
707/// blocks. Adds the new block to the innermost cycle or loop that the
708/// callbr/switch instruction and the original target block share.
709/// \p LCI cycle or loop information to update
710/// \p MultiBrBlock block containing the callbr/switch instruction
711/// \p BrTarget new target block of the callbr/switch instruction
712/// \p Succ original target block of the callbr/switch instruction
713template <typename TI, typename T>
714static bool updateCycleLoopInfo(TI *LCI, BasicBlock *MultiBrBlock,
715 BasicBlock *BrTarget, BasicBlock *Succ) {
716 static_assert(std::is_same_v<TI, CycleInfo> || std::is_same_v<TI, LoopInfo>,
717 "type must be CycleInfo or LoopInfo");
718 if (!LCI)
719 return false;
720
721 if constexpr (std::is_same_v<TI, CycleInfo>) {
722 T LC = LCI->getSmallestCommonCycle(MultiBrBlock, Succ);
723 if (!LC)
724 return false;
725 LCI->addBlockToCycle(BrTarget, LC);
726 } else {
727 T *LC = LCI->getSmallestCommonLoop(MultiBrBlock, Succ);
728 if (!LC)
729 return false;
730 LC->addBasicBlockToLoop(BrTarget, *LCI);
731 }
732
733 return true;
734}
735
736BasicBlock *llvm::SplitMultiBrEdge(BasicBlock *MultiBrBlock, BasicBlock *Succ,
737 unsigned SuccIdx, BasicBlock *BrTarget,
738 DomTreeUpdater *DTU, CycleInfo *CI,
739 LoopInfo *LI, bool *UpdatedLI) {
740 Instruction *Term = MultiBrBlock->getTerminator();
741 assert((isa_and_present<SwitchInst>(Term) ||
742 isa_and_present<CallBrInst>(Term)) &&
743 "expected callbr or switch terminator");
744 assert(SuccIdx < Term->getNumSuccessors() &&
745 Succ == Term->getSuccessor(SuccIdx) && "invalid successor index");
746
747 if (UpdatedLI)
748 *UpdatedLI = false;
749
750 bool ReusesBrTarget = BrTarget;
751
752 // Create a new block between terminator and the specified successor.
753 // splitBlockBefore cannot be re-used here since it cannot split if the split
754 // point is a PHI node (because BasicBlock::splitBasicBlockBefore cannot
755 // handle that). But we don't need to rewire every part of a potential PHI
756 // node. We only care about the edge between MultiBrBlock and the original
757 // successor.
758 if (!ReusesBrTarget) {
759 BrTarget = BasicBlock::Create(Context&: MultiBrBlock->getContext(),
760 Name: MultiBrBlock->getName() + ".target." +
761 Succ->getName(),
762 Parent: MultiBrBlock->getParent());
763 // Jump from the new target block to the original successor.
764 UncondBrInst::Create(Target: Succ, InsertBefore: BrTarget);
765 // Replace a single incoming value with the target block. We cannot
766 // use replacePhiUsesWith, as this would replace the value for every edge
767 // from the MultiBrBlock to the successor.
768 for (PHINode &PN : Succ->phis()) {
769 int BBIdx = PN.getBasicBlockIndex(BB: MultiBrBlock);
770 assert(BBIdx != -1 && "expected incoming value from MultiBrBlock");
771 PN.setIncomingBlock(i: BBIdx, BB: BrTarget);
772 }
773
774 bool Updated =
775 updateCycleLoopInfo<LoopInfo, Loop>(LCI: LI, MultiBrBlock, BrTarget, Succ);
776 if (UpdatedLI)
777 *UpdatedLI = Updated;
778 updateCycleLoopInfo<CycleInfo, CycleRef>(LCI: CI, MultiBrBlock, BrTarget, Succ);
779 } else {
780 for (PHINode &PN : Succ->phis())
781 PN.removeIncomingValue(BB: MultiBrBlock, DeletePHIIfEmpty: false);
782 }
783
784 // Rewire control flow from the MultiBrBlock to the new target block.
785 Term->setSuccessor(Idx: SuccIdx, BB: BrTarget);
786
787 if (DTU) {
788 if (!ReusesBrTarget)
789 DTU->applyUpdates(Updates: {{DominatorTree::Insert, MultiBrBlock, BrTarget}});
790 if (DTU->getDomTree().dominates(A: MultiBrBlock, B: Succ)) {
791 if (!is_contained(Range: successors(BB: MultiBrBlock), Element: Succ))
792 DTU->applyUpdates(Updates: {{DominatorTree::Delete, MultiBrBlock, Succ}});
793 if (!ReusesBrTarget)
794 DTU->applyUpdates(Updates: {{DominatorTree::Insert, BrTarget, Succ}});
795 }
796 }
797
798 return BrTarget;
799}
800
801void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
802 if (auto *II = dyn_cast<InvokeInst>(Val: TI))
803 II->setUnwindDest(Succ);
804 else if (auto *CS = dyn_cast<CatchSwitchInst>(Val: TI))
805 CS->setUnwindDest(Succ);
806 else if (auto *CR = dyn_cast<CleanupReturnInst>(Val: TI))
807 CR->setUnwindDest(Succ);
808 else
809 llvm_unreachable("unexpected terminator instruction");
810}
811
812void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
813 BasicBlock *NewPred, PHINode *Until) {
814 int BBIdx = 0;
815 for (PHINode &PN : DestBB->phis()) {
816 // We manually update the LandingPadReplacement PHINode and it is the last
817 // PHI Node. So, if we find it, we are done.
818 if (Until == &PN)
819 break;
820
821 // Reuse the previous value of BBIdx if it lines up. In cases where we
822 // have multiple phi nodes with *lots* of predecessors, this is a speed
823 // win because we don't have to scan the PHI looking for TIBB. This
824 // happens because the BB list of PHI nodes are usually in the same
825 // order.
826 if (PN.getIncomingBlock(i: BBIdx) != OldPred)
827 BBIdx = PN.getBasicBlockIndex(BB: OldPred);
828
829 assert(BBIdx != -1 && "Invalid PHI Index!");
830 PN.setIncomingBlock(i: BBIdx, BB: NewPred);
831 }
832}
833
834BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
835 LandingPadInst *OriginalPad,
836 PHINode *LandingPadReplacement,
837 const CriticalEdgeSplittingOptions &Options,
838 const Twine &BBName) {
839
840 auto PadInst = Succ->getFirstNonPHIIt();
841 if (!LandingPadReplacement && !PadInst->isEHPad())
842 return SplitEdge(BB, Succ, DT: Options.DT, LI: Options.LI, MSSAU: Options.MSSAU, BBName);
843
844 auto *LI = Options.LI;
845 SmallVector<BasicBlock *, 4> LoopPreds;
846 // Check if extra modifications will be required to preserve loop-simplify
847 // form after splitting. If it would require splitting blocks with IndirectBr
848 // terminators, bail out if preserving loop-simplify form is requested.
849 if (Options.PreserveLoopSimplify && LI) {
850 if (Loop *BBLoop = LI->getLoopFor(BB)) {
851
852 // The only way that we can break LoopSimplify form by splitting a
853 // critical edge is when there exists some edge from BBLoop to Succ *and*
854 // the only edge into Succ from outside of BBLoop is that of NewBB after
855 // the split. If the first isn't true, then LoopSimplify still holds,
856 // NewBB is the new exit block and it has no non-loop predecessors. If the
857 // second isn't true, then Succ was not in LoopSimplify form prior to
858 // the split as it had a non-loop predecessor. In both of these cases,
859 // the predecessor must be directly in BBLoop, not in a subloop, or again
860 // LoopSimplify doesn't hold.
861 for (BasicBlock *P : predecessors(BB: Succ)) {
862 if (P == BB)
863 continue; // The new block is known.
864 if (LI->getLoopFor(BB: P) != BBLoop) {
865 // Loop is not in LoopSimplify form, no need to re simplify after
866 // splitting edge.
867 LoopPreds.clear();
868 break;
869 }
870 LoopPreds.push_back(Elt: P);
871 }
872 // Loop-simplify form can be preserved, if we can split all in-loop
873 // predecessors.
874 if (any_of(Range&: LoopPreds, P: [](BasicBlock *Pred) {
875 return isa<IndirectBrInst>(Val: Pred->getTerminator());
876 })) {
877 return nullptr;
878 }
879 }
880 }
881
882 auto *NewBB =
883 BasicBlock::Create(Context&: BB->getContext(), Name: BBName, Parent: BB->getParent(), InsertBefore: Succ);
884 setUnwindEdgeTo(TI: BB->getTerminator(), Succ: NewBB);
885 updatePhiNodes(DestBB: Succ, OldPred: BB, NewPred: NewBB, Until: LandingPadReplacement);
886
887 if (LandingPadReplacement) {
888 auto *NewLP = OriginalPad->clone();
889 auto *Terminator = UncondBrInst::Create(Target: Succ, InsertBefore: NewBB);
890 NewLP->insertBefore(InsertPos: Terminator->getIterator());
891 LandingPadReplacement->addIncoming(V: NewLP, BB: NewBB);
892 } else {
893 Value *ParentPad = nullptr;
894 if (auto *FuncletPad = dyn_cast<FuncletPadInst>(Val&: PadInst))
895 ParentPad = FuncletPad->getParentPad();
896 else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: PadInst))
897 ParentPad = CatchSwitch->getParentPad();
898 else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(Val&: PadInst))
899 ParentPad = CleanupPad->getParentPad();
900 else if (auto *LandingPad = dyn_cast<LandingPadInst>(Val&: PadInst))
901 ParentPad = LandingPad->getParent();
902 else
903 llvm_unreachable("handling for other EHPads not implemented yet");
904
905 auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, Args: {}, NameStr: BBName, InsertBefore: NewBB);
906 CleanupReturnInst::Create(CleanupPad: NewCleanupPad, UnwindBB: Succ, InsertBefore: NewBB);
907 }
908
909 auto *DT = Options.DT;
910 auto *MSSAU = Options.MSSAU;
911 if (!DT && !LI)
912 return NewBB;
913
914 if (DT) {
915 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
916 SmallVector<DominatorTree::UpdateType, 3> Updates;
917
918 Updates.push_back(Elt: {DominatorTree::Insert, BB, NewBB});
919 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, Succ});
920 Updates.push_back(Elt: {DominatorTree::Delete, BB, Succ});
921
922 DTU.applyUpdates(Updates);
923 DTU.flush();
924
925 if (MSSAU) {
926 MSSAU->applyUpdates(Updates, DT&: *DT);
927 if (VerifyMemorySSA)
928 MSSAU->getMemorySSA()->verifyMemorySSA();
929 }
930 }
931
932 if (LI) {
933 if (Loop *BBLoop = LI->getLoopFor(BB)) {
934 // If one or the other blocks were not in a loop, the new block is not
935 // either, and thus LI doesn't need to be updated.
936 if (Loop *SuccLoop = LI->getLoopFor(BB: Succ)) {
937 if (BBLoop == SuccLoop) {
938 // Both in the same loop, the NewBB joins loop.
939 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
940 } else if (BBLoop->contains(L: SuccLoop)) {
941 // Edge from an outer loop to an inner loop. Add to the outer loop.
942 BBLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
943 } else if (SuccLoop->contains(L: BBLoop)) {
944 // Edge from an inner loop to an outer loop. Add to the outer loop.
945 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
946 } else {
947 // Edge from two loops with no containment relation. Because these
948 // are natural loops, we know that the destination block must be the
949 // header of its loop (adding a branch into a loop elsewhere would
950 // create an irreducible loop).
951 assert(SuccLoop->getHeader() == Succ &&
952 "Should not create irreducible loops!");
953 if (Loop *P = SuccLoop->getParentLoop())
954 P->addBasicBlockToLoop(NewBB, LI&: *LI);
955 }
956 }
957
958 // If BB is in a loop and Succ is outside of that loop, we may need to
959 // update LoopSimplify form and LCSSA form.
960 if (!BBLoop->contains(BB: Succ)) {
961 assert(!BBLoop->contains(NewBB) &&
962 "Split point for loop exit is contained in loop!");
963
964 // Update LCSSA form in the newly created exit block.
965 if (Options.PreserveLCSSA) {
966 createPHIsForSplitLoopExit(Preds: BB, SplitBB: NewBB, DestBB: Succ);
967 }
968
969 if (!LoopPreds.empty()) {
970 BasicBlock *NewExitBB = SplitBlockPredecessors(
971 BB: Succ, Preds: LoopPreds, Suffix: "split", DT, LI, MSSAU, PreserveLCSSA: Options.PreserveLCSSA);
972 if (Options.PreserveLCSSA)
973 createPHIsForSplitLoopExit(Preds: LoopPreds, SplitBB: NewExitBB, DestBB: Succ);
974 }
975 }
976 }
977 }
978
979 return NewBB;
980}
981
982void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
983 BasicBlock *SplitBB, BasicBlock *DestBB) {
984 // SplitBB shouldn't have anything non-trivial in it yet.
985 assert((&*SplitBB->getFirstNonPHIIt() == SplitBB->getTerminator() ||
986 SplitBB->isLandingPad()) &&
987 "SplitBB has non-PHI nodes!");
988
989 // For each PHI in the destination block.
990 for (PHINode &PN : DestBB->phis()) {
991 int Idx = PN.getBasicBlockIndex(BB: SplitBB);
992 assert(Idx >= 0 && "Invalid Block Index");
993 Value *V = PN.getIncomingValue(i: Idx);
994
995 // If the input is a PHI which already satisfies LCSSA, don't create
996 // a new one.
997 if (const PHINode *VP = dyn_cast<PHINode>(Val: V))
998 if (VP->getParent() == SplitBB)
999 continue;
1000
1001 // Otherwise a new PHI is needed. Create one and populate it.
1002 PHINode *NewPN = PHINode::Create(Ty: PN.getType(), NumReservedValues: Preds.size(), NameStr: "split");
1003 BasicBlock::iterator InsertPos =
1004 SplitBB->isLandingPad() ? SplitBB->begin()
1005 : SplitBB->getTerminator()->getIterator();
1006 NewPN->insertBefore(InsertPos);
1007 for (BasicBlock *BB : Preds)
1008 NewPN->addIncoming(V, BB);
1009
1010 // Update the original PHI.
1011 PN.setIncomingValue(i: Idx, V: NewPN);
1012 }
1013}
1014
1015unsigned
1016llvm::SplitAllCriticalEdges(Function &F,
1017 const CriticalEdgeSplittingOptions &Options) {
1018 unsigned NumBroken = 0;
1019 for (BasicBlock &BB : F) {
1020 Instruction *TI = BB.getTerminator();
1021 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(Val: TI))
1022 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
1023 if (SplitCriticalEdge(TI, SuccNum: i, Options))
1024 ++NumBroken;
1025 }
1026 return NumBroken;
1027}
1028
1029static BasicBlock *SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt,
1030 DomTreeUpdater *DTU, DominatorTree *DT,
1031 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1032 const Twine &BBName) {
1033 BasicBlock::iterator SplitIt = SplitPt;
1034 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad()) {
1035 ++SplitIt;
1036 assert(SplitIt != SplitPt->getParent()->end());
1037 }
1038 std::string Name = BBName.str();
1039 BasicBlock *New = Old->splitBasicBlock(
1040 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name);
1041
1042 // The new block lives in whichever loop the old one did. This preserves
1043 // LCSSA as well, because we force the split point to be after any PHI nodes.
1044 if (LI)
1045 if (Loop *L = LI->getLoopFor(BB: Old))
1046 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
1047
1048 if (DTU) {
1049 SmallVector<DominatorTree::UpdateType, 8> Updates;
1050 // Old dominates New. New node dominates all other nodes dominated by Old.
1051 SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
1052 Updates.push_back(Elt: {DominatorTree::Insert, Old, New});
1053 Updates.reserve(N: Updates.size() + 2 * succ_size(BB: New));
1054 for (BasicBlock *SuccessorOfOld : successors(BB: New))
1055 if (UniqueSuccessorsOfOld.insert(Ptr: SuccessorOfOld).second) {
1056 Updates.push_back(Elt: {DominatorTree::Insert, New, SuccessorOfOld});
1057 Updates.push_back(Elt: {DominatorTree::Delete, Old, SuccessorOfOld});
1058 }
1059
1060 DTU->applyUpdates(Updates);
1061 } else if (DT)
1062 // Old dominates New. New node dominates all other nodes dominated by Old.
1063 if (DomTreeNode *OldNode = DT->getNode(BB: Old)) {
1064 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1065
1066 DomTreeNode *NewNode = DT->addNewBlock(BB: New, DomBB: Old);
1067 for (DomTreeNode *I : Children)
1068 DT->changeImmediateDominator(N: I, NewIDom: NewNode);
1069 }
1070
1071 // Move MemoryAccesses still tracked in Old, but part of New now.
1072 // Update accesses in successor blocks accordingly.
1073 if (MSSAU)
1074 MSSAU->moveAllAfterSpliceBlocks(From: Old, To: New, Start: &*(New->begin()));
1075
1076 return New;
1077}
1078
1079BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1080 DominatorTree *DT, LoopInfo *LI,
1081 MemorySSAUpdater *MSSAU, const Twine &BBName) {
1082 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName);
1083}
1084BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1085 DomTreeUpdater *DTU, LoopInfo *LI,
1086 MemorySSAUpdater *MSSAU, const Twine &BBName) {
1087 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName);
1088}
1089
1090static bool hasReachableLoopEntryToHeader(const Loop &L,
1091 const DominatorTree &DT) {
1092 for (const BasicBlock *Pred : predecessors(BB: L.getHeader()))
1093 if (!L.contains(BB: Pred) && DT.isReachableFromEntry(A: Pred))
1094 return true;
1095
1096 return false;
1097}
1098
1099/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
1100/// Invalidates DFS Numbering when DTU or DT is provided.
1101static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
1102 ArrayRef<BasicBlock *> Preds,
1103 DomTreeUpdater *DTU, DominatorTree *DT,
1104 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1105 bool PreserveLCSSA, bool &HasLoopExit) {
1106 // Update dominator tree if available.
1107 if (DTU) {
1108 // Recalculation of DomTree is needed when updating a forward DomTree and
1109 // the Entry BB is replaced.
1110 if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1111 // The entry block was removed and there is no external interface for
1112 // the dominator tree to be notified of this change. In this corner-case
1113 // we recalculate the entire tree.
1114 DTU->recalculate(F&: *NewBB->getParent());
1115 } else {
1116 // Split block expects NewBB to have a non-empty set of predecessors.
1117 SmallVector<DominatorTree::UpdateType, 8> Updates;
1118 SmallPtrSet<BasicBlock *, 8> UniquePreds;
1119 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, OldBB});
1120 Updates.reserve(N: Updates.size() + 2 * Preds.size());
1121 for (auto *Pred : Preds)
1122 if (UniquePreds.insert(Ptr: Pred).second) {
1123 Updates.push_back(Elt: {DominatorTree::Insert, Pred, NewBB});
1124 Updates.push_back(Elt: {DominatorTree::Delete, Pred, OldBB});
1125 }
1126 DTU->applyUpdates(Updates);
1127 }
1128 } else if (DT) {
1129 if (OldBB == DT->getRootNode()->getBlock()) {
1130 assert(NewBB->isEntryBlock());
1131 DT->setNewRoot(NewBB);
1132 } else {
1133 // Split block expects NewBB to have a non-empty set of predecessors.
1134 DT->splitBlock(NewBB);
1135 }
1136 }
1137
1138 // Update MemoryPhis after split if MemorySSA is available
1139 if (MSSAU)
1140 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(Old: OldBB, New: NewBB, Preds);
1141
1142 // The rest of the logic is only relevant for updating the loop structures.
1143 if (!LI)
1144 return;
1145
1146 if (DTU && DTU->hasDomTree())
1147 DT = &DTU->getDomTree();
1148 assert(DT && "DT should be available to update LoopInfo!");
1149 Loop *L = LI->getLoopFor(BB: OldBB);
1150
1151 // If we need to preserve loop analyses, collect some information about how
1152 // this split will affect loops.
1153 bool IsLoopEntry = !!L;
1154 bool SplitMakesNewLoopHeader = false;
1155 for (BasicBlock *Pred : Preds) {
1156 // Preds that are not reachable from entry should not be used to identify if
1157 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1158 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1159 // as true and make the NewBB the header of some loop. This breaks LI.
1160 if (!DT->isReachableFromEntry(A: Pred))
1161 continue;
1162 // If we need to preserve LCSSA, determine if any of the preds is a loop
1163 // exit.
1164 if (PreserveLCSSA)
1165 if (Loop *PL = LI->getLoopFor(BB: Pred))
1166 if (!PL->contains(BB: OldBB))
1167 HasLoopExit = true;
1168
1169 // If we need to preserve LoopInfo, note whether any of the preds crosses
1170 // an interesting loop boundary.
1171 if (!L)
1172 continue;
1173 if (L->contains(BB: Pred))
1174 IsLoopEntry = false;
1175 else
1176 SplitMakesNewLoopHeader = true;
1177 }
1178
1179 // Unless we have a loop for OldBB, nothing else to do here.
1180 if (!L)
1181 return;
1182
1183 if (IsLoopEntry) {
1184 // Add the new block to the nearest enclosing loop (and not an adjacent
1185 // loop). To find this, examine each of the predecessors and determine which
1186 // loops enclose them, and select the most-nested loop which contains the
1187 // loop containing the block being split.
1188 Loop *InnermostPredLoop = nullptr;
1189 for (BasicBlock *Pred : Preds) {
1190 if (Loop *PredLoop = LI->getLoopFor(BB: Pred)) {
1191 // Seek a loop which actually contains the block being split (to avoid
1192 // adjacent loops).
1193 while (PredLoop && !PredLoop->contains(BB: OldBB))
1194 PredLoop = PredLoop->getParentLoop();
1195
1196 // Select the most-nested of these loops which contains the block.
1197 if (PredLoop && PredLoop->contains(BB: OldBB) &&
1198 (!InnermostPredLoop ||
1199 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1200 InnermostPredLoop = PredLoop;
1201 }
1202 }
1203
1204 if (InnermostPredLoop)
1205 InnermostPredLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1206 } else {
1207 L->addBasicBlockToLoop(NewBB, LI&: *LI);
1208 if (SplitMakesNewLoopHeader) {
1209 // The old header might still have a loop entry. If so the loop becomes
1210 // irreducible and must be erased. Otherwise the NewBB becomes the loop
1211 // header.
1212 if (hasReachableLoopEntryToHeader(L: *L, DT: *DT))
1213 LI->erase(L);
1214 else
1215 L->moveToHeader(BB: NewBB);
1216 }
1217 }
1218}
1219
1220BasicBlock *llvm::splitBlockBefore(BasicBlock *Old,
1221 BasicBlock::iterator SplitPt,
1222 DomTreeUpdater *DTU, LoopInfo *LI,
1223 MemorySSAUpdater *MSSAU,
1224 const Twine &BBName) {
1225 BasicBlock::iterator SplitIt = SplitPt;
1226 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad())
1227 ++SplitIt;
1228 SmallVector<BasicBlock *, 4> Preds(predecessors(BB: Old));
1229 BasicBlock *New = Old->splitBasicBlockBefore(
1230 I: SplitIt, BBName: BBName.isTriviallyEmpty() ? Old->getName() + ".split" : BBName);
1231
1232 bool HasLoopExit = false;
1233 UpdateAnalysisInformation(OldBB: Old, NewBB: New, Preds, DTU, DT: nullptr, LI, MSSAU, PreserveLCSSA: false,
1234 HasLoopExit);
1235
1236 return New;
1237}
1238
1239/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1240/// This also updates AliasAnalysis, if available.
1241static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1242 ArrayRef<BasicBlock *> Preds, Instruction *BI,
1243 bool HasLoopExit) {
1244 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1245 SmallPtrSet<BasicBlock *, 16> PredSet(llvm::from_range, Preds);
1246 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(Val: I); ) {
1247 PHINode *PN = cast<PHINode>(Val: I++);
1248
1249 // Check to see if all of the values coming in are the same. If so, we
1250 // don't need to create a new PHI node, unless it's needed for LCSSA.
1251 Value *InVal = nullptr;
1252 if (!HasLoopExit) {
1253 InVal = PN->getIncomingValueForBlock(BB: Preds[0]);
1254 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1255 if (!PredSet.count(Ptr: PN->getIncomingBlock(i)))
1256 continue;
1257 if (!InVal)
1258 InVal = PN->getIncomingValue(i);
1259 else if (InVal != PN->getIncomingValue(i)) {
1260 InVal = nullptr;
1261 break;
1262 }
1263 }
1264 }
1265
1266 if (InVal) {
1267 // If all incoming values for the new PHI would be the same, just don't
1268 // make a new PHI. Instead, just remove the incoming values from the old
1269 // PHI.
1270 PN->removeIncomingValueIf(
1271 Predicate: [&](unsigned Idx) {
1272 return PredSet.contains(Ptr: PN->getIncomingBlock(i: Idx));
1273 },
1274 /* DeletePHIIfEmpty */ false);
1275
1276 // Add an incoming value to the PHI node in the loop for the preheader
1277 // edge.
1278 PN->addIncoming(V: InVal, BB: NewBB);
1279 continue;
1280 }
1281
1282 // If the values coming into the block are not the same, we need a new
1283 // PHI.
1284 // Create the new PHI node, insert it into NewBB at the end of the block
1285 PHINode *NewPHI =
1286 PHINode::Create(Ty: PN->getType(), NumReservedValues: Preds.size(), NameStr: PN->getName() + ".ph", InsertBefore: BI->getIterator());
1287
1288 // NOTE! This loop walks backwards for a reason! First off, this minimizes
1289 // the cost of removal if we end up removing a large number of values, and
1290 // second off, this ensures that the indices for the incoming values aren't
1291 // invalidated when we remove one.
1292 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1293 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1294 if (PredSet.count(Ptr: IncomingBB)) {
1295 Value *V = PN->removeIncomingValue(Idx: i, DeletePHIIfEmpty: false);
1296 NewPHI->addIncoming(V, BB: IncomingBB);
1297 }
1298 }
1299
1300 PN->addIncoming(V: NewPHI, BB: NewBB);
1301 }
1302}
1303
1304static void SplitLandingPadPredecessorsImpl(
1305 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1306 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1307 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1308 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1309
1310static BasicBlock *
1311SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
1312 const char *Suffix, DomTreeUpdater *DTU,
1313 DominatorTree *DT, LoopInfo *LI,
1314 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1315 // Do not attempt to split that which cannot be split.
1316 if (!BB->canSplitPredecessors())
1317 return nullptr;
1318
1319 // For the landingpads we need to act a bit differently.
1320 // Delegate this work to the SplitLandingPadPredecessors.
1321 if (BB->isLandingPad()) {
1322 SmallVector<BasicBlock*, 2> NewBBs;
1323 std::string NewName = std::string(Suffix) + ".split-lp";
1324
1325 SplitLandingPadPredecessorsImpl(OrigBB: BB, Preds, Suffix1: Suffix, Suffix2: NewName.c_str(), NewBBs,
1326 DTU, DT, LI, MSSAU, PreserveLCSSA);
1327 return NewBBs[0];
1328 }
1329
1330 // Create new basic block, insert right before the original block.
1331 BasicBlock *NewBB = BasicBlock::Create(
1332 Context&: BB->getContext(), Name: BB->getName() + Suffix, Parent: BB->getParent(), InsertBefore: BB);
1333
1334 // The new block unconditionally branches to the old block.
1335 UncondBrInst *BI = UncondBrInst::Create(Target: BB, InsertBefore: NewBB);
1336
1337 Loop *L = nullptr;
1338 BasicBlock *OldLatch = nullptr;
1339 // Splitting the predecessors of a loop header creates a preheader block.
1340 if (LI && LI->isLoopHeader(BB)) {
1341 L = LI->getLoopFor(BB);
1342 // Using the loop start line number prevents debuggers stepping into the
1343 // loop body for this instruction.
1344 BI->setDebugLoc(L->getStartLoc());
1345
1346 // If BB is the header of the Loop, it is possible that the loop is
1347 // modified, such that the current latch does not remain the latch of the
1348 // loop. If that is the case, the loop metadata from the current latch needs
1349 // to be applied to the new latch.
1350 OldLatch = L->getLoopLatch();
1351 } else
1352 BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
1353
1354 // Move the edges from Preds to point to NewBB instead of BB.
1355 for (BasicBlock *Pred : Preds) {
1356 // This is slightly more strict than necessary; the minimum requirement
1357 // is that there be no more than one indirectbr branching to BB. And
1358 // all BlockAddress uses would need to be updated.
1359 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1360 "Cannot split an edge from an IndirectBrInst");
1361 Pred->getTerminator()->replaceSuccessorWith(OldBB: BB, NewBB);
1362 }
1363
1364 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1365 // node becomes an incoming value for BB's phi node. However, if the Preds
1366 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1367 // account for the newly created predecessor.
1368 if (Preds.empty()) {
1369 // Insert dummy values as the incoming value.
1370 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(Val: I); ++I)
1371 cast<PHINode>(Val&: I)->addIncoming(V: PoisonValue::get(T: I->getType()), BB: NewBB);
1372 }
1373
1374 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1375 bool HasLoopExit = false;
1376 UpdateAnalysisInformation(OldBB: BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1377 HasLoopExit);
1378
1379 if (!Preds.empty()) {
1380 // Update the PHI nodes in BB with the values coming from NewBB.
1381 UpdatePHINodes(OrigBB: BB, NewBB, Preds, BI, HasLoopExit);
1382 }
1383
1384 if (OldLatch) {
1385 BasicBlock *NewLatch = L->getLoopLatch();
1386 if (NewLatch != OldLatch) {
1387 MDNode *MD = OldLatch->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
1388 NewLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: MD);
1389 // It's still possible that OldLatch is the latch of another inner loop,
1390 // in which case we do not remove the metadata.
1391 Loop *IL = LI->getLoopFor(BB: OldLatch);
1392 if (IL && IL->getLoopLatch() != OldLatch)
1393 OldLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: nullptr);
1394 }
1395 }
1396
1397 return NewBB;
1398}
1399
1400BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1401 ArrayRef<BasicBlock *> Preds,
1402 const char *Suffix, DominatorTree *DT,
1403 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1404 bool PreserveLCSSA) {
1405 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1406 MSSAU, PreserveLCSSA);
1407}
1408BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1409 ArrayRef<BasicBlock *> Preds,
1410 const char *Suffix,
1411 DomTreeUpdater *DTU, LoopInfo *LI,
1412 MemorySSAUpdater *MSSAU,
1413 bool PreserveLCSSA) {
1414 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1415 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1416}
1417
1418static void SplitLandingPadPredecessorsImpl(
1419 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1420 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1421 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1422 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1423 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1424
1425 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1426 // it right before the original block.
1427 BasicBlock *NewBB1 = BasicBlock::Create(Context&: OrigBB->getContext(),
1428 Name: OrigBB->getName() + Suffix1,
1429 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1430 NewBBs.push_back(Elt: NewBB1);
1431
1432 // The new block unconditionally branches to the old block.
1433 UncondBrInst *BI1 = UncondBrInst::Create(Target: OrigBB, InsertBefore: NewBB1);
1434 BI1->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1435
1436 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1437 for (BasicBlock *Pred : Preds) {
1438 // This is slightly more strict than necessary; the minimum requirement
1439 // is that there be no more than one indirectbr branching to BB. And
1440 // all BlockAddress uses would need to be updated.
1441 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1442 "Cannot split an edge from an IndirectBrInst");
1443 Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB1);
1444 }
1445
1446 bool HasLoopExit = false;
1447 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB1, Preds, DTU, DT, LI, MSSAU,
1448 PreserveLCSSA, HasLoopExit);
1449
1450 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1451 UpdatePHINodes(OrigBB, NewBB: NewBB1, Preds, BI: BI1, HasLoopExit);
1452
1453 // Move the remaining edges from OrigBB to point to NewBB2.
1454 SmallVector<BasicBlock*, 8> NewBB2Preds;
1455 for (pred_iterator i = pred_begin(BB: OrigBB), e = pred_end(BB: OrigBB);
1456 i != e; ) {
1457 BasicBlock *Pred = *i++;
1458 if (Pred == NewBB1) continue;
1459 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1460 "Cannot split an edge from an IndirectBrInst");
1461 NewBB2Preds.push_back(Elt: Pred);
1462 e = pred_end(BB: OrigBB);
1463 }
1464
1465 BasicBlock *NewBB2 = nullptr;
1466 if (!NewBB2Preds.empty()) {
1467 // Create another basic block for the rest of OrigBB's predecessors.
1468 NewBB2 = BasicBlock::Create(Context&: OrigBB->getContext(),
1469 Name: OrigBB->getName() + Suffix2,
1470 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1471 NewBBs.push_back(Elt: NewBB2);
1472
1473 // The new block unconditionally branches to the old block.
1474 UncondBrInst *BI2 = UncondBrInst::Create(Target: OrigBB, InsertBefore: NewBB2);
1475 BI2->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1476
1477 // Move the remaining edges from OrigBB to point to NewBB2.
1478 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1479 NewBB2Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB2);
1480
1481 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1482 HasLoopExit = false;
1483 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, DTU, DT, LI, MSSAU,
1484 PreserveLCSSA, HasLoopExit);
1485
1486 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1487 UpdatePHINodes(OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, BI: BI2, HasLoopExit);
1488 }
1489
1490 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1491 Instruction *Clone1 = LPad->clone();
1492 Clone1->setName(Twine("lpad") + Suffix1);
1493 Clone1->insertInto(ParentBB: NewBB1, It: NewBB1->getFirstInsertionPt());
1494
1495 if (NewBB2) {
1496 Instruction *Clone2 = LPad->clone();
1497 Clone2->setName(Twine("lpad") + Suffix2);
1498 Clone2->insertInto(ParentBB: NewBB2, It: NewBB2->getFirstInsertionPt());
1499
1500 // Create a PHI node for the two cloned landingpad instructions only
1501 // if the original landingpad instruction has some uses.
1502 if (!LPad->use_empty()) {
1503 assert(!LPad->getType()->isTokenTy() &&
1504 "Split cannot be applied if LPad is token type. Otherwise an "
1505 "invalid PHINode of token type would be created.");
1506 PHINode *PN = PHINode::Create(Ty: LPad->getType(), NumReservedValues: 2, NameStr: "lpad.phi", InsertBefore: LPad->getIterator());
1507 PN->addIncoming(V: Clone1, BB: NewBB1);
1508 PN->addIncoming(V: Clone2, BB: NewBB2);
1509 LPad->replaceAllUsesWith(V: PN);
1510 }
1511 LPad->eraseFromParent();
1512 } else {
1513 // There is no second clone. Just replace the landing pad with the first
1514 // clone.
1515 LPad->replaceAllUsesWith(V: Clone1);
1516 LPad->eraseFromParent();
1517 }
1518}
1519
1520void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1521 ArrayRef<BasicBlock *> Preds,
1522 const char *Suffix1, const char *Suffix2,
1523 SmallVectorImpl<BasicBlock *> &NewBBs,
1524 DomTreeUpdater *DTU, LoopInfo *LI,
1525 MemorySSAUpdater *MSSAU,
1526 bool PreserveLCSSA) {
1527 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1528 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1529 PreserveLCSSA);
1530}
1531
1532ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1533 BasicBlock *Pred,
1534 DomTreeUpdater *DTU) {
1535 Instruction *UncondBranch = Pred->getTerminator();
1536 // Clone the return and add it to the end of the predecessor.
1537 Instruction *NewRet = RI->clone();
1538 NewRet->insertInto(ParentBB: Pred, It: Pred->end());
1539
1540 // If the return instruction returns a value, and if the value was a
1541 // PHI node in "BB", propagate the right value into the return.
1542 for (Use &Op : NewRet->operands()) {
1543 Value *V = Op;
1544 Instruction *NewBC = nullptr;
1545 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Val: V)) {
1546 // Return value might be bitcasted. Clone and insert it before the
1547 // return instruction.
1548 V = BCI->getOperand(i_nocapture: 0);
1549 NewBC = BCI->clone();
1550 NewBC->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1551 Op = NewBC;
1552 }
1553
1554 Instruction *NewEV = nullptr;
1555 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: V)) {
1556 V = EVI->getOperand(i_nocapture: 0);
1557 NewEV = EVI->clone();
1558 if (NewBC) {
1559 NewBC->setOperand(i: 0, Val: NewEV);
1560 NewEV->insertInto(ParentBB: Pred, It: NewBC->getIterator());
1561 } else {
1562 NewEV->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1563 Op = NewEV;
1564 }
1565 }
1566
1567 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
1568 if (PN->getParent() == BB) {
1569 if (NewEV) {
1570 NewEV->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1571 } else if (NewBC)
1572 NewBC->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1573 else
1574 Op = PN->getIncomingValueForBlock(BB: Pred);
1575 }
1576 }
1577 }
1578
1579 // Update any PHI nodes in the returning block to realize that we no
1580 // longer branch to them.
1581 BB->removePredecessor(Pred);
1582 UncondBranch->eraseFromParent();
1583
1584 if (DTU)
1585 DTU->applyUpdates(Updates: {{DominatorTree::Delete, Pred, BB}});
1586
1587 return cast<ReturnInst>(Val: NewRet);
1588}
1589
1590Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1591 BasicBlock::iterator SplitBefore,
1592 bool Unreachable,
1593 MDNode *BranchWeights,
1594 DomTreeUpdater *DTU, LoopInfo *LI,
1595 BasicBlock *ThenBlock) {
1596 SplitBlockAndInsertIfThenElse(
1597 Cond, SplitBefore, ThenBlock: &ThenBlock, /* ElseBlock */ nullptr,
1598 /* UnreachableThen */ Unreachable,
1599 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1600 return ThenBlock->getTerminator();
1601}
1602
1603Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
1604 BasicBlock::iterator SplitBefore,
1605 bool Unreachable,
1606 MDNode *BranchWeights,
1607 DomTreeUpdater *DTU, LoopInfo *LI,
1608 BasicBlock *ElseBlock) {
1609 SplitBlockAndInsertIfThenElse(
1610 Cond, SplitBefore, /* ThenBlock */ nullptr, ElseBlock: &ElseBlock,
1611 /* UnreachableThen */ false,
1612 /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1613 return ElseBlock->getTerminator();
1614}
1615
1616void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore,
1617 Instruction **ThenTerm,
1618 Instruction **ElseTerm,
1619 MDNode *BranchWeights,
1620 DomTreeUpdater *DTU, LoopInfo *LI) {
1621 BasicBlock *ThenBlock = nullptr;
1622 BasicBlock *ElseBlock = nullptr;
1623 SplitBlockAndInsertIfThenElse(
1624 Cond, SplitBefore, ThenBlock: &ThenBlock, ElseBlock: &ElseBlock, /* UnreachableThen */ false,
1625 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1626
1627 *ThenTerm = ThenBlock->getTerminator();
1628 *ElseTerm = ElseBlock->getTerminator();
1629}
1630
1631void llvm::SplitBlockAndInsertIfThenElse(
1632 Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
1633 BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1634 MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1635 assert((ThenBlock || ElseBlock) &&
1636 "At least one branch block must be created");
1637 assert((!UnreachableThen || !UnreachableElse) &&
1638 "Split block tail must be reachable");
1639
1640 SmallVector<DominatorTree::UpdateType, 8> Updates;
1641 SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1642 BasicBlock *Head = SplitBefore->getParent();
1643 if (DTU) {
1644 UniqueOrigSuccessors.insert_range(R: successors(BB: Head));
1645 Updates.reserve(N: 4 + 2 * UniqueOrigSuccessors.size());
1646 }
1647
1648 LLVMContext &C = Head->getContext();
1649 BasicBlock *Tail = Head->splitBasicBlock(I: SplitBefore);
1650 BasicBlock *TrueBlock = Tail;
1651 BasicBlock *FalseBlock = Tail;
1652 bool ThenToTailEdge = false;
1653 bool ElseToTailEdge = false;
1654
1655 // Encapsulate the logic around creation/insertion/etc of a new block.
1656 auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1657 bool &ToTailEdge) {
1658 if (PBB == nullptr)
1659 return; // Do not create/insert a block.
1660
1661 if (*PBB)
1662 BB = *PBB; // Caller supplied block, use it.
1663 else {
1664 // Create a new block.
1665 BB = BasicBlock::Create(Context&: C, Name: "", Parent: Head->getParent(), InsertBefore: Tail);
1666 if (Unreachable)
1667 (void)new UnreachableInst(C, BB);
1668 else {
1669 (void)UncondBrInst::Create(Target: Tail, InsertBefore: BB);
1670 ToTailEdge = true;
1671 }
1672 BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1673 // Pass the new block back to the caller.
1674 *PBB = BB;
1675 }
1676 };
1677
1678 handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1679 handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1680
1681 Instruction *HeadOldTerm = Head->getTerminator();
1682 CondBrInst *HeadNewTerm = CondBrInst::Create(Cond, IfTrue: TrueBlock, IfFalse: FalseBlock);
1683 HeadNewTerm->setMetadata(KindID: LLVMContext::MD_prof, Node: BranchWeights);
1684 ReplaceInstWithInst(From: HeadOldTerm, To: HeadNewTerm);
1685
1686 if (DTU) {
1687 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: TrueBlock);
1688 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: FalseBlock);
1689 if (ThenToTailEdge)
1690 Updates.emplace_back(Args: DominatorTree::Insert, Args&: TrueBlock, Args&: Tail);
1691 if (ElseToTailEdge)
1692 Updates.emplace_back(Args: DominatorTree::Insert, Args&: FalseBlock, Args&: Tail);
1693 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1694 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Tail, Args&: UniqueOrigSuccessor);
1695 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1696 Updates.emplace_back(Args: DominatorTree::Delete, Args&: Head, Args&: UniqueOrigSuccessor);
1697 DTU->applyUpdates(Updates);
1698 }
1699
1700 if (LI) {
1701 if (Loop *L = LI->getLoopFor(BB: Head); L) {
1702 if (ThenToTailEdge)
1703 L->addBasicBlockToLoop(NewBB: TrueBlock, LI&: *LI);
1704 if (ElseToTailEdge)
1705 L->addBasicBlockToLoop(NewBB: FalseBlock, LI&: *LI);
1706 L->addBasicBlockToLoop(NewBB: Tail, LI&: *LI);
1707 }
1708 }
1709}
1710
1711std::pair<Instruction *, Value *>
1712llvm::SplitBlockAndInsertSimpleForLoop(Value *End,
1713 BasicBlock::iterator SplitBefore) {
1714 BasicBlock *LoopPred = SplitBefore->getParent();
1715 BasicBlock *LoopBody = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1716 BasicBlock *LoopExit = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1717
1718 auto *Ty = End->getType();
1719 auto &DL = SplitBefore->getDataLayout();
1720 const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1721
1722 IRBuilder<> Builder(LoopBody->getTerminator());
1723 auto *IV = Builder.CreatePHI(Ty, NumReservedValues: 2, Name: "iv");
1724 auto *IVNext =
1725 Builder.CreateAdd(LHS: IV, RHS: ConstantInt::get(Ty, V: 1), Name: IV->getName() + ".next",
1726 /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1727 auto *IVCheck = Builder.CreateICmpEQ(LHS: IVNext, RHS: End,
1728 Name: IV->getName() + ".check");
1729 Builder.CreateCondBr(Cond: IVCheck, True: LoopExit, False: LoopBody);
1730 LoopBody->getTerminator()->eraseFromParent();
1731
1732 // Populate the IV PHI.
1733 IV->addIncoming(V: ConstantInt::get(Ty, V: 0), BB: LoopPred);
1734 IV->addIncoming(V: IVNext, BB: LoopBody);
1735
1736 return std::make_pair(x: &*LoopBody->getFirstNonPHIIt(), y&: IV);
1737}
1738
1739void llvm::SplitBlockAndInsertForEachLane(
1740 ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
1741 std::function<void(IRBuilderBase &, Value *)> Func) {
1742
1743 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1744
1745 if (EC.isScalable()) {
1746 Value *NumElements = IRB.CreateElementCount(Ty: IndexTy, EC);
1747
1748 auto [BodyIP, Index] =
1749 SplitBlockAndInsertSimpleForLoop(End: NumElements, SplitBefore: InsertBefore);
1750
1751 IRB.SetInsertPoint(BodyIP);
1752 Func(IRB, Index);
1753 return;
1754 }
1755
1756 unsigned Num = EC.getFixedValue();
1757 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1758 IRB.SetInsertPoint(InsertBefore);
1759 Func(IRB, ConstantInt::get(Ty: IndexTy, V: Idx));
1760 }
1761}
1762
1763void llvm::SplitBlockAndInsertForEachLane(
1764 Value *EVL, BasicBlock::iterator InsertBefore,
1765 std::function<void(IRBuilderBase &, Value *)> Func) {
1766
1767 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1768 Type *Ty = EVL->getType();
1769
1770 if (!isa<ConstantInt>(Val: EVL)) {
1771 auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(End: EVL, SplitBefore: InsertBefore);
1772 IRB.SetInsertPoint(BodyIP);
1773 Func(IRB, Index);
1774 return;
1775 }
1776
1777 unsigned Num = cast<ConstantInt>(Val: EVL)->getZExtValue();
1778 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1779 IRB.SetInsertPoint(InsertBefore);
1780 Func(IRB, ConstantInt::get(Ty, V: Idx));
1781 }
1782}
1783
1784CondBrInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1785 BasicBlock *&IfFalse) {
1786 PHINode *SomePHI = dyn_cast<PHINode>(Val: BB->begin());
1787 BasicBlock *Pred1 = nullptr;
1788 BasicBlock *Pred2 = nullptr;
1789
1790 if (SomePHI) {
1791 if (SomePHI->getNumIncomingValues() != 2)
1792 return nullptr;
1793 Pred1 = SomePHI->getIncomingBlock(i: 0);
1794 Pred2 = SomePHI->getIncomingBlock(i: 1);
1795 } else {
1796 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1797 if (PI == PE) // No predecessor
1798 return nullptr;
1799 Pred1 = *PI++;
1800 if (PI == PE) // Only one predecessor
1801 return nullptr;
1802 Pred2 = *PI++;
1803 if (PI != PE) // More than two predecessors
1804 return nullptr;
1805 }
1806
1807 Instruction *Pred1Term = Pred1->getTerminator();
1808 Instruction *Pred2Term = Pred2->getTerminator();
1809
1810 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1811 // either are.
1812 if (isa<CondBrInst>(Val: Pred2Term)) {
1813 // If both branches are conditional, we don't have an "if statement". In
1814 // reality, we could transform this case, but since the condition will be
1815 // required anyway, we stand no chance of eliminating it, so the xform is
1816 // probably not profitable.
1817 if (isa<CondBrInst>(Val: Pred1Term))
1818 return nullptr;
1819
1820 std::swap(a&: Pred1, b&: Pred2);
1821 std::swap(a&: Pred1Term, b&: Pred2Term);
1822 }
1823
1824 // We can only handle branches. Other control flow will be lowered to
1825 // branches if possible anyway.
1826 if (!isa<UncondBrInst>(Val: Pred2Term))
1827 return nullptr;
1828
1829 if (auto *Pred1Br = dyn_cast<CondBrInst>(Val: Pred1Term)) {
1830 // The only thing we have to watch out for here is to make sure that Pred2
1831 // doesn't have incoming edges from other blocks. If it does, the condition
1832 // doesn't dominate BB.
1833 if (!Pred2->getSinglePredecessor())
1834 return nullptr;
1835
1836 // If we found a conditional branch predecessor, make sure that it branches
1837 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1838 if (Pred1Br->getSuccessor(i: 0) == BB &&
1839 Pred1Br->getSuccessor(i: 1) == Pred2) {
1840 IfTrue = Pred1;
1841 IfFalse = Pred2;
1842 } else if (Pred1Br->getSuccessor(i: 0) == Pred2 &&
1843 Pred1Br->getSuccessor(i: 1) == BB) {
1844 IfTrue = Pred2;
1845 IfFalse = Pred1;
1846 } else {
1847 // We know that one arm of the conditional goes to BB, so the other must
1848 // go somewhere unrelated, and this must not be an "if statement".
1849 return nullptr;
1850 }
1851
1852 return Pred1Br;
1853 }
1854
1855 if (!isa<UncondBrInst>(Val: Pred1Term))
1856 return nullptr;
1857
1858 // Ok, if we got here, both predecessors end with an unconditional branch to
1859 // BB. Don't panic! If both blocks only have a single (identical)
1860 // predecessor, and THAT is a conditional branch, then we're all ok!
1861 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1862 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1863 return nullptr;
1864
1865 // Otherwise, if this is a conditional branch, then we can use it!
1866 CondBrInst *BI = dyn_cast<CondBrInst>(Val: CommonPred->getTerminator());
1867 if (!BI) return nullptr;
1868
1869 if (BI->getSuccessor(i: 0) == Pred1) {
1870 IfTrue = Pred1;
1871 IfFalse = Pred2;
1872 } else {
1873 IfTrue = Pred2;
1874 IfFalse = Pred1;
1875 }
1876 return BI;
1877}
1878
1879void llvm::InvertBranch(CondBrInst *PBI, IRBuilderBase &Builder) {
1880 Value *NewCond = PBI->getCondition();
1881 // If this is a "cmp" instruction, only used for branching (and nowhere
1882 // else), then we can simply invert the predicate.
1883 if (NewCond->hasOneUse() && isa<CmpInst>(Val: NewCond)) {
1884 CmpInst *CI = cast<CmpInst>(Val: NewCond);
1885 CI->setPredicate(CI->getInversePredicate());
1886 } else
1887 NewCond = Builder.CreateNot(V: NewCond, Name: NewCond->getName() + ".not");
1888
1889 PBI->setCondition(NewCond);
1890 PBI->swapSuccessors();
1891}
1892
1893bool llvm::hasOnlySimpleTerminator(const Function &F) {
1894 for (auto &BB : F) {
1895 auto *Term = BB.getTerminator();
1896 if (!isa<ReturnInst, UnreachableInst, UncondBrInst, CondBrInst>(Val: Term))
1897 return false;
1898 }
1899 return true;
1900}
1901
1902Printable llvm::printBasicBlock(const BasicBlock *BB) {
1903 return Printable([BB](raw_ostream &OS) {
1904 if (!BB) {
1905 OS << "<nullptr>";
1906 return;
1907 }
1908 BB->printAsOperand(O&: OS);
1909 });
1910}
1911