1//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions perform manipulations on basic blocks, and
10// instructions contained within basic blocks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/BasicBlockUtils.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallPtrSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/CFG.h"
20#include "llvm/Analysis/DomTreeUpdater.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/Analysis/MemoryDependenceAnalysis.h"
23#include "llvm/Analysis/MemorySSAUpdater.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/CFG.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/CycleInfo.h"
28#include "llvm/IR/DebugInfo.h"
29#include "llvm/IR/DebugInfoMetadata.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/InstrTypes.h"
34#include "llvm/IR/Instruction.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/LLVMContext.h"
37#include "llvm/IR/Type.h"
38#include "llvm/IR/User.h"
39#include "llvm/IR/Value.h"
40#include "llvm/IR/ValueHandle.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/Utils/Local.h"
46#include <cassert>
47#include <cstdint>
48#include <string>
49#include <utility>
50#include <vector>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "basicblock-utils"
55
56static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
57 "max-deopt-or-unreachable-succ-check-depth", cl::init(Val: 8), cl::Hidden,
58 cl::desc("Set the maximum path length when checking whether a basic block "
59 "is followed by a block that either has a terminating "
60 "deoptimizing call or is terminated with an unreachable"));
61
62/// Zap all the instructions in the block and replace them with an unreachable
63/// instruction and notify the basic block's successors that one of their
64/// predecessors is going away.
65static void
66emptyAndDetachBlock(BasicBlock *BB,
67 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
68 bool KeepOneInputPHIs) {
69 // Loop through all of our successors and make sure they know that one
70 // of their predecessors is going away.
71 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
72 for (BasicBlock *Succ : successors(BB)) {
73 Succ->removePredecessor(Pred: BB, KeepOneInputPHIs);
74 if (Updates && UniqueSuccessors.insert(Ptr: Succ).second)
75 Updates->push_back(Elt: {DominatorTree::Delete, BB, Succ});
76 }
77
78 // Zap all the instructions in the block.
79 while (!BB->empty()) {
80 Instruction &I = BB->back();
81 // If this instruction is used, replace uses with an arbitrary value.
82 // Because control flow can't get here, we don't care what we replace the
83 // value with. Note that since this block is unreachable, and all values
84 // contained within it must dominate their uses, that all uses will
85 // eventually be removed (they are themselves dead).
86 if (!I.use_empty())
87 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
88 BB->back().eraseFromParent();
89 }
90 new UnreachableInst(BB->getContext(), BB);
91 assert(BB->size() == 1 && isa<UnreachableInst>(BB->getTerminator()) &&
92 "The successor list of BB isn't empty before "
93 "applying corresponding DTU updates.");
94}
95
96bool llvm::HasLoopOrEntryConvergenceToken(const BasicBlock *BB) {
97 for (const Instruction &I : *BB) {
98 const ConvergenceControlInst *CCI = dyn_cast<ConvergenceControlInst>(Val: &I);
99 if (CCI && (CCI->isLoop() || CCI->isEntry()))
100 return true;
101 }
102 return false;
103}
104
105void llvm::detachDeadBlocks(ArrayRef<BasicBlock *> BBs,
106 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
107 bool KeepOneInputPHIs) {
108 SmallPtrSet<BasicBlock *, 4> UniqueEHRetBlocksToDelete;
109 for (auto *BB : BBs) {
110 auto NonFirstPhiIt = BB->getFirstNonPHIIt();
111 if (NonFirstPhiIt != BB->end()) {
112 Instruction &I = *NonFirstPhiIt;
113 // Exception handling funclets need to be explicitly addressed.
114 // These funclets must begin with cleanuppad or catchpad and end with
115 // cleanupred or catchret. The return instructions can be in different
116 // basic blocks than the pad instruction. If we would only delete the
117 // first block, the we would have possible cleanupret and catchret
118 // instructions with poison arguments, which wouldn't be valid.
119 if (isa<FuncletPadInst>(Val: I)) {
120 UniqueEHRetBlocksToDelete.clear();
121
122 for (User *User : I.users()) {
123 Instruction *ReturnInstr = dyn_cast<Instruction>(Val: User);
124 // If we have a cleanupret or catchret block, replace it with just an
125 // unreachable. The other alternative, that may use a catchpad is a
126 // catchswitch. That does not need special handling for now.
127 if (isa<CatchReturnInst>(Val: ReturnInstr) ||
128 isa<CleanupReturnInst>(Val: ReturnInstr)) {
129 BasicBlock *ReturnInstrBB = ReturnInstr->getParent();
130 UniqueEHRetBlocksToDelete.insert(Ptr: ReturnInstrBB);
131 }
132 }
133
134 for (BasicBlock *EHRetBB : UniqueEHRetBlocksToDelete)
135 emptyAndDetachBlock(BB: EHRetBB, Updates, KeepOneInputPHIs);
136 }
137 }
138
139 UniqueEHRetBlocksToDelete.clear();
140
141 // Detaching and emptying the current basic block.
142 emptyAndDetachBlock(BB, Updates, KeepOneInputPHIs);
143 }
144}
145
146void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
147 bool KeepOneInputPHIs) {
148 DeleteDeadBlocks(BBs: {BB}, DTU, KeepOneInputPHIs);
149}
150
151void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
152 bool KeepOneInputPHIs) {
153#ifndef NDEBUG
154 // Make sure that all predecessors of each dead block is also dead.
155 SmallPtrSet<BasicBlock *, 4> Dead(llvm::from_range, BBs);
156 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
157 for (auto *BB : Dead)
158 for (BasicBlock *Pred : predecessors(BB))
159 assert(Dead.count(Pred) && "All predecessors must be dead!");
160#endif
161
162 SmallVector<DominatorTree::UpdateType, 4> Updates;
163 detachDeadBlocks(BBs, Updates: DTU ? &Updates : nullptr, KeepOneInputPHIs);
164
165 if (DTU)
166 DTU->applyUpdates(Updates);
167
168 for (BasicBlock *BB : BBs)
169 if (DTU)
170 DTU->deleteBB(DelBB: BB);
171 else
172 BB->eraseFromParent();
173}
174
175bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
176 bool KeepOneInputPHIs) {
177 df_iterator_default_set<BasicBlock*> Reachable;
178
179 // Mark all reachable blocks.
180 for (BasicBlock *BB : depth_first_ext(G: &F, S&: Reachable))
181 (void)BB/* Mark all reachable blocks */;
182
183 // Collect all dead blocks.
184 std::vector<BasicBlock*> DeadBlocks;
185 for (BasicBlock &BB : F)
186 if (!Reachable.count(Ptr: &BB))
187 DeadBlocks.push_back(x: &BB);
188
189 // Delete the dead blocks.
190 DeleteDeadBlocks(BBs: DeadBlocks, DTU, KeepOneInputPHIs);
191
192 return !DeadBlocks.empty();
193}
194
195bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
196 MemoryDependenceResults *MemDep) {
197 if (!isa<PHINode>(Val: BB->begin()))
198 return false;
199
200 while (PHINode *PN = dyn_cast<PHINode>(Val: BB->begin())) {
201 if (PN->getIncomingValue(i: 0) != PN)
202 PN->replaceAllUsesWith(V: PN->getIncomingValue(i: 0));
203 else
204 PN->replaceAllUsesWith(V: PoisonValue::get(T: PN->getType()));
205
206 if (MemDep)
207 MemDep->removeInstruction(InstToRemove: PN); // Memdep updates AA itself.
208
209 PN->eraseFromParent();
210 }
211 return true;
212}
213
214bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
215 MemorySSAUpdater *MSSAU) {
216 // Recursively deleting a PHI may cause multiple PHIs to be deleted
217 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
218 SmallVector<WeakTrackingVH, 8> PHIs(llvm::make_pointer_range(Range: BB->phis()));
219
220 bool Changed = false;
221 for (const auto &PHI : PHIs)
222 if (PHINode *PN = dyn_cast_or_null<PHINode>(Val: PHI.operator Value *()))
223 Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
224
225 return Changed;
226}
227
228bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
229 LoopInfo *LI, MemorySSAUpdater *MSSAU,
230 MemoryDependenceResults *MemDep,
231 bool PredecessorWithTwoSuccessors,
232 DominatorTree *DT) {
233 if (BB->hasAddressTaken())
234 return false;
235
236 // Can't merge if there are multiple predecessors, or no predecessors.
237 BasicBlock *PredBB = BB->getUniquePredecessor();
238 if (!PredBB) return false;
239
240 // Don't break self-loops.
241 if (PredBB == BB) return false;
242
243 // Don't break unwinding instructions or terminators with other side-effects.
244 Instruction *PTI = PredBB->getTerminator();
245 if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
246 return false;
247
248 // Can't merge if there are multiple distinct successors.
249 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
250 return false;
251
252 // Currently only allow PredBB to have two predecessors, one being BB.
253 // Update BI to branch to BB's only successor instead of BB.
254 CondBrInst *PredBB_BI;
255 BasicBlock *NewSucc = nullptr;
256 unsigned FallThruPath;
257 if (PredecessorWithTwoSuccessors) {
258 if (!(PredBB_BI = dyn_cast<CondBrInst>(Val: PTI)))
259 return false;
260 UncondBrInst *BB_JmpI = dyn_cast<UncondBrInst>(Val: BB->getTerminator());
261 if (!BB_JmpI)
262 return false;
263 NewSucc = BB_JmpI->getSuccessor();
264 FallThruPath = PredBB_BI->getSuccessor(i: 0) == BB ? 0 : 1;
265 }
266
267 // Can't merge if there is PHI loop.
268 for (PHINode &PN : BB->phis())
269 if (llvm::is_contained(Range: PN.incoming_values(), Element: &PN))
270 return false;
271
272 // Don't break if both the basic block and the predecessor contain loop or
273 // entry convergent intrinsics, since there may only be one convergence token
274 // per block.
275 if (HasLoopOrEntryConvergenceToken(BB) &&
276 HasLoopOrEntryConvergenceToken(BB: PredBB))
277 return false;
278
279 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
280 << PredBB->getName() << "\n");
281
282 // Begin by getting rid of unneeded PHIs.
283 SmallVector<AssertingVH<Value>, 4> IncomingValues;
284 if (isa<PHINode>(Val: BB->front())) {
285 for (PHINode &PN : BB->phis())
286 if (!isa<PHINode>(Val: PN.getIncomingValue(i: 0)) ||
287 cast<PHINode>(Val: PN.getIncomingValue(i: 0))->getParent() != BB)
288 IncomingValues.push_back(Elt: PN.getIncomingValue(i: 0));
289 FoldSingleEntryPHINodes(BB, MemDep);
290 }
291
292 if (DT) {
293 assert(!DTU && "cannot use both DT and DTU for updates");
294 DomTreeNode *PredNode = DT->getNode(BB: PredBB);
295 DomTreeNode *BBNode = DT->getNode(BB);
296 if (PredNode) {
297 assert(BBNode && "PredNode unreachable but BBNode reachable?");
298 for (DomTreeNode *C : to_vector(Range: BBNode->children()))
299 C->setIDom(PredNode);
300 }
301 }
302 // DTU update: Collect all the edges that exit BB.
303 // These dominator edges will be redirected from Pred.
304 std::vector<DominatorTree::UpdateType> Updates;
305 if (DTU) {
306 assert(!DT && "cannot use both DT and DTU for updates");
307 // To avoid processing the same predecessor more than once.
308 SmallPtrSet<BasicBlock *, 8> SeenSuccs;
309 SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(llvm::from_range,
310 successors(BB: PredBB));
311 Updates.reserve(n: Updates.size() + 2 * succ_size(BB) + 1);
312 // Add insert edges first. Experimentally, for the particular case of two
313 // blocks that can be merged, with a single successor and single predecessor
314 // respectively, it is beneficial to have all insert updates first. Deleting
315 // edges first may lead to unreachable blocks, followed by inserting edges
316 // making the blocks reachable again. Such DT updates lead to high compile
317 // times. We add inserts before deletes here to reduce compile time.
318 for (BasicBlock *SuccOfBB : successors(BB))
319 // This successor of BB may already be a PredBB's successor.
320 if (!SuccsOfPredBB.contains(Ptr: SuccOfBB))
321 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
322 Updates.push_back(x: {DominatorTree::Insert, PredBB, SuccOfBB});
323 SeenSuccs.clear();
324 for (BasicBlock *SuccOfBB : successors(BB))
325 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
326 Updates.push_back(x: {DominatorTree::Delete, BB, SuccOfBB});
327 Updates.push_back(x: {DominatorTree::Delete, PredBB, BB});
328 }
329
330 Instruction *STI = BB->getTerminator();
331 Instruction *Start = &*BB->begin();
332 // If there's nothing to move, mark the starting instruction as the last
333 // instruction in the block. Terminator instruction is handled separately.
334 if (Start == STI)
335 Start = PTI;
336
337 // Move all definitions in the successor to the predecessor...
338 PredBB->splice(ToIt: PTI->getIterator(), FromBB: BB, FromBeginIt: BB->begin(), FromEndIt: STI->getIterator());
339
340 if (MSSAU)
341 MSSAU->moveAllAfterMergeBlocks(From: BB, To: PredBB, Start);
342
343 // Make all PHI nodes that referred to BB now refer to Pred as their
344 // source...
345 BB->replaceAllUsesWith(V: PredBB);
346
347 if (PredecessorWithTwoSuccessors) {
348 // Delete the unconditional branch from BB.
349 BB->back().eraseFromParent();
350 // Add unreachable to now empty BB.
351 new UnreachableInst(BB->getContext(), BB);
352
353 // Update branch in the predecessor.
354 PredBB_BI->setSuccessor(idx: FallThruPath, NewSucc);
355 } else {
356 // Delete the unconditional branch from the predecessor.
357 PredBB->back().eraseFromParent();
358
359 // Move terminator instruction.
360 BB->back().moveBeforePreserving(BB&: *PredBB, I: PredBB->end());
361 // Add unreachable to now empty BB.
362 new UnreachableInst(BB->getContext(), BB);
363
364 // Terminator may be a memory accessing instruction too.
365 if (MSSAU)
366 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
367 Val: MSSAU->getMemorySSA()->getMemoryAccess(I: PredBB->getTerminator())))
368 MSSAU->moveToPlace(What: MUD, BB: PredBB, Where: MemorySSA::End);
369 }
370
371 // Inherit predecessors name if it exists.
372 if (!PredBB->hasName())
373 PredBB->takeName(V: BB);
374
375 if (LI)
376 LI->removeBlock(BB);
377
378 if (MemDep)
379 MemDep->invalidateCachedPredecessors();
380
381 if (DTU)
382 DTU->applyUpdates(Updates);
383
384 if (DT) {
385 assert(succ_empty(BB) &&
386 "successors should have been transferred to PredBB");
387 DT->eraseNode(BB);
388 }
389
390 // Finally, erase the old block and update dominator info.
391 DeleteDeadBlock(BB, DTU);
392
393 return true;
394}
395
396bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
397 SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
398 LoopInfo *LI) {
399 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
400
401 bool BlocksHaveBeenMerged = false;
402 while (!MergeBlocks.empty()) {
403 BasicBlock *BB = *MergeBlocks.begin();
404 BasicBlock *Dest = BB->getSingleSuccessor();
405 if (Dest && (!L || L->contains(BB: Dest))) {
406 BasicBlock *Fold = Dest->getUniquePredecessor();
407 (void)Fold;
408 if (MergeBlockIntoPredecessor(BB: Dest, DTU, LI)) {
409 assert(Fold == BB &&
410 "Expecting BB to be unique predecessor of the Dest block");
411 MergeBlocks.erase(Ptr: Dest);
412 BlocksHaveBeenMerged = true;
413 } else
414 MergeBlocks.erase(Ptr: BB);
415 } else
416 MergeBlocks.erase(Ptr: BB);
417 }
418 return BlocksHaveBeenMerged;
419}
420
421/// Remove redundant instructions within sequences of consecutive dbg.value
422/// instructions. This is done using a backward scan to keep the last dbg.value
423/// describing a specific variable/fragment.
424///
425/// BackwardScan strategy:
426/// ----------------------
427/// Given a sequence of consecutive DbgValueInst like this
428///
429/// dbg.value ..., "x", FragmentX1 (*)
430/// dbg.value ..., "y", FragmentY1
431/// dbg.value ..., "x", FragmentX2
432/// dbg.value ..., "x", FragmentX1 (**)
433///
434/// then the instruction marked with (*) can be removed (it is guaranteed to be
435/// obsoleted by the instruction marked with (**) as the latter instruction is
436/// describing the same variable using the same fragment info).
437///
438/// Possible improvements:
439/// - Check fully overlapping fragments and not only identical fragments.
440static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
441 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
442 SmallDenseSet<DebugVariable> VariableSet;
443 for (auto &I : reverse(C&: *BB)) {
444 for (DbgVariableRecord &DVR :
445 reverse(C: filterDbgVars(R: I.getDbgRecordRange()))) {
446 DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
447 DVR.getDebugLoc()->getInlinedAt());
448 auto R = VariableSet.insert(V: Key);
449 // If the same variable fragment is described more than once it is enough
450 // to keep the last one (i.e. the first found since we for reverse
451 // iteration).
452 if (R.second)
453 continue;
454
455 if (DVR.isDbgAssign()) {
456 // Don't delete dbg.assign intrinsics that are linked to instructions.
457 if (!at::getAssignmentInsts(DVR: &DVR).empty())
458 continue;
459 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
460 }
461
462 ToBeRemoved.push_back(Elt: &DVR);
463 }
464 // Sequence with consecutive dbg.value instrs ended. Clear the map to
465 // restart identifying redundant instructions if case we find another
466 // dbg.value sequence.
467 VariableSet.clear();
468 }
469
470 for (auto &DVR : ToBeRemoved)
471 DVR->eraseFromParent();
472
473 return !ToBeRemoved.empty();
474}
475
476/// Remove redundant dbg.value instructions using a forward scan. This can
477/// remove a dbg.value instruction that is redundant due to indicating that a
478/// variable has the same value as already being indicated by an earlier
479/// dbg.value.
480///
481/// ForwardScan strategy:
482/// ---------------------
483/// Given two identical dbg.value instructions, separated by a block of
484/// instructions that isn't describing the same variable, like this
485///
486/// dbg.value X1, "x", FragmentX1 (**)
487/// <block of instructions, none being "dbg.value ..., "x", ...">
488/// dbg.value X1, "x", FragmentX1 (*)
489///
490/// then the instruction marked with (*) can be removed. Variable "x" is already
491/// described as being mapped to the SSA value X1.
492///
493/// Possible improvements:
494/// - Keep track of non-overlapping fragments.
495static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
496 bool RemovedAny = false;
497 SmallDenseMap<DebugVariable,
498 std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
499 VariableMap;
500 for (auto &I : *BB) {
501 for (DbgVariableRecord &DVR :
502 make_early_inc_range(Range: filterDbgVars(R: I.getDbgRecordRange()))) {
503 if (DVR.getType() == DbgVariableRecord::LocationType::Declare)
504 continue;
505 DebugVariable Key(DVR.getVariable(), std::nullopt,
506 DVR.getDebugLoc()->getInlinedAt());
507 auto [VMI, Inserted] = VariableMap.try_emplace(Key);
508 // A dbg.assign with no linked instructions can be treated like a
509 // dbg.value (i.e. can be deleted).
510 bool IsDbgValueKind =
511 (!DVR.isDbgAssign() || at::getAssignmentInsts(DVR: &DVR).empty());
512
513 // Update the map if we found a new value/expression describing the
514 // variable, or if the variable wasn't mapped already.
515 SmallVector<Value *, 4> Values(DVR.location_ops());
516 if (Inserted || VMI->second.first != Values ||
517 VMI->second.second != DVR.getExpression()) {
518 if (IsDbgValueKind)
519 VMI->second = {Values, DVR.getExpression()};
520 else
521 VMI->second = {Values, nullptr};
522 continue;
523 }
524 // Don't delete dbg.assign intrinsics that are linked to instructions.
525 if (!IsDbgValueKind)
526 continue;
527 // Found an identical mapping. Remember the instruction for later removal.
528 DVR.eraseFromParent();
529 RemovedAny = true;
530 }
531 }
532
533 return RemovedAny;
534}
535
536/// Remove redundant undef dbg.assign intrinsic from an entry block using a
537/// forward scan.
538/// Strategy:
539/// ---------------------
540/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
541/// linked to an intrinsic, and don't share an aggregate variable with a debug
542/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
543/// that come before non-undef debug intrinsics for the variable are
544/// deleted. Given:
545///
546/// dbg.assign undef, "x", FragmentX1 (*)
547/// <block of instructions, none being "dbg.value ..., "x", ...">
548/// dbg.value %V, "x", FragmentX2
549/// <block of instructions, none being "dbg.value ..., "x", ...">
550/// dbg.assign undef, "x", FragmentX1
551///
552/// then (only) the instruction marked with (*) can be removed.
553/// Possible improvements:
554/// - Keep track of non-overlapping fragments.
555static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
556 assert(BB->isEntryBlock() && "expected entry block");
557 bool RemovedAny = false;
558 DenseSet<DebugVariableAggregate> SeenDefForAggregate;
559
560 // Remove undef dbg.assign intrinsics that are encountered before
561 // any non-undef intrinsics from the entry block.
562 for (auto &I : *BB) {
563 for (DbgVariableRecord &DVR :
564 make_early_inc_range(Range: filterDbgVars(R: I.getDbgRecordRange()))) {
565 if (!DVR.isDbgValue() && !DVR.isDbgAssign())
566 continue;
567 bool IsDbgValueKind =
568 (DVR.isDbgValue() || at::getAssignmentInsts(DVR: &DVR).empty());
569
570 DebugVariableAggregate Aggregate(&DVR);
571 if (!SeenDefForAggregate.contains(V: Aggregate)) {
572 bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
573 if (!IsKill) {
574 SeenDefForAggregate.insert(V: Aggregate);
575 } else if (DVR.isDbgAssign()) {
576 DVR.eraseFromParent();
577 RemovedAny = true;
578 }
579 }
580 }
581 }
582
583 return RemovedAny;
584}
585
586bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
587 bool MadeChanges = false;
588 // By using the "backward scan" strategy before the "forward scan" strategy we
589 // can remove both dbg.value (2) and (3) in a situation like this:
590 //
591 // (1) dbg.value V1, "x", DIExpression()
592 // ...
593 // (2) dbg.value V2, "x", DIExpression()
594 // (3) dbg.value V1, "x", DIExpression()
595 //
596 // The backward scan will remove (2), it is made obsolete by (3). After
597 // getting (2) out of the way, the foward scan will remove (3) since "x"
598 // already is described as having the value V1 at (1).
599 MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
600 if (BB->isEntryBlock() &&
601 isAssignmentTrackingEnabled(M: *BB->getParent()->getParent()))
602 MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
603 MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
604
605 if (MadeChanges)
606 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
607 << BB->getName() << "\n");
608 return MadeChanges;
609}
610
611void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
612 Instruction &I = *BI;
613 // Replaces all of the uses of the instruction with uses of the value
614 I.replaceAllUsesWith(V);
615
616 // Make sure to propagate a name if there is one already.
617 if (I.hasName() && !V->hasName())
618 V->takeName(V: &I);
619
620 // Delete the unnecessary instruction now...
621 BI = BI->eraseFromParent();
622}
623
624void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
625 Instruction *I) {
626 assert(I->getParent() == nullptr &&
627 "ReplaceInstWithInst: Instruction already inserted into basic block!");
628
629 // Copy debug location to newly added instruction, if it wasn't already set
630 // by the caller.
631 if (!I->getDebugLoc())
632 I->setDebugLoc(BI->getDebugLoc());
633
634 // Insert the new instruction into the basic block...
635 BasicBlock::iterator New = I->insertInto(ParentBB: BB, It: BI);
636
637 // Replace all uses of the old instruction, and delete it.
638 ReplaceInstWithValue(BI, V: I);
639
640 // Move BI back to point to the newly inserted instruction
641 BI = New;
642}
643
644bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
645 // Remember visited blocks to avoid infinite loop
646 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
647 unsigned Depth = 0;
648 while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
649 VisitedBlocks.insert(Ptr: BB).second) {
650 if (isa<UnreachableInst>(Val: BB->getTerminator()) ||
651 BB->getTerminatingDeoptimizeCall())
652 return true;
653 BB = BB->getUniqueSuccessor();
654 }
655 return false;
656}
657
658void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
659 BasicBlock::iterator BI(From);
660 ReplaceInstWithInst(BB: From->getParent(), BI, I: To);
661}
662
663BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
664 LoopInfo *LI, MemorySSAUpdater *MSSAU,
665 const Twine &BBName) {
666 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
667
668 Instruction *LatchTerm = BB->getTerminator();
669
670 CriticalEdgeSplittingOptions Options =
671 CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
672
673 if ((isCriticalEdge(TI: LatchTerm, SuccNum, AllowIdenticalEdges: Options.MergeIdenticalEdges))) {
674 // If this is a critical edge, let SplitKnownCriticalEdge do it.
675 return SplitKnownCriticalEdge(TI: LatchTerm, SuccNum, Options, BBName);
676 }
677
678 // If the edge isn't critical, then BB has a single successor or Succ has a
679 // single pred. Split the block.
680 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
681 // If the successor only has a single pred, split the top of the successor
682 // block.
683 assert(SP == BB && "CFG broken");
684 (void)SP;
685 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
686 return splitBlockBefore(Old: Succ, SplitPt: &Succ->front(), DTU: &DTU, LI, MSSAU, BBName);
687 }
688
689 // Otherwise, if BB has a single successor, split it at the bottom of the
690 // block.
691 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
692 "Should have a single succ!");
693 return SplitBlock(Old: BB, SplitPt: BB->getTerminator(), DT, LI, MSSAU, BBName);
694}
695
696/// Helper function to update the cycle or loop information after inserting a
697/// new block between a callbr instruction and one of its target blocks. Adds
698/// the new block to the innermost cycle or loop that the callbr instruction and
699/// the original target block share.
700/// \p LCI cycle or loop information to update
701/// \p CallBrBlock block containing the callbr instruction
702/// \p CallBrTarget new target block of the callbr instruction
703/// \p Succ original target block of the callbr instruction
704template <typename TI, typename T>
705static bool updateCycleLoopInfo(TI *LCI, BasicBlock *CallBrBlock,
706 BasicBlock *CallBrTarget, BasicBlock *Succ) {
707 static_assert(std::is_same_v<TI, CycleInfo> || std::is_same_v<TI, LoopInfo>,
708 "type must be CycleInfo or LoopInfo");
709 if (!LCI)
710 return false;
711
712 T *LC;
713 if constexpr (std::is_same_v<TI, CycleInfo>)
714 LC = LCI->getSmallestCommonCycle(CallBrBlock, Succ);
715 else
716 LC = LCI->getSmallestCommonLoop(CallBrBlock, Succ);
717 if (!LC)
718 return false;
719
720 if constexpr (std::is_same_v<TI, CycleInfo>)
721 LCI->addBlockToCycle(CallBrTarget, LC);
722 else
723 LC->addBasicBlockToLoop(CallBrTarget, *LCI);
724
725 return true;
726}
727
728BasicBlock *llvm::SplitCallBrEdge(BasicBlock *CallBrBlock, BasicBlock *Succ,
729 unsigned SuccIdx, DomTreeUpdater *DTU,
730 CycleInfo *CI, LoopInfo *LI,
731 bool *UpdatedLI) {
732 CallBrInst *CallBr = dyn_cast<CallBrInst>(Val: CallBrBlock->getTerminator());
733 assert(CallBr && "expected callbr terminator");
734 assert(SuccIdx < CallBr->getNumSuccessors() &&
735 Succ == CallBr->getSuccessor(SuccIdx) && "invalid successor index");
736
737 // Create a new block between callbr and the specified successor.
738 // splitBlockBefore cannot be re-used here since it cannot split if the split
739 // point is a PHI node (because BasicBlock::splitBasicBlockBefore cannot
740 // handle that). But we don't need to rewire every part of a potential PHI
741 // node. We only care about the edge between CallBrBlock and the original
742 // successor.
743 BasicBlock *CallBrTarget =
744 BasicBlock::Create(Context&: CallBrBlock->getContext(),
745 Name: CallBrBlock->getName() + ".target." + Succ->getName(),
746 Parent: CallBrBlock->getParent());
747 // Rewire control flow from the new target block to the original successor.
748 Succ->replacePhiUsesWith(Old: CallBrBlock, New: CallBrTarget);
749 // Rewire control flow from callbr to the new target block.
750 CallBr->setSuccessor(i: SuccIdx, NewSucc: CallBrTarget);
751 // Jump from the new target block to the original successor.
752 UncondBrInst::Create(IfTrue: Succ, InsertBefore: CallBrTarget);
753
754 bool Updated =
755 updateCycleLoopInfo<LoopInfo, Loop>(LCI: LI, CallBrBlock, CallBrTarget, Succ);
756 if (UpdatedLI)
757 *UpdatedLI = Updated;
758 updateCycleLoopInfo<CycleInfo, Cycle>(LCI: CI, CallBrBlock, CallBrTarget, Succ);
759 if (DTU) {
760 DTU->applyUpdates(Updates: {{DominatorTree::Insert, CallBrBlock, CallBrTarget}});
761 if (DTU->getDomTree().dominates(A: CallBrBlock, B: Succ)) {
762 if (!is_contained(Range: successors(BB: CallBrBlock), Element: Succ))
763 DTU->applyUpdates(Updates: {{DominatorTree::Delete, CallBrBlock, Succ}});
764 DTU->applyUpdates(Updates: {{DominatorTree::Insert, CallBrTarget, Succ}});
765 }
766 }
767
768 return CallBrTarget;
769}
770
771void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
772 if (auto *II = dyn_cast<InvokeInst>(Val: TI))
773 II->setUnwindDest(Succ);
774 else if (auto *CS = dyn_cast<CatchSwitchInst>(Val: TI))
775 CS->setUnwindDest(Succ);
776 else if (auto *CR = dyn_cast<CleanupReturnInst>(Val: TI))
777 CR->setUnwindDest(Succ);
778 else
779 llvm_unreachable("unexpected terminator instruction");
780}
781
782void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
783 BasicBlock *NewPred, PHINode *Until) {
784 int BBIdx = 0;
785 for (PHINode &PN : DestBB->phis()) {
786 // We manually update the LandingPadReplacement PHINode and it is the last
787 // PHI Node. So, if we find it, we are done.
788 if (Until == &PN)
789 break;
790
791 // Reuse the previous value of BBIdx if it lines up. In cases where we
792 // have multiple phi nodes with *lots* of predecessors, this is a speed
793 // win because we don't have to scan the PHI looking for TIBB. This
794 // happens because the BB list of PHI nodes are usually in the same
795 // order.
796 if (PN.getIncomingBlock(i: BBIdx) != OldPred)
797 BBIdx = PN.getBasicBlockIndex(BB: OldPred);
798
799 assert(BBIdx != -1 && "Invalid PHI Index!");
800 PN.setIncomingBlock(i: BBIdx, BB: NewPred);
801 }
802}
803
804BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
805 LandingPadInst *OriginalPad,
806 PHINode *LandingPadReplacement,
807 const CriticalEdgeSplittingOptions &Options,
808 const Twine &BBName) {
809
810 auto PadInst = Succ->getFirstNonPHIIt();
811 if (!LandingPadReplacement && !PadInst->isEHPad())
812 return SplitEdge(BB, Succ, DT: Options.DT, LI: Options.LI, MSSAU: Options.MSSAU, BBName);
813
814 auto *LI = Options.LI;
815 SmallVector<BasicBlock *, 4> LoopPreds;
816 // Check if extra modifications will be required to preserve loop-simplify
817 // form after splitting. If it would require splitting blocks with IndirectBr
818 // terminators, bail out if preserving loop-simplify form is requested.
819 if (Options.PreserveLoopSimplify && LI) {
820 if (Loop *BBLoop = LI->getLoopFor(BB)) {
821
822 // The only way that we can break LoopSimplify form by splitting a
823 // critical edge is when there exists some edge from BBLoop to Succ *and*
824 // the only edge into Succ from outside of BBLoop is that of NewBB after
825 // the split. If the first isn't true, then LoopSimplify still holds,
826 // NewBB is the new exit block and it has no non-loop predecessors. If the
827 // second isn't true, then Succ was not in LoopSimplify form prior to
828 // the split as it had a non-loop predecessor. In both of these cases,
829 // the predecessor must be directly in BBLoop, not in a subloop, or again
830 // LoopSimplify doesn't hold.
831 for (BasicBlock *P : predecessors(BB: Succ)) {
832 if (P == BB)
833 continue; // The new block is known.
834 if (LI->getLoopFor(BB: P) != BBLoop) {
835 // Loop is not in LoopSimplify form, no need to re simplify after
836 // splitting edge.
837 LoopPreds.clear();
838 break;
839 }
840 LoopPreds.push_back(Elt: P);
841 }
842 // Loop-simplify form can be preserved, if we can split all in-loop
843 // predecessors.
844 if (any_of(Range&: LoopPreds, P: [](BasicBlock *Pred) {
845 return isa<IndirectBrInst>(Val: Pred->getTerminator());
846 })) {
847 return nullptr;
848 }
849 }
850 }
851
852 auto *NewBB =
853 BasicBlock::Create(Context&: BB->getContext(), Name: BBName, Parent: BB->getParent(), InsertBefore: Succ);
854 setUnwindEdgeTo(TI: BB->getTerminator(), Succ: NewBB);
855 updatePhiNodes(DestBB: Succ, OldPred: BB, NewPred: NewBB, Until: LandingPadReplacement);
856
857 if (LandingPadReplacement) {
858 auto *NewLP = OriginalPad->clone();
859 auto *Terminator = UncondBrInst::Create(IfTrue: Succ, InsertBefore: NewBB);
860 NewLP->insertBefore(InsertPos: Terminator->getIterator());
861 LandingPadReplacement->addIncoming(V: NewLP, BB: NewBB);
862 } else {
863 Value *ParentPad = nullptr;
864 if (auto *FuncletPad = dyn_cast<FuncletPadInst>(Val&: PadInst))
865 ParentPad = FuncletPad->getParentPad();
866 else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val&: PadInst))
867 ParentPad = CatchSwitch->getParentPad();
868 else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(Val&: PadInst))
869 ParentPad = CleanupPad->getParentPad();
870 else if (auto *LandingPad = dyn_cast<LandingPadInst>(Val&: PadInst))
871 ParentPad = LandingPad->getParent();
872 else
873 llvm_unreachable("handling for other EHPads not implemented yet");
874
875 auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, Args: {}, NameStr: BBName, InsertBefore: NewBB);
876 CleanupReturnInst::Create(CleanupPad: NewCleanupPad, UnwindBB: Succ, InsertBefore: NewBB);
877 }
878
879 auto *DT = Options.DT;
880 auto *MSSAU = Options.MSSAU;
881 if (!DT && !LI)
882 return NewBB;
883
884 if (DT) {
885 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
886 SmallVector<DominatorTree::UpdateType, 3> Updates;
887
888 Updates.push_back(Elt: {DominatorTree::Insert, BB, NewBB});
889 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, Succ});
890 Updates.push_back(Elt: {DominatorTree::Delete, BB, Succ});
891
892 DTU.applyUpdates(Updates);
893 DTU.flush();
894
895 if (MSSAU) {
896 MSSAU->applyUpdates(Updates, DT&: *DT);
897 if (VerifyMemorySSA)
898 MSSAU->getMemorySSA()->verifyMemorySSA();
899 }
900 }
901
902 if (LI) {
903 if (Loop *BBLoop = LI->getLoopFor(BB)) {
904 // If one or the other blocks were not in a loop, the new block is not
905 // either, and thus LI doesn't need to be updated.
906 if (Loop *SuccLoop = LI->getLoopFor(BB: Succ)) {
907 if (BBLoop == SuccLoop) {
908 // Both in the same loop, the NewBB joins loop.
909 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
910 } else if (BBLoop->contains(L: SuccLoop)) {
911 // Edge from an outer loop to an inner loop. Add to the outer loop.
912 BBLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
913 } else if (SuccLoop->contains(L: BBLoop)) {
914 // Edge from an inner loop to an outer loop. Add to the outer loop.
915 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
916 } else {
917 // Edge from two loops with no containment relation. Because these
918 // are natural loops, we know that the destination block must be the
919 // header of its loop (adding a branch into a loop elsewhere would
920 // create an irreducible loop).
921 assert(SuccLoop->getHeader() == Succ &&
922 "Should not create irreducible loops!");
923 if (Loop *P = SuccLoop->getParentLoop())
924 P->addBasicBlockToLoop(NewBB, LI&: *LI);
925 }
926 }
927
928 // If BB is in a loop and Succ is outside of that loop, we may need to
929 // update LoopSimplify form and LCSSA form.
930 if (!BBLoop->contains(BB: Succ)) {
931 assert(!BBLoop->contains(NewBB) &&
932 "Split point for loop exit is contained in loop!");
933
934 // Update LCSSA form in the newly created exit block.
935 if (Options.PreserveLCSSA) {
936 createPHIsForSplitLoopExit(Preds: BB, SplitBB: NewBB, DestBB: Succ);
937 }
938
939 if (!LoopPreds.empty()) {
940 BasicBlock *NewExitBB = SplitBlockPredecessors(
941 BB: Succ, Preds: LoopPreds, Suffix: "split", DT, LI, MSSAU, PreserveLCSSA: Options.PreserveLCSSA);
942 if (Options.PreserveLCSSA)
943 createPHIsForSplitLoopExit(Preds: LoopPreds, SplitBB: NewExitBB, DestBB: Succ);
944 }
945 }
946 }
947 }
948
949 return NewBB;
950}
951
952void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
953 BasicBlock *SplitBB, BasicBlock *DestBB) {
954 // SplitBB shouldn't have anything non-trivial in it yet.
955 assert((&*SplitBB->getFirstNonPHIIt() == SplitBB->getTerminator() ||
956 SplitBB->isLandingPad()) &&
957 "SplitBB has non-PHI nodes!");
958
959 // For each PHI in the destination block.
960 for (PHINode &PN : DestBB->phis()) {
961 int Idx = PN.getBasicBlockIndex(BB: SplitBB);
962 assert(Idx >= 0 && "Invalid Block Index");
963 Value *V = PN.getIncomingValue(i: Idx);
964
965 // If the input is a PHI which already satisfies LCSSA, don't create
966 // a new one.
967 if (const PHINode *VP = dyn_cast<PHINode>(Val: V))
968 if (VP->getParent() == SplitBB)
969 continue;
970
971 // Otherwise a new PHI is needed. Create one and populate it.
972 PHINode *NewPN = PHINode::Create(Ty: PN.getType(), NumReservedValues: Preds.size(), NameStr: "split");
973 BasicBlock::iterator InsertPos =
974 SplitBB->isLandingPad() ? SplitBB->begin()
975 : SplitBB->getTerminator()->getIterator();
976 NewPN->insertBefore(InsertPos);
977 for (BasicBlock *BB : Preds)
978 NewPN->addIncoming(V, BB);
979
980 // Update the original PHI.
981 PN.setIncomingValue(i: Idx, V: NewPN);
982 }
983}
984
985unsigned
986llvm::SplitAllCriticalEdges(Function &F,
987 const CriticalEdgeSplittingOptions &Options) {
988 unsigned NumBroken = 0;
989 for (BasicBlock &BB : F) {
990 Instruction *TI = BB.getTerminator();
991 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(Val: TI))
992 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
993 if (SplitCriticalEdge(TI, SuccNum: i, Options))
994 ++NumBroken;
995 }
996 return NumBroken;
997}
998
999static BasicBlock *SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt,
1000 DomTreeUpdater *DTU, DominatorTree *DT,
1001 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1002 const Twine &BBName) {
1003 BasicBlock::iterator SplitIt = SplitPt;
1004 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad()) {
1005 ++SplitIt;
1006 assert(SplitIt != SplitPt->getParent()->end());
1007 }
1008 std::string Name = BBName.str();
1009 BasicBlock *New = Old->splitBasicBlock(
1010 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name);
1011
1012 // The new block lives in whichever loop the old one did. This preserves
1013 // LCSSA as well, because we force the split point to be after any PHI nodes.
1014 if (LI)
1015 if (Loop *L = LI->getLoopFor(BB: Old))
1016 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
1017
1018 if (DTU) {
1019 SmallVector<DominatorTree::UpdateType, 8> Updates;
1020 // Old dominates New. New node dominates all other nodes dominated by Old.
1021 SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
1022 Updates.push_back(Elt: {DominatorTree::Insert, Old, New});
1023 Updates.reserve(N: Updates.size() + 2 * succ_size(BB: New));
1024 for (BasicBlock *SuccessorOfOld : successors(BB: New))
1025 if (UniqueSuccessorsOfOld.insert(Ptr: SuccessorOfOld).second) {
1026 Updates.push_back(Elt: {DominatorTree::Insert, New, SuccessorOfOld});
1027 Updates.push_back(Elt: {DominatorTree::Delete, Old, SuccessorOfOld});
1028 }
1029
1030 DTU->applyUpdates(Updates);
1031 } else if (DT)
1032 // Old dominates New. New node dominates all other nodes dominated by Old.
1033 if (DomTreeNode *OldNode = DT->getNode(BB: Old)) {
1034 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1035
1036 DomTreeNode *NewNode = DT->addNewBlock(BB: New, DomBB: Old);
1037 for (DomTreeNode *I : Children)
1038 DT->changeImmediateDominator(N: I, NewIDom: NewNode);
1039 }
1040
1041 // Move MemoryAccesses still tracked in Old, but part of New now.
1042 // Update accesses in successor blocks accordingly.
1043 if (MSSAU)
1044 MSSAU->moveAllAfterSpliceBlocks(From: Old, To: New, Start: &*(New->begin()));
1045
1046 return New;
1047}
1048
1049BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1050 DominatorTree *DT, LoopInfo *LI,
1051 MemorySSAUpdater *MSSAU, const Twine &BBName) {
1052 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName);
1053}
1054BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1055 DomTreeUpdater *DTU, LoopInfo *LI,
1056 MemorySSAUpdater *MSSAU, const Twine &BBName) {
1057 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName);
1058}
1059
1060/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
1061/// Invalidates DFS Numbering when DTU or DT is provided.
1062static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
1063 ArrayRef<BasicBlock *> Preds,
1064 DomTreeUpdater *DTU, DominatorTree *DT,
1065 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1066 bool PreserveLCSSA, bool &HasLoopExit) {
1067 // Update dominator tree if available.
1068 if (DTU) {
1069 // Recalculation of DomTree is needed when updating a forward DomTree and
1070 // the Entry BB is replaced.
1071 if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1072 // The entry block was removed and there is no external interface for
1073 // the dominator tree to be notified of this change. In this corner-case
1074 // we recalculate the entire tree.
1075 DTU->recalculate(F&: *NewBB->getParent());
1076 } else {
1077 // Split block expects NewBB to have a non-empty set of predecessors.
1078 SmallVector<DominatorTree::UpdateType, 8> Updates;
1079 SmallPtrSet<BasicBlock *, 8> UniquePreds;
1080 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, OldBB});
1081 Updates.reserve(N: Updates.size() + 2 * Preds.size());
1082 for (auto *Pred : Preds)
1083 if (UniquePreds.insert(Ptr: Pred).second) {
1084 Updates.push_back(Elt: {DominatorTree::Insert, Pred, NewBB});
1085 Updates.push_back(Elt: {DominatorTree::Delete, Pred, OldBB});
1086 }
1087 DTU->applyUpdates(Updates);
1088 }
1089 } else if (DT) {
1090 if (OldBB == DT->getRootNode()->getBlock()) {
1091 assert(NewBB->isEntryBlock());
1092 DT->setNewRoot(NewBB);
1093 } else {
1094 // Split block expects NewBB to have a non-empty set of predecessors.
1095 DT->splitBlock(NewBB);
1096 }
1097 }
1098
1099 // Update MemoryPhis after split if MemorySSA is available
1100 if (MSSAU)
1101 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(Old: OldBB, New: NewBB, Preds);
1102
1103 // The rest of the logic is only relevant for updating the loop structures.
1104 if (!LI)
1105 return;
1106
1107 if (DTU && DTU->hasDomTree())
1108 DT = &DTU->getDomTree();
1109 assert(DT && "DT should be available to update LoopInfo!");
1110 Loop *L = LI->getLoopFor(BB: OldBB);
1111
1112 // If we need to preserve loop analyses, collect some information about how
1113 // this split will affect loops.
1114 bool IsLoopEntry = !!L;
1115 bool SplitMakesNewLoopHeader = false;
1116 for (BasicBlock *Pred : Preds) {
1117 // Preds that are not reachable from entry should not be used to identify if
1118 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1119 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1120 // as true and make the NewBB the header of some loop. This breaks LI.
1121 if (!DT->isReachableFromEntry(A: Pred))
1122 continue;
1123 // If we need to preserve LCSSA, determine if any of the preds is a loop
1124 // exit.
1125 if (PreserveLCSSA)
1126 if (Loop *PL = LI->getLoopFor(BB: Pred))
1127 if (!PL->contains(BB: OldBB))
1128 HasLoopExit = true;
1129
1130 // If we need to preserve LoopInfo, note whether any of the preds crosses
1131 // an interesting loop boundary.
1132 if (!L)
1133 continue;
1134 if (L->contains(BB: Pred))
1135 IsLoopEntry = false;
1136 else
1137 SplitMakesNewLoopHeader = true;
1138 }
1139
1140 // Unless we have a loop for OldBB, nothing else to do here.
1141 if (!L)
1142 return;
1143
1144 if (IsLoopEntry) {
1145 // Add the new block to the nearest enclosing loop (and not an adjacent
1146 // loop). To find this, examine each of the predecessors and determine which
1147 // loops enclose them, and select the most-nested loop which contains the
1148 // loop containing the block being split.
1149 Loop *InnermostPredLoop = nullptr;
1150 for (BasicBlock *Pred : Preds) {
1151 if (Loop *PredLoop = LI->getLoopFor(BB: Pred)) {
1152 // Seek a loop which actually contains the block being split (to avoid
1153 // adjacent loops).
1154 while (PredLoop && !PredLoop->contains(BB: OldBB))
1155 PredLoop = PredLoop->getParentLoop();
1156
1157 // Select the most-nested of these loops which contains the block.
1158 if (PredLoop && PredLoop->contains(BB: OldBB) &&
1159 (!InnermostPredLoop ||
1160 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1161 InnermostPredLoop = PredLoop;
1162 }
1163 }
1164
1165 if (InnermostPredLoop)
1166 InnermostPredLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1167 } else {
1168 L->addBasicBlockToLoop(NewBB, LI&: *LI);
1169 if (SplitMakesNewLoopHeader)
1170 L->moveToHeader(BB: NewBB);
1171 }
1172}
1173
1174BasicBlock *llvm::splitBlockBefore(BasicBlock *Old,
1175 BasicBlock::iterator SplitPt,
1176 DomTreeUpdater *DTU, LoopInfo *LI,
1177 MemorySSAUpdater *MSSAU,
1178 const Twine &BBName) {
1179 BasicBlock::iterator SplitIt = SplitPt;
1180 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad())
1181 ++SplitIt;
1182 SmallVector<BasicBlock *, 4> Preds(predecessors(BB: Old));
1183 BasicBlock *New = Old->splitBasicBlockBefore(
1184 I: SplitIt, BBName: BBName.isTriviallyEmpty() ? Old->getName() + ".split" : BBName);
1185
1186 bool HasLoopExit = false;
1187 UpdateAnalysisInformation(OldBB: Old, NewBB: New, Preds, DTU, DT: nullptr, LI, MSSAU, PreserveLCSSA: false,
1188 HasLoopExit);
1189
1190 return New;
1191}
1192
1193/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1194/// This also updates AliasAnalysis, if available.
1195static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1196 ArrayRef<BasicBlock *> Preds, Instruction *BI,
1197 bool HasLoopExit) {
1198 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1199 SmallPtrSet<BasicBlock *, 16> PredSet(llvm::from_range, Preds);
1200 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(Val: I); ) {
1201 PHINode *PN = cast<PHINode>(Val: I++);
1202
1203 // Check to see if all of the values coming in are the same. If so, we
1204 // don't need to create a new PHI node, unless it's needed for LCSSA.
1205 Value *InVal = nullptr;
1206 if (!HasLoopExit) {
1207 InVal = PN->getIncomingValueForBlock(BB: Preds[0]);
1208 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1209 if (!PredSet.count(Ptr: PN->getIncomingBlock(i)))
1210 continue;
1211 if (!InVal)
1212 InVal = PN->getIncomingValue(i);
1213 else if (InVal != PN->getIncomingValue(i)) {
1214 InVal = nullptr;
1215 break;
1216 }
1217 }
1218 }
1219
1220 if (InVal) {
1221 // If all incoming values for the new PHI would be the same, just don't
1222 // make a new PHI. Instead, just remove the incoming values from the old
1223 // PHI.
1224 PN->removeIncomingValueIf(
1225 Predicate: [&](unsigned Idx) {
1226 return PredSet.contains(Ptr: PN->getIncomingBlock(i: Idx));
1227 },
1228 /* DeletePHIIfEmpty */ false);
1229
1230 // Add an incoming value to the PHI node in the loop for the preheader
1231 // edge.
1232 PN->addIncoming(V: InVal, BB: NewBB);
1233 continue;
1234 }
1235
1236 // If the values coming into the block are not the same, we need a new
1237 // PHI.
1238 // Create the new PHI node, insert it into NewBB at the end of the block
1239 PHINode *NewPHI =
1240 PHINode::Create(Ty: PN->getType(), NumReservedValues: Preds.size(), NameStr: PN->getName() + ".ph", InsertBefore: BI->getIterator());
1241
1242 // NOTE! This loop walks backwards for a reason! First off, this minimizes
1243 // the cost of removal if we end up removing a large number of values, and
1244 // second off, this ensures that the indices for the incoming values aren't
1245 // invalidated when we remove one.
1246 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1247 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1248 if (PredSet.count(Ptr: IncomingBB)) {
1249 Value *V = PN->removeIncomingValue(Idx: i, DeletePHIIfEmpty: false);
1250 NewPHI->addIncoming(V, BB: IncomingBB);
1251 }
1252 }
1253
1254 PN->addIncoming(V: NewPHI, BB: NewBB);
1255 }
1256}
1257
1258static void SplitLandingPadPredecessorsImpl(
1259 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1260 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1261 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1262 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1263
1264static BasicBlock *
1265SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
1266 const char *Suffix, DomTreeUpdater *DTU,
1267 DominatorTree *DT, LoopInfo *LI,
1268 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1269 // Do not attempt to split that which cannot be split.
1270 if (!BB->canSplitPredecessors())
1271 return nullptr;
1272
1273 // For the landingpads we need to act a bit differently.
1274 // Delegate this work to the SplitLandingPadPredecessors.
1275 if (BB->isLandingPad()) {
1276 SmallVector<BasicBlock*, 2> NewBBs;
1277 std::string NewName = std::string(Suffix) + ".split-lp";
1278
1279 SplitLandingPadPredecessorsImpl(OrigBB: BB, Preds, Suffix1: Suffix, Suffix2: NewName.c_str(), NewBBs,
1280 DTU, DT, LI, MSSAU, PreserveLCSSA);
1281 return NewBBs[0];
1282 }
1283
1284 // Create new basic block, insert right before the original block.
1285 BasicBlock *NewBB = BasicBlock::Create(
1286 Context&: BB->getContext(), Name: BB->getName() + Suffix, Parent: BB->getParent(), InsertBefore: BB);
1287
1288 // The new block unconditionally branches to the old block.
1289 UncondBrInst *BI = UncondBrInst::Create(IfTrue: BB, InsertBefore: NewBB);
1290
1291 Loop *L = nullptr;
1292 BasicBlock *OldLatch = nullptr;
1293 // Splitting the predecessors of a loop header creates a preheader block.
1294 if (LI && LI->isLoopHeader(BB)) {
1295 L = LI->getLoopFor(BB);
1296 // Using the loop start line number prevents debuggers stepping into the
1297 // loop body for this instruction.
1298 BI->setDebugLoc(L->getStartLoc());
1299
1300 // If BB is the header of the Loop, it is possible that the loop is
1301 // modified, such that the current latch does not remain the latch of the
1302 // loop. If that is the case, the loop metadata from the current latch needs
1303 // to be applied to the new latch.
1304 OldLatch = L->getLoopLatch();
1305 } else
1306 BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
1307
1308 // Move the edges from Preds to point to NewBB instead of BB.
1309 for (BasicBlock *Pred : Preds) {
1310 // This is slightly more strict than necessary; the minimum requirement
1311 // is that there be no more than one indirectbr branching to BB. And
1312 // all BlockAddress uses would need to be updated.
1313 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1314 "Cannot split an edge from an IndirectBrInst");
1315 Pred->getTerminator()->replaceSuccessorWith(OldBB: BB, NewBB);
1316 }
1317
1318 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1319 // node becomes an incoming value for BB's phi node. However, if the Preds
1320 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1321 // account for the newly created predecessor.
1322 if (Preds.empty()) {
1323 // Insert dummy values as the incoming value.
1324 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(Val: I); ++I)
1325 cast<PHINode>(Val&: I)->addIncoming(V: PoisonValue::get(T: I->getType()), BB: NewBB);
1326 }
1327
1328 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1329 bool HasLoopExit = false;
1330 UpdateAnalysisInformation(OldBB: BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1331 HasLoopExit);
1332
1333 if (!Preds.empty()) {
1334 // Update the PHI nodes in BB with the values coming from NewBB.
1335 UpdatePHINodes(OrigBB: BB, NewBB, Preds, BI, HasLoopExit);
1336 }
1337
1338 if (OldLatch) {
1339 BasicBlock *NewLatch = L->getLoopLatch();
1340 if (NewLatch != OldLatch) {
1341 MDNode *MD = OldLatch->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
1342 NewLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: MD);
1343 // It's still possible that OldLatch is the latch of another inner loop,
1344 // in which case we do not remove the metadata.
1345 Loop *IL = LI->getLoopFor(BB: OldLatch);
1346 if (IL && IL->getLoopLatch() != OldLatch)
1347 OldLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: nullptr);
1348 }
1349 }
1350
1351 return NewBB;
1352}
1353
1354BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1355 ArrayRef<BasicBlock *> Preds,
1356 const char *Suffix, DominatorTree *DT,
1357 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1358 bool PreserveLCSSA) {
1359 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1360 MSSAU, PreserveLCSSA);
1361}
1362BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1363 ArrayRef<BasicBlock *> Preds,
1364 const char *Suffix,
1365 DomTreeUpdater *DTU, LoopInfo *LI,
1366 MemorySSAUpdater *MSSAU,
1367 bool PreserveLCSSA) {
1368 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1369 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1370}
1371
1372static void SplitLandingPadPredecessorsImpl(
1373 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1374 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1375 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1376 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1377 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1378
1379 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1380 // it right before the original block.
1381 BasicBlock *NewBB1 = BasicBlock::Create(Context&: OrigBB->getContext(),
1382 Name: OrigBB->getName() + Suffix1,
1383 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1384 NewBBs.push_back(Elt: NewBB1);
1385
1386 // The new block unconditionally branches to the old block.
1387 UncondBrInst *BI1 = UncondBrInst::Create(IfTrue: OrigBB, InsertBefore: NewBB1);
1388 BI1->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1389
1390 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1391 for (BasicBlock *Pred : Preds) {
1392 // This is slightly more strict than necessary; the minimum requirement
1393 // is that there be no more than one indirectbr branching to BB. And
1394 // all BlockAddress uses would need to be updated.
1395 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1396 "Cannot split an edge from an IndirectBrInst");
1397 Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB1);
1398 }
1399
1400 bool HasLoopExit = false;
1401 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB1, Preds, DTU, DT, LI, MSSAU,
1402 PreserveLCSSA, HasLoopExit);
1403
1404 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1405 UpdatePHINodes(OrigBB, NewBB: NewBB1, Preds, BI: BI1, HasLoopExit);
1406
1407 // Move the remaining edges from OrigBB to point to NewBB2.
1408 SmallVector<BasicBlock*, 8> NewBB2Preds;
1409 for (pred_iterator i = pred_begin(BB: OrigBB), e = pred_end(BB: OrigBB);
1410 i != e; ) {
1411 BasicBlock *Pred = *i++;
1412 if (Pred == NewBB1) continue;
1413 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1414 "Cannot split an edge from an IndirectBrInst");
1415 NewBB2Preds.push_back(Elt: Pred);
1416 e = pred_end(BB: OrigBB);
1417 }
1418
1419 BasicBlock *NewBB2 = nullptr;
1420 if (!NewBB2Preds.empty()) {
1421 // Create another basic block for the rest of OrigBB's predecessors.
1422 NewBB2 = BasicBlock::Create(Context&: OrigBB->getContext(),
1423 Name: OrigBB->getName() + Suffix2,
1424 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1425 NewBBs.push_back(Elt: NewBB2);
1426
1427 // The new block unconditionally branches to the old block.
1428 UncondBrInst *BI2 = UncondBrInst::Create(IfTrue: OrigBB, InsertBefore: NewBB2);
1429 BI2->setDebugLoc(OrigBB->getFirstNonPHIIt()->getDebugLoc());
1430
1431 // Move the remaining edges from OrigBB to point to NewBB2.
1432 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1433 NewBB2Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB2);
1434
1435 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1436 HasLoopExit = false;
1437 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, DTU, DT, LI, MSSAU,
1438 PreserveLCSSA, HasLoopExit);
1439
1440 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1441 UpdatePHINodes(OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, BI: BI2, HasLoopExit);
1442 }
1443
1444 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1445 Instruction *Clone1 = LPad->clone();
1446 Clone1->setName(Twine("lpad") + Suffix1);
1447 Clone1->insertInto(ParentBB: NewBB1, It: NewBB1->getFirstInsertionPt());
1448
1449 if (NewBB2) {
1450 Instruction *Clone2 = LPad->clone();
1451 Clone2->setName(Twine("lpad") + Suffix2);
1452 Clone2->insertInto(ParentBB: NewBB2, It: NewBB2->getFirstInsertionPt());
1453
1454 // Create a PHI node for the two cloned landingpad instructions only
1455 // if the original landingpad instruction has some uses.
1456 if (!LPad->use_empty()) {
1457 assert(!LPad->getType()->isTokenTy() &&
1458 "Split cannot be applied if LPad is token type. Otherwise an "
1459 "invalid PHINode of token type would be created.");
1460 PHINode *PN = PHINode::Create(Ty: LPad->getType(), NumReservedValues: 2, NameStr: "lpad.phi", InsertBefore: LPad->getIterator());
1461 PN->addIncoming(V: Clone1, BB: NewBB1);
1462 PN->addIncoming(V: Clone2, BB: NewBB2);
1463 LPad->replaceAllUsesWith(V: PN);
1464 }
1465 LPad->eraseFromParent();
1466 } else {
1467 // There is no second clone. Just replace the landing pad with the first
1468 // clone.
1469 LPad->replaceAllUsesWith(V: Clone1);
1470 LPad->eraseFromParent();
1471 }
1472}
1473
1474void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1475 ArrayRef<BasicBlock *> Preds,
1476 const char *Suffix1, const char *Suffix2,
1477 SmallVectorImpl<BasicBlock *> &NewBBs,
1478 DomTreeUpdater *DTU, LoopInfo *LI,
1479 MemorySSAUpdater *MSSAU,
1480 bool PreserveLCSSA) {
1481 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1482 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1483 PreserveLCSSA);
1484}
1485
1486ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1487 BasicBlock *Pred,
1488 DomTreeUpdater *DTU) {
1489 Instruction *UncondBranch = Pred->getTerminator();
1490 // Clone the return and add it to the end of the predecessor.
1491 Instruction *NewRet = RI->clone();
1492 NewRet->insertInto(ParentBB: Pred, It: Pred->end());
1493
1494 // If the return instruction returns a value, and if the value was a
1495 // PHI node in "BB", propagate the right value into the return.
1496 for (Use &Op : NewRet->operands()) {
1497 Value *V = Op;
1498 Instruction *NewBC = nullptr;
1499 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Val: V)) {
1500 // Return value might be bitcasted. Clone and insert it before the
1501 // return instruction.
1502 V = BCI->getOperand(i_nocapture: 0);
1503 NewBC = BCI->clone();
1504 NewBC->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1505 Op = NewBC;
1506 }
1507
1508 Instruction *NewEV = nullptr;
1509 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: V)) {
1510 V = EVI->getOperand(i_nocapture: 0);
1511 NewEV = EVI->clone();
1512 if (NewBC) {
1513 NewBC->setOperand(i: 0, Val: NewEV);
1514 NewEV->insertInto(ParentBB: Pred, It: NewBC->getIterator());
1515 } else {
1516 NewEV->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1517 Op = NewEV;
1518 }
1519 }
1520
1521 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
1522 if (PN->getParent() == BB) {
1523 if (NewEV) {
1524 NewEV->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1525 } else if (NewBC)
1526 NewBC->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1527 else
1528 Op = PN->getIncomingValueForBlock(BB: Pred);
1529 }
1530 }
1531 }
1532
1533 // Update any PHI nodes in the returning block to realize that we no
1534 // longer branch to them.
1535 BB->removePredecessor(Pred);
1536 UncondBranch->eraseFromParent();
1537
1538 if (DTU)
1539 DTU->applyUpdates(Updates: {{DominatorTree::Delete, Pred, BB}});
1540
1541 return cast<ReturnInst>(Val: NewRet);
1542}
1543
1544Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1545 BasicBlock::iterator SplitBefore,
1546 bool Unreachable,
1547 MDNode *BranchWeights,
1548 DomTreeUpdater *DTU, LoopInfo *LI,
1549 BasicBlock *ThenBlock) {
1550 SplitBlockAndInsertIfThenElse(
1551 Cond, SplitBefore, ThenBlock: &ThenBlock, /* ElseBlock */ nullptr,
1552 /* UnreachableThen */ Unreachable,
1553 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1554 return ThenBlock->getTerminator();
1555}
1556
1557Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
1558 BasicBlock::iterator SplitBefore,
1559 bool Unreachable,
1560 MDNode *BranchWeights,
1561 DomTreeUpdater *DTU, LoopInfo *LI,
1562 BasicBlock *ElseBlock) {
1563 SplitBlockAndInsertIfThenElse(
1564 Cond, SplitBefore, /* ThenBlock */ nullptr, ElseBlock: &ElseBlock,
1565 /* UnreachableThen */ false,
1566 /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1567 return ElseBlock->getTerminator();
1568}
1569
1570void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore,
1571 Instruction **ThenTerm,
1572 Instruction **ElseTerm,
1573 MDNode *BranchWeights,
1574 DomTreeUpdater *DTU, LoopInfo *LI) {
1575 BasicBlock *ThenBlock = nullptr;
1576 BasicBlock *ElseBlock = nullptr;
1577 SplitBlockAndInsertIfThenElse(
1578 Cond, SplitBefore, ThenBlock: &ThenBlock, ElseBlock: &ElseBlock, /* UnreachableThen */ false,
1579 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1580
1581 *ThenTerm = ThenBlock->getTerminator();
1582 *ElseTerm = ElseBlock->getTerminator();
1583}
1584
1585void llvm::SplitBlockAndInsertIfThenElse(
1586 Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
1587 BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1588 MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1589 assert((ThenBlock || ElseBlock) &&
1590 "At least one branch block must be created");
1591 assert((!UnreachableThen || !UnreachableElse) &&
1592 "Split block tail must be reachable");
1593
1594 SmallVector<DominatorTree::UpdateType, 8> Updates;
1595 SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1596 BasicBlock *Head = SplitBefore->getParent();
1597 if (DTU) {
1598 UniqueOrigSuccessors.insert_range(R: successors(BB: Head));
1599 Updates.reserve(N: 4 + 2 * UniqueOrigSuccessors.size());
1600 }
1601
1602 LLVMContext &C = Head->getContext();
1603 BasicBlock *Tail = Head->splitBasicBlock(I: SplitBefore);
1604 BasicBlock *TrueBlock = Tail;
1605 BasicBlock *FalseBlock = Tail;
1606 bool ThenToTailEdge = false;
1607 bool ElseToTailEdge = false;
1608
1609 // Encapsulate the logic around creation/insertion/etc of a new block.
1610 auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1611 bool &ToTailEdge) {
1612 if (PBB == nullptr)
1613 return; // Do not create/insert a block.
1614
1615 if (*PBB)
1616 BB = *PBB; // Caller supplied block, use it.
1617 else {
1618 // Create a new block.
1619 BB = BasicBlock::Create(Context&: C, Name: "", Parent: Head->getParent(), InsertBefore: Tail);
1620 if (Unreachable)
1621 (void)new UnreachableInst(C, BB);
1622 else {
1623 (void)UncondBrInst::Create(IfTrue: Tail, InsertBefore: BB);
1624 ToTailEdge = true;
1625 }
1626 BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1627 // Pass the new block back to the caller.
1628 *PBB = BB;
1629 }
1630 };
1631
1632 handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1633 handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1634
1635 Instruction *HeadOldTerm = Head->getTerminator();
1636 CondBrInst *HeadNewTerm = CondBrInst::Create(Cond, IfTrue: TrueBlock, IfFalse: FalseBlock);
1637 HeadNewTerm->setMetadata(KindID: LLVMContext::MD_prof, Node: BranchWeights);
1638 ReplaceInstWithInst(From: HeadOldTerm, To: HeadNewTerm);
1639
1640 if (DTU) {
1641 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: TrueBlock);
1642 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: FalseBlock);
1643 if (ThenToTailEdge)
1644 Updates.emplace_back(Args: DominatorTree::Insert, Args&: TrueBlock, Args&: Tail);
1645 if (ElseToTailEdge)
1646 Updates.emplace_back(Args: DominatorTree::Insert, Args&: FalseBlock, Args&: Tail);
1647 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1648 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Tail, Args&: UniqueOrigSuccessor);
1649 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1650 Updates.emplace_back(Args: DominatorTree::Delete, Args&: Head, Args&: UniqueOrigSuccessor);
1651 DTU->applyUpdates(Updates);
1652 }
1653
1654 if (LI) {
1655 if (Loop *L = LI->getLoopFor(BB: Head); L) {
1656 if (ThenToTailEdge)
1657 L->addBasicBlockToLoop(NewBB: TrueBlock, LI&: *LI);
1658 if (ElseToTailEdge)
1659 L->addBasicBlockToLoop(NewBB: FalseBlock, LI&: *LI);
1660 L->addBasicBlockToLoop(NewBB: Tail, LI&: *LI);
1661 }
1662 }
1663}
1664
1665std::pair<Instruction *, Value *>
1666llvm::SplitBlockAndInsertSimpleForLoop(Value *End,
1667 BasicBlock::iterator SplitBefore) {
1668 BasicBlock *LoopPred = SplitBefore->getParent();
1669 BasicBlock *LoopBody = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1670 BasicBlock *LoopExit = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1671
1672 auto *Ty = End->getType();
1673 auto &DL = SplitBefore->getDataLayout();
1674 const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1675
1676 IRBuilder<> Builder(LoopBody->getTerminator());
1677 auto *IV = Builder.CreatePHI(Ty, NumReservedValues: 2, Name: "iv");
1678 auto *IVNext =
1679 Builder.CreateAdd(LHS: IV, RHS: ConstantInt::get(Ty, V: 1), Name: IV->getName() + ".next",
1680 /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1681 auto *IVCheck = Builder.CreateICmpEQ(LHS: IVNext, RHS: End,
1682 Name: IV->getName() + ".check");
1683 Builder.CreateCondBr(Cond: IVCheck, True: LoopExit, False: LoopBody);
1684 LoopBody->getTerminator()->eraseFromParent();
1685
1686 // Populate the IV PHI.
1687 IV->addIncoming(V: ConstantInt::get(Ty, V: 0), BB: LoopPred);
1688 IV->addIncoming(V: IVNext, BB: LoopBody);
1689
1690 return std::make_pair(x: &*LoopBody->getFirstNonPHIIt(), y&: IV);
1691}
1692
1693void llvm::SplitBlockAndInsertForEachLane(
1694 ElementCount EC, Type *IndexTy, BasicBlock::iterator InsertBefore,
1695 std::function<void(IRBuilderBase &, Value *)> Func) {
1696
1697 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1698
1699 if (EC.isScalable()) {
1700 Value *NumElements = IRB.CreateElementCount(Ty: IndexTy, EC);
1701
1702 auto [BodyIP, Index] =
1703 SplitBlockAndInsertSimpleForLoop(End: NumElements, SplitBefore: InsertBefore);
1704
1705 IRB.SetInsertPoint(BodyIP);
1706 Func(IRB, Index);
1707 return;
1708 }
1709
1710 unsigned Num = EC.getFixedValue();
1711 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1712 IRB.SetInsertPoint(InsertBefore);
1713 Func(IRB, ConstantInt::get(Ty: IndexTy, V: Idx));
1714 }
1715}
1716
1717void llvm::SplitBlockAndInsertForEachLane(
1718 Value *EVL, BasicBlock::iterator InsertBefore,
1719 std::function<void(IRBuilderBase &, Value *)> Func) {
1720
1721 IRBuilder<> IRB(InsertBefore->getParent(), InsertBefore);
1722 Type *Ty = EVL->getType();
1723
1724 if (!isa<ConstantInt>(Val: EVL)) {
1725 auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(End: EVL, SplitBefore: InsertBefore);
1726 IRB.SetInsertPoint(BodyIP);
1727 Func(IRB, Index);
1728 return;
1729 }
1730
1731 unsigned Num = cast<ConstantInt>(Val: EVL)->getZExtValue();
1732 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1733 IRB.SetInsertPoint(InsertBefore);
1734 Func(IRB, ConstantInt::get(Ty, V: Idx));
1735 }
1736}
1737
1738CondBrInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1739 BasicBlock *&IfFalse) {
1740 PHINode *SomePHI = dyn_cast<PHINode>(Val: BB->begin());
1741 BasicBlock *Pred1 = nullptr;
1742 BasicBlock *Pred2 = nullptr;
1743
1744 if (SomePHI) {
1745 if (SomePHI->getNumIncomingValues() != 2)
1746 return nullptr;
1747 Pred1 = SomePHI->getIncomingBlock(i: 0);
1748 Pred2 = SomePHI->getIncomingBlock(i: 1);
1749 } else {
1750 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1751 if (PI == PE) // No predecessor
1752 return nullptr;
1753 Pred1 = *PI++;
1754 if (PI == PE) // Only one predecessor
1755 return nullptr;
1756 Pred2 = *PI++;
1757 if (PI != PE) // More than two predecessors
1758 return nullptr;
1759 }
1760
1761 Instruction *Pred1Term = Pred1->getTerminator();
1762 Instruction *Pred2Term = Pred2->getTerminator();
1763
1764 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1765 // either are.
1766 if (isa<CondBrInst>(Val: Pred2Term)) {
1767 // If both branches are conditional, we don't have an "if statement". In
1768 // reality, we could transform this case, but since the condition will be
1769 // required anyway, we stand no chance of eliminating it, so the xform is
1770 // probably not profitable.
1771 if (isa<CondBrInst>(Val: Pred1Term))
1772 return nullptr;
1773
1774 std::swap(a&: Pred1, b&: Pred2);
1775 std::swap(a&: Pred1Term, b&: Pred2Term);
1776 }
1777
1778 // We can only handle branches. Other control flow will be lowered to
1779 // branches if possible anyway.
1780 if (!isa<UncondBrInst>(Val: Pred2Term))
1781 return nullptr;
1782
1783 if (auto *Pred1Br = dyn_cast<CondBrInst>(Val: Pred1Term)) {
1784 // The only thing we have to watch out for here is to make sure that Pred2
1785 // doesn't have incoming edges from other blocks. If it does, the condition
1786 // doesn't dominate BB.
1787 if (!Pred2->getSinglePredecessor())
1788 return nullptr;
1789
1790 // If we found a conditional branch predecessor, make sure that it branches
1791 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1792 if (Pred1Br->getSuccessor(i: 0) == BB &&
1793 Pred1Br->getSuccessor(i: 1) == Pred2) {
1794 IfTrue = Pred1;
1795 IfFalse = Pred2;
1796 } else if (Pred1Br->getSuccessor(i: 0) == Pred2 &&
1797 Pred1Br->getSuccessor(i: 1) == BB) {
1798 IfTrue = Pred2;
1799 IfFalse = Pred1;
1800 } else {
1801 // We know that one arm of the conditional goes to BB, so the other must
1802 // go somewhere unrelated, and this must not be an "if statement".
1803 return nullptr;
1804 }
1805
1806 return Pred1Br;
1807 }
1808
1809 if (!isa<UncondBrInst>(Val: Pred1Term))
1810 return nullptr;
1811
1812 // Ok, if we got here, both predecessors end with an unconditional branch to
1813 // BB. Don't panic! If both blocks only have a single (identical)
1814 // predecessor, and THAT is a conditional branch, then we're all ok!
1815 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1816 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1817 return nullptr;
1818
1819 // Otherwise, if this is a conditional branch, then we can use it!
1820 CondBrInst *BI = dyn_cast<CondBrInst>(Val: CommonPred->getTerminator());
1821 if (!BI) return nullptr;
1822
1823 if (BI->getSuccessor(i: 0) == Pred1) {
1824 IfTrue = Pred1;
1825 IfFalse = Pred2;
1826 } else {
1827 IfTrue = Pred2;
1828 IfFalse = Pred1;
1829 }
1830 return BI;
1831}
1832
1833void llvm::InvertBranch(CondBrInst *PBI, IRBuilderBase &Builder) {
1834 Value *NewCond = PBI->getCondition();
1835 // If this is a "cmp" instruction, only used for branching (and nowhere
1836 // else), then we can simply invert the predicate.
1837 if (NewCond->hasOneUse() && isa<CmpInst>(Val: NewCond)) {
1838 CmpInst *CI = cast<CmpInst>(Val: NewCond);
1839 CI->setPredicate(CI->getInversePredicate());
1840 } else
1841 NewCond = Builder.CreateNot(V: NewCond, Name: NewCond->getName() + ".not");
1842
1843 PBI->setCondition(NewCond);
1844 PBI->swapSuccessors();
1845}
1846
1847bool llvm::hasOnlySimpleTerminator(const Function &F) {
1848 for (auto &BB : F) {
1849 auto *Term = BB.getTerminator();
1850 if (!isa<ReturnInst, UnreachableInst, UncondBrInst, CondBrInst>(Val: Term))
1851 return false;
1852 }
1853 return true;
1854}
1855
1856Printable llvm::printBasicBlock(const BasicBlock *BB) {
1857 return Printable([BB](raw_ostream &OS) {
1858 if (!BB) {
1859 OS << "<nullptr>";
1860 return;
1861 }
1862 BB->printAsOperand(O&: OS);
1863 });
1864}
1865