1//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions perform manipulations on basic blocks, and
10// instructions contained within basic blocks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/BasicBlockUtils.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SmallPtrSet.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/CFG.h"
20#include "llvm/Analysis/DomTreeUpdater.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/Analysis/MemoryDependenceAnalysis.h"
23#include "llvm/Analysis/MemorySSAUpdater.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/CFG.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DebugInfo.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/Dominators.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicInst.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/LLVMContext.h"
37#include "llvm/IR/Type.h"
38#include "llvm/IR/User.h"
39#include "llvm/IR/Value.h"
40#include "llvm/IR/ValueHandle.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/Utils/Local.h"
46#include <cassert>
47#include <cstdint>
48#include <string>
49#include <utility>
50#include <vector>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "basicblock-utils"
55
56static cl::opt<unsigned> MaxDeoptOrUnreachableSuccessorCheckDepth(
57 "max-deopt-or-unreachable-succ-check-depth", cl::init(Val: 8), cl::Hidden,
58 cl::desc("Set the maximum path length when checking whether a basic block "
59 "is followed by a block that either has a terminating "
60 "deoptimizing call or is terminated with an unreachable"));
61
62void llvm::detachDeadBlocks(
63 ArrayRef<BasicBlock *> BBs,
64 SmallVectorImpl<DominatorTree::UpdateType> *Updates,
65 bool KeepOneInputPHIs) {
66 for (auto *BB : BBs) {
67 // Loop through all of our successors and make sure they know that one
68 // of their predecessors is going away.
69 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
70 for (BasicBlock *Succ : successors(BB)) {
71 Succ->removePredecessor(Pred: BB, KeepOneInputPHIs);
72 if (Updates && UniqueSuccessors.insert(Ptr: Succ).second)
73 Updates->push_back(Elt: {DominatorTree::Delete, BB, Succ});
74 }
75
76 // Zap all the instructions in the block.
77 while (!BB->empty()) {
78 Instruction &I = BB->back();
79 // If this instruction is used, replace uses with an arbitrary value.
80 // Because control flow can't get here, we don't care what we replace the
81 // value with. Note that since this block is unreachable, and all values
82 // contained within it must dominate their uses, that all uses will
83 // eventually be removed (they are themselves dead).
84 if (!I.use_empty())
85 I.replaceAllUsesWith(V: PoisonValue::get(T: I.getType()));
86 BB->back().eraseFromParent();
87 }
88 new UnreachableInst(BB->getContext(), BB);
89 assert(BB->size() == 1 &&
90 isa<UnreachableInst>(BB->getTerminator()) &&
91 "The successor list of BB isn't empty before "
92 "applying corresponding DTU updates.");
93 }
94}
95
96void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
97 bool KeepOneInputPHIs) {
98 DeleteDeadBlocks(BBs: {BB}, DTU, KeepOneInputPHIs);
99}
100
101void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
102 bool KeepOneInputPHIs) {
103#ifndef NDEBUG
104 // Make sure that all predecessors of each dead block is also dead.
105 SmallPtrSet<BasicBlock *, 4> Dead(BBs.begin(), BBs.end());
106 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
107 for (auto *BB : Dead)
108 for (BasicBlock *Pred : predecessors(BB))
109 assert(Dead.count(Pred) && "All predecessors must be dead!");
110#endif
111
112 SmallVector<DominatorTree::UpdateType, 4> Updates;
113 detachDeadBlocks(BBs, Updates: DTU ? &Updates : nullptr, KeepOneInputPHIs);
114
115 if (DTU)
116 DTU->applyUpdates(Updates);
117
118 for (BasicBlock *BB : BBs)
119 if (DTU)
120 DTU->deleteBB(DelBB: BB);
121 else
122 BB->eraseFromParent();
123}
124
125bool llvm::EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
126 bool KeepOneInputPHIs) {
127 df_iterator_default_set<BasicBlock*> Reachable;
128
129 // Mark all reachable blocks.
130 for (BasicBlock *BB : depth_first_ext(G: &F, S&: Reachable))
131 (void)BB/* Mark all reachable blocks */;
132
133 // Collect all dead blocks.
134 std::vector<BasicBlock*> DeadBlocks;
135 for (BasicBlock &BB : F)
136 if (!Reachable.count(Ptr: &BB))
137 DeadBlocks.push_back(x: &BB);
138
139 // Delete the dead blocks.
140 DeleteDeadBlocks(BBs: DeadBlocks, DTU, KeepOneInputPHIs);
141
142 return !DeadBlocks.empty();
143}
144
145bool llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
146 MemoryDependenceResults *MemDep) {
147 if (!isa<PHINode>(Val: BB->begin()))
148 return false;
149
150 while (PHINode *PN = dyn_cast<PHINode>(Val: BB->begin())) {
151 if (PN->getIncomingValue(i: 0) != PN)
152 PN->replaceAllUsesWith(V: PN->getIncomingValue(i: 0));
153 else
154 PN->replaceAllUsesWith(V: PoisonValue::get(T: PN->getType()));
155
156 if (MemDep)
157 MemDep->removeInstruction(InstToRemove: PN); // Memdep updates AA itself.
158
159 PN->eraseFromParent();
160 }
161 return true;
162}
163
164bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI,
165 MemorySSAUpdater *MSSAU) {
166 // Recursively deleting a PHI may cause multiple PHIs to be deleted
167 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
168 SmallVector<WeakTrackingVH, 8> PHIs;
169 for (PHINode &PN : BB->phis())
170 PHIs.push_back(Elt: &PN);
171
172 bool Changed = false;
173 for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
174 if (PHINode *PN = dyn_cast_or_null<PHINode>(Val: PHIs[i].operator Value*()))
175 Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
176
177 return Changed;
178}
179
180bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU,
181 LoopInfo *LI, MemorySSAUpdater *MSSAU,
182 MemoryDependenceResults *MemDep,
183 bool PredecessorWithTwoSuccessors,
184 DominatorTree *DT) {
185 if (BB->hasAddressTaken())
186 return false;
187
188 // Can't merge if there are multiple predecessors, or no predecessors.
189 BasicBlock *PredBB = BB->getUniquePredecessor();
190 if (!PredBB) return false;
191
192 // Don't break self-loops.
193 if (PredBB == BB) return false;
194
195 // Don't break unwinding instructions or terminators with other side-effects.
196 Instruction *PTI = PredBB->getTerminator();
197 if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
198 return false;
199
200 // Can't merge if there are multiple distinct successors.
201 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
202 return false;
203
204 // Currently only allow PredBB to have two predecessors, one being BB.
205 // Update BI to branch to BB's only successor instead of BB.
206 BranchInst *PredBB_BI;
207 BasicBlock *NewSucc = nullptr;
208 unsigned FallThruPath;
209 if (PredecessorWithTwoSuccessors) {
210 if (!(PredBB_BI = dyn_cast<BranchInst>(Val: PTI)))
211 return false;
212 BranchInst *BB_JmpI = dyn_cast<BranchInst>(Val: BB->getTerminator());
213 if (!BB_JmpI || !BB_JmpI->isUnconditional())
214 return false;
215 NewSucc = BB_JmpI->getSuccessor(i: 0);
216 FallThruPath = PredBB_BI->getSuccessor(i: 0) == BB ? 0 : 1;
217 }
218
219 // Can't merge if there is PHI loop.
220 for (PHINode &PN : BB->phis())
221 if (llvm::is_contained(Range: PN.incoming_values(), Element: &PN))
222 return false;
223
224 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
225 << PredBB->getName() << "\n");
226
227 // Begin by getting rid of unneeded PHIs.
228 SmallVector<AssertingVH<Value>, 4> IncomingValues;
229 if (isa<PHINode>(Val: BB->front())) {
230 for (PHINode &PN : BB->phis())
231 if (!isa<PHINode>(Val: PN.getIncomingValue(i: 0)) ||
232 cast<PHINode>(Val: PN.getIncomingValue(i: 0))->getParent() != BB)
233 IncomingValues.push_back(Elt: PN.getIncomingValue(i: 0));
234 FoldSingleEntryPHINodes(BB, MemDep);
235 }
236
237 if (DT) {
238 assert(!DTU && "cannot use both DT and DTU for updates");
239 DomTreeNode *PredNode = DT->getNode(BB: PredBB);
240 DomTreeNode *BBNode = DT->getNode(BB);
241 if (PredNode) {
242 assert(BBNode && "PredNode unreachable but BBNode reachable?");
243 for (DomTreeNode *C : to_vector(Range: BBNode->children()))
244 C->setIDom(PredNode);
245 }
246 }
247 // DTU update: Collect all the edges that exit BB.
248 // These dominator edges will be redirected from Pred.
249 std::vector<DominatorTree::UpdateType> Updates;
250 if (DTU) {
251 assert(!DT && "cannot use both DT and DTU for updates");
252 // To avoid processing the same predecessor more than once.
253 SmallPtrSet<BasicBlock *, 8> SeenSuccs;
254 SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(succ_begin(BB: PredBB),
255 succ_end(BB: PredBB));
256 Updates.reserve(n: Updates.size() + 2 * succ_size(BB) + 1);
257 // Add insert edges first. Experimentally, for the particular case of two
258 // blocks that can be merged, with a single successor and single predecessor
259 // respectively, it is beneficial to have all insert updates first. Deleting
260 // edges first may lead to unreachable blocks, followed by inserting edges
261 // making the blocks reachable again. Such DT updates lead to high compile
262 // times. We add inserts before deletes here to reduce compile time.
263 for (BasicBlock *SuccOfBB : successors(BB))
264 // This successor of BB may already be a PredBB's successor.
265 if (!SuccsOfPredBB.contains(Ptr: SuccOfBB))
266 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
267 Updates.push_back(x: {DominatorTree::Insert, PredBB, SuccOfBB});
268 SeenSuccs.clear();
269 for (BasicBlock *SuccOfBB : successors(BB))
270 if (SeenSuccs.insert(Ptr: SuccOfBB).second)
271 Updates.push_back(x: {DominatorTree::Delete, BB, SuccOfBB});
272 Updates.push_back(x: {DominatorTree::Delete, PredBB, BB});
273 }
274
275 Instruction *STI = BB->getTerminator();
276 Instruction *Start = &*BB->begin();
277 // If there's nothing to move, mark the starting instruction as the last
278 // instruction in the block. Terminator instruction is handled separately.
279 if (Start == STI)
280 Start = PTI;
281
282 // Move all definitions in the successor to the predecessor...
283 PredBB->splice(ToIt: PTI->getIterator(), FromBB: BB, FromBeginIt: BB->begin(), FromEndIt: STI->getIterator());
284
285 if (MSSAU)
286 MSSAU->moveAllAfterMergeBlocks(From: BB, To: PredBB, Start);
287
288 // Make all PHI nodes that referred to BB now refer to Pred as their
289 // source...
290 BB->replaceAllUsesWith(V: PredBB);
291
292 if (PredecessorWithTwoSuccessors) {
293 // Delete the unconditional branch from BB.
294 BB->back().eraseFromParent();
295
296 // Update branch in the predecessor.
297 PredBB_BI->setSuccessor(idx: FallThruPath, NewSucc);
298 } else {
299 // Delete the unconditional branch from the predecessor.
300 PredBB->back().eraseFromParent();
301
302 // Move terminator instruction.
303 BB->back().moveBeforePreserving(BB&: *PredBB, I: PredBB->end());
304
305 // Terminator may be a memory accessing instruction too.
306 if (MSSAU)
307 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
308 Val: MSSAU->getMemorySSA()->getMemoryAccess(I: PredBB->getTerminator())))
309 MSSAU->moveToPlace(What: MUD, BB: PredBB, Where: MemorySSA::End);
310 }
311 // Add unreachable to now empty BB.
312 new UnreachableInst(BB->getContext(), BB);
313
314 // Inherit predecessors name if it exists.
315 if (!PredBB->hasName())
316 PredBB->takeName(V: BB);
317
318 if (LI)
319 LI->removeBlock(BB);
320
321 if (MemDep)
322 MemDep->invalidateCachedPredecessors();
323
324 if (DTU)
325 DTU->applyUpdates(Updates);
326
327 if (DT) {
328 assert(succ_empty(BB) &&
329 "successors should have been transferred to PredBB");
330 DT->eraseNode(BB);
331 }
332
333 // Finally, erase the old block and update dominator info.
334 DeleteDeadBlock(BB, DTU);
335
336 return true;
337}
338
339bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
340 SmallPtrSetImpl<BasicBlock *> &MergeBlocks, Loop *L, DomTreeUpdater *DTU,
341 LoopInfo *LI) {
342 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
343
344 bool BlocksHaveBeenMerged = false;
345 while (!MergeBlocks.empty()) {
346 BasicBlock *BB = *MergeBlocks.begin();
347 BasicBlock *Dest = BB->getSingleSuccessor();
348 if (Dest && (!L || L->contains(BB: Dest))) {
349 BasicBlock *Fold = Dest->getUniquePredecessor();
350 (void)Fold;
351 if (MergeBlockIntoPredecessor(BB: Dest, DTU, LI)) {
352 assert(Fold == BB &&
353 "Expecting BB to be unique predecessor of the Dest block");
354 MergeBlocks.erase(Ptr: Dest);
355 BlocksHaveBeenMerged = true;
356 } else
357 MergeBlocks.erase(Ptr: BB);
358 } else
359 MergeBlocks.erase(Ptr: BB);
360 }
361 return BlocksHaveBeenMerged;
362}
363
364/// Remove redundant instructions within sequences of consecutive dbg.value
365/// instructions. This is done using a backward scan to keep the last dbg.value
366/// describing a specific variable/fragment.
367///
368/// BackwardScan strategy:
369/// ----------------------
370/// Given a sequence of consecutive DbgValueInst like this
371///
372/// dbg.value ..., "x", FragmentX1 (*)
373/// dbg.value ..., "y", FragmentY1
374/// dbg.value ..., "x", FragmentX2
375/// dbg.value ..., "x", FragmentX1 (**)
376///
377/// then the instruction marked with (*) can be removed (it is guaranteed to be
378/// obsoleted by the instruction marked with (**) as the latter instruction is
379/// describing the same variable using the same fragment info).
380///
381/// Possible improvements:
382/// - Check fully overlapping fragments and not only identical fragments.
383/// - Support dbg.declare. dbg.label, and possibly other meta instructions being
384/// part of the sequence of consecutive instructions.
385static bool
386DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
387 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
388 SmallDenseSet<DebugVariable> VariableSet;
389 for (auto &I : reverse(C&: *BB)) {
390 for (DbgRecord &DR : reverse(C: I.getDbgRecordRange())) {
391 if (isa<DbgLabelRecord>(Val: DR)) {
392 // Emulate existing behaviour (see comment below for dbg.declares).
393 // FIXME: Don't do this.
394 VariableSet.clear();
395 continue;
396 }
397
398 DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
399 // Skip declare-type records, as the debug intrinsic method only works
400 // on dbg.value intrinsics.
401 if (DVR.getType() == DbgVariableRecord::LocationType::Declare) {
402 // The debug intrinsic method treats dbg.declares are "non-debug"
403 // instructions (i.e., a break in a consecutive range of debug
404 // intrinsics). Emulate that to create identical outputs. See
405 // "Possible improvements" above.
406 // FIXME: Delete the line below.
407 VariableSet.clear();
408 continue;
409 }
410
411 DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
412 DVR.getDebugLoc()->getInlinedAt());
413 auto R = VariableSet.insert(V: Key);
414 // If the same variable fragment is described more than once it is enough
415 // to keep the last one (i.e. the first found since we for reverse
416 // iteration).
417 if (R.second)
418 continue;
419
420 if (DVR.isDbgAssign()) {
421 // Don't delete dbg.assign intrinsics that are linked to instructions.
422 if (!at::getAssignmentInsts(DVR: &DVR).empty())
423 continue;
424 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
425 }
426
427 ToBeRemoved.push_back(Elt: &DVR);
428 continue;
429 }
430 // Sequence with consecutive dbg.value instrs ended. Clear the map to
431 // restart identifying redundant instructions if case we find another
432 // dbg.value sequence.
433 VariableSet.clear();
434 }
435
436 for (auto &DVR : ToBeRemoved)
437 DVR->eraseFromParent();
438
439 return !ToBeRemoved.empty();
440}
441
442static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
443 if (BB->IsNewDbgInfoFormat)
444 return DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BB);
445
446 SmallVector<DbgValueInst *, 8> ToBeRemoved;
447 SmallDenseSet<DebugVariable> VariableSet;
448 for (auto &I : reverse(C&: *BB)) {
449 if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(Val: &I)) {
450 DebugVariable Key(DVI->getVariable(),
451 DVI->getExpression(),
452 DVI->getDebugLoc()->getInlinedAt());
453 auto R = VariableSet.insert(V: Key);
454 // If the variable fragment hasn't been seen before then we don't want
455 // to remove this dbg intrinsic.
456 if (R.second)
457 continue;
458
459 if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(Val: DVI)) {
460 // Don't delete dbg.assign intrinsics that are linked to instructions.
461 if (!at::getAssignmentInsts(DAI).empty())
462 continue;
463 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
464 }
465
466 // If the same variable fragment is described more than once it is enough
467 // to keep the last one (i.e. the first found since we for reverse
468 // iteration).
469 ToBeRemoved.push_back(Elt: DVI);
470 continue;
471 }
472 // Sequence with consecutive dbg.value instrs ended. Clear the map to
473 // restart identifying redundant instructions if case we find another
474 // dbg.value sequence.
475 VariableSet.clear();
476 }
477
478 for (auto &Instr : ToBeRemoved)
479 Instr->eraseFromParent();
480
481 return !ToBeRemoved.empty();
482}
483
484/// Remove redundant dbg.value instructions using a forward scan. This can
485/// remove a dbg.value instruction that is redundant due to indicating that a
486/// variable has the same value as already being indicated by an earlier
487/// dbg.value.
488///
489/// ForwardScan strategy:
490/// ---------------------
491/// Given two identical dbg.value instructions, separated by a block of
492/// instructions that isn't describing the same variable, like this
493///
494/// dbg.value X1, "x", FragmentX1 (**)
495/// <block of instructions, none being "dbg.value ..., "x", ...">
496/// dbg.value X1, "x", FragmentX1 (*)
497///
498/// then the instruction marked with (*) can be removed. Variable "x" is already
499/// described as being mapped to the SSA value X1.
500///
501/// Possible improvements:
502/// - Keep track of non-overlapping fragments.
503static bool
504DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
505 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
506 DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
507 VariableMap;
508 for (auto &I : *BB) {
509 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
510 if (DVR.getType() == DbgVariableRecord::LocationType::Declare)
511 continue;
512 DebugVariable Key(DVR.getVariable(), std::nullopt,
513 DVR.getDebugLoc()->getInlinedAt());
514 auto VMI = VariableMap.find(Val: Key);
515 // A dbg.assign with no linked instructions can be treated like a
516 // dbg.value (i.e. can be deleted).
517 bool IsDbgValueKind =
518 (!DVR.isDbgAssign() || at::getAssignmentInsts(DVR: &DVR).empty());
519
520 // Update the map if we found a new value/expression describing the
521 // variable, or if the variable wasn't mapped already.
522 SmallVector<Value *, 4> Values(DVR.location_ops());
523 if (VMI == VariableMap.end() || VMI->second.first != Values ||
524 VMI->second.second != DVR.getExpression()) {
525 if (IsDbgValueKind)
526 VariableMap[Key] = {Values, DVR.getExpression()};
527 else
528 VariableMap[Key] = {Values, nullptr};
529 continue;
530 }
531 // Don't delete dbg.assign intrinsics that are linked to instructions.
532 if (!IsDbgValueKind)
533 continue;
534 // Found an identical mapping. Remember the instruction for later removal.
535 ToBeRemoved.push_back(Elt: &DVR);
536 }
537 }
538
539 for (auto *DVR : ToBeRemoved)
540 DVR->eraseFromParent();
541
542 return !ToBeRemoved.empty();
543}
544
545static bool
546DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
547 assert(BB->isEntryBlock() && "expected entry block");
548 SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
549 DenseSet<DebugVariable> SeenDefForAggregate;
550 // Returns the DebugVariable for DVI with no fragment info.
551 auto GetAggregateVariable = [](const DbgVariableRecord &DVR) {
552 return DebugVariable(DVR.getVariable(), std::nullopt,
553 DVR.getDebugLoc().getInlinedAt());
554 };
555
556 // Remove undef dbg.assign intrinsics that are encountered before
557 // any non-undef intrinsics from the entry block.
558 for (auto &I : *BB) {
559 for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
560 if (!DVR.isDbgValue() && !DVR.isDbgAssign())
561 continue;
562 bool IsDbgValueKind =
563 (DVR.isDbgValue() || at::getAssignmentInsts(DVR: &DVR).empty());
564 DebugVariable Aggregate = GetAggregateVariable(DVR);
565 if (!SeenDefForAggregate.contains(V: Aggregate)) {
566 bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
567 if (!IsKill) {
568 SeenDefForAggregate.insert(V: Aggregate);
569 } else if (DVR.isDbgAssign()) {
570 ToBeRemoved.push_back(Elt: &DVR);
571 }
572 }
573 }
574 }
575
576 for (DbgVariableRecord *DVR : ToBeRemoved)
577 DVR->eraseFromParent();
578
579 return !ToBeRemoved.empty();
580}
581
582static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
583 if (BB->IsNewDbgInfoFormat)
584 return DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BB);
585
586 SmallVector<DbgValueInst *, 8> ToBeRemoved;
587 DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
588 VariableMap;
589 for (auto &I : *BB) {
590 if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(Val: &I)) {
591 DebugVariable Key(DVI->getVariable(), std::nullopt,
592 DVI->getDebugLoc()->getInlinedAt());
593 auto VMI = VariableMap.find(Val: Key);
594 auto *DAI = dyn_cast<DbgAssignIntrinsic>(Val: DVI);
595 // A dbg.assign with no linked instructions can be treated like a
596 // dbg.value (i.e. can be deleted).
597 bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
598
599 // Update the map if we found a new value/expression describing the
600 // variable, or if the variable wasn't mapped already.
601 SmallVector<Value *, 4> Values(DVI->getValues());
602 if (VMI == VariableMap.end() || VMI->second.first != Values ||
603 VMI->second.second != DVI->getExpression()) {
604 // Use a sentinel value (nullptr) for the DIExpression when we see a
605 // linked dbg.assign so that the next debug intrinsic will never match
606 // it (i.e. always treat linked dbg.assigns as if they're unique).
607 if (IsDbgValueKind)
608 VariableMap[Key] = {Values, DVI->getExpression()};
609 else
610 VariableMap[Key] = {Values, nullptr};
611 continue;
612 }
613
614 // Don't delete dbg.assign intrinsics that are linked to instructions.
615 if (!IsDbgValueKind)
616 continue;
617 ToBeRemoved.push_back(Elt: DVI);
618 }
619 }
620
621 for (auto &Instr : ToBeRemoved)
622 Instr->eraseFromParent();
623
624 return !ToBeRemoved.empty();
625}
626
627/// Remove redundant undef dbg.assign intrinsic from an entry block using a
628/// forward scan.
629/// Strategy:
630/// ---------------------
631/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
632/// linked to an intrinsic, and don't share an aggregate variable with a debug
633/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
634/// that come before non-undef debug intrinsics for the variable are
635/// deleted. Given:
636///
637/// dbg.assign undef, "x", FragmentX1 (*)
638/// <block of instructions, none being "dbg.value ..., "x", ...">
639/// dbg.value %V, "x", FragmentX2
640/// <block of instructions, none being "dbg.value ..., "x", ...">
641/// dbg.assign undef, "x", FragmentX1
642///
643/// then (only) the instruction marked with (*) can be removed.
644/// Possible improvements:
645/// - Keep track of non-overlapping fragments.
646static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
647 if (BB->IsNewDbgInfoFormat)
648 return DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BB);
649
650 assert(BB->isEntryBlock() && "expected entry block");
651 SmallVector<DbgAssignIntrinsic *, 8> ToBeRemoved;
652 DenseSet<DebugVariable> SeenDefForAggregate;
653 // Returns the DebugVariable for DVI with no fragment info.
654 auto GetAggregateVariable = [](DbgValueInst *DVI) {
655 return DebugVariable(DVI->getVariable(), std::nullopt,
656 DVI->getDebugLoc()->getInlinedAt());
657 };
658
659 // Remove undef dbg.assign intrinsics that are encountered before
660 // any non-undef intrinsics from the entry block.
661 for (auto &I : *BB) {
662 DbgValueInst *DVI = dyn_cast<DbgValueInst>(Val: &I);
663 if (!DVI)
664 continue;
665 auto *DAI = dyn_cast<DbgAssignIntrinsic>(Val: DVI);
666 bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
667 DebugVariable Aggregate = GetAggregateVariable(DVI);
668 if (!SeenDefForAggregate.contains(V: Aggregate)) {
669 bool IsKill = DVI->isKillLocation() && IsDbgValueKind;
670 if (!IsKill) {
671 SeenDefForAggregate.insert(V: Aggregate);
672 } else if (DAI) {
673 ToBeRemoved.push_back(Elt: DAI);
674 }
675 }
676 }
677
678 for (DbgAssignIntrinsic *DAI : ToBeRemoved)
679 DAI->eraseFromParent();
680
681 return !ToBeRemoved.empty();
682}
683
684bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
685 bool MadeChanges = false;
686 // By using the "backward scan" strategy before the "forward scan" strategy we
687 // can remove both dbg.value (2) and (3) in a situation like this:
688 //
689 // (1) dbg.value V1, "x", DIExpression()
690 // ...
691 // (2) dbg.value V2, "x", DIExpression()
692 // (3) dbg.value V1, "x", DIExpression()
693 //
694 // The backward scan will remove (2), it is made obsolete by (3). After
695 // getting (2) out of the way, the foward scan will remove (3) since "x"
696 // already is described as having the value V1 at (1).
697 MadeChanges |= removeRedundantDbgInstrsUsingBackwardScan(BB);
698 if (BB->isEntryBlock() &&
699 isAssignmentTrackingEnabled(M: *BB->getParent()->getParent()))
700 MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
701 MadeChanges |= removeRedundantDbgInstrsUsingForwardScan(BB);
702
703 if (MadeChanges)
704 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
705 << BB->getName() << "\n");
706 return MadeChanges;
707}
708
709void llvm::ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V) {
710 Instruction &I = *BI;
711 // Replaces all of the uses of the instruction with uses of the value
712 I.replaceAllUsesWith(V);
713
714 // Make sure to propagate a name if there is one already.
715 if (I.hasName() && !V->hasName())
716 V->takeName(V: &I);
717
718 // Delete the unnecessary instruction now...
719 BI = BI->eraseFromParent();
720}
721
722void llvm::ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI,
723 Instruction *I) {
724 assert(I->getParent() == nullptr &&
725 "ReplaceInstWithInst: Instruction already inserted into basic block!");
726
727 // Copy debug location to newly added instruction, if it wasn't already set
728 // by the caller.
729 if (!I->getDebugLoc())
730 I->setDebugLoc(BI->getDebugLoc());
731
732 // Insert the new instruction into the basic block...
733 BasicBlock::iterator New = I->insertInto(ParentBB: BB, It: BI);
734
735 // Replace all uses of the old instruction, and delete it.
736 ReplaceInstWithValue(BI, V: I);
737
738 // Move BI back to point to the newly inserted instruction
739 BI = New;
740}
741
742bool llvm::IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB) {
743 // Remember visited blocks to avoid infinite loop
744 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
745 unsigned Depth = 0;
746 while (BB && Depth++ < MaxDeoptOrUnreachableSuccessorCheckDepth &&
747 VisitedBlocks.insert(Ptr: BB).second) {
748 if (isa<UnreachableInst>(Val: BB->getTerminator()) ||
749 BB->getTerminatingDeoptimizeCall())
750 return true;
751 BB = BB->getUniqueSuccessor();
752 }
753 return false;
754}
755
756void llvm::ReplaceInstWithInst(Instruction *From, Instruction *To) {
757 BasicBlock::iterator BI(From);
758 ReplaceInstWithInst(BB: From->getParent(), BI, I: To);
759}
760
761BasicBlock *llvm::SplitEdge(BasicBlock *BB, BasicBlock *Succ, DominatorTree *DT,
762 LoopInfo *LI, MemorySSAUpdater *MSSAU,
763 const Twine &BBName) {
764 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
765
766 Instruction *LatchTerm = BB->getTerminator();
767
768 CriticalEdgeSplittingOptions Options =
769 CriticalEdgeSplittingOptions(DT, LI, MSSAU).setPreserveLCSSA();
770
771 if ((isCriticalEdge(TI: LatchTerm, SuccNum, AllowIdenticalEdges: Options.MergeIdenticalEdges))) {
772 // If it is a critical edge, and the succesor is an exception block, handle
773 // the split edge logic in this specific function
774 if (Succ->isEHPad())
775 return ehAwareSplitEdge(BB, Succ, OriginalPad: nullptr, LandingPadReplacement: nullptr, Options, BBName);
776
777 // If this is a critical edge, let SplitKnownCriticalEdge do it.
778 return SplitKnownCriticalEdge(TI: LatchTerm, SuccNum, Options, BBName);
779 }
780
781 // If the edge isn't critical, then BB has a single successor or Succ has a
782 // single pred. Split the block.
783 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
784 // If the successor only has a single pred, split the top of the successor
785 // block.
786 assert(SP == BB && "CFG broken");
787 (void)SP;
788 return SplitBlock(Old: Succ, SplitPt: &Succ->front(), DT, LI, MSSAU, BBName,
789 /*Before=*/true);
790 }
791
792 // Otherwise, if BB has a single successor, split it at the bottom of the
793 // block.
794 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
795 "Should have a single succ!");
796 return SplitBlock(Old: BB, SplitPt: BB->getTerminator(), DT, LI, MSSAU, BBName);
797}
798
799void llvm::setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ) {
800 if (auto *II = dyn_cast<InvokeInst>(Val: TI))
801 II->setUnwindDest(Succ);
802 else if (auto *CS = dyn_cast<CatchSwitchInst>(Val: TI))
803 CS->setUnwindDest(Succ);
804 else if (auto *CR = dyn_cast<CleanupReturnInst>(Val: TI))
805 CR->setUnwindDest(Succ);
806 else
807 llvm_unreachable("unexpected terminator instruction");
808}
809
810void llvm::updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred,
811 BasicBlock *NewPred, PHINode *Until) {
812 int BBIdx = 0;
813 for (PHINode &PN : DestBB->phis()) {
814 // We manually update the LandingPadReplacement PHINode and it is the last
815 // PHI Node. So, if we find it, we are done.
816 if (Until == &PN)
817 break;
818
819 // Reuse the previous value of BBIdx if it lines up. In cases where we
820 // have multiple phi nodes with *lots* of predecessors, this is a speed
821 // win because we don't have to scan the PHI looking for TIBB. This
822 // happens because the BB list of PHI nodes are usually in the same
823 // order.
824 if (PN.getIncomingBlock(i: BBIdx) != OldPred)
825 BBIdx = PN.getBasicBlockIndex(BB: OldPred);
826
827 assert(BBIdx != -1 && "Invalid PHI Index!");
828 PN.setIncomingBlock(i: BBIdx, BB: NewPred);
829 }
830}
831
832BasicBlock *llvm::ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ,
833 LandingPadInst *OriginalPad,
834 PHINode *LandingPadReplacement,
835 const CriticalEdgeSplittingOptions &Options,
836 const Twine &BBName) {
837
838 auto *PadInst = Succ->getFirstNonPHI();
839 if (!LandingPadReplacement && !PadInst->isEHPad())
840 return SplitEdge(BB, Succ, DT: Options.DT, LI: Options.LI, MSSAU: Options.MSSAU, BBName);
841
842 auto *LI = Options.LI;
843 SmallVector<BasicBlock *, 4> LoopPreds;
844 // Check if extra modifications will be required to preserve loop-simplify
845 // form after splitting. If it would require splitting blocks with IndirectBr
846 // terminators, bail out if preserving loop-simplify form is requested.
847 if (Options.PreserveLoopSimplify && LI) {
848 if (Loop *BBLoop = LI->getLoopFor(BB)) {
849
850 // The only way that we can break LoopSimplify form by splitting a
851 // critical edge is when there exists some edge from BBLoop to Succ *and*
852 // the only edge into Succ from outside of BBLoop is that of NewBB after
853 // the split. If the first isn't true, then LoopSimplify still holds,
854 // NewBB is the new exit block and it has no non-loop predecessors. If the
855 // second isn't true, then Succ was not in LoopSimplify form prior to
856 // the split as it had a non-loop predecessor. In both of these cases,
857 // the predecessor must be directly in BBLoop, not in a subloop, or again
858 // LoopSimplify doesn't hold.
859 for (BasicBlock *P : predecessors(BB: Succ)) {
860 if (P == BB)
861 continue; // The new block is known.
862 if (LI->getLoopFor(BB: P) != BBLoop) {
863 // Loop is not in LoopSimplify form, no need to re simplify after
864 // splitting edge.
865 LoopPreds.clear();
866 break;
867 }
868 LoopPreds.push_back(Elt: P);
869 }
870 // Loop-simplify form can be preserved, if we can split all in-loop
871 // predecessors.
872 if (any_of(Range&: LoopPreds, P: [](BasicBlock *Pred) {
873 return isa<IndirectBrInst>(Val: Pred->getTerminator());
874 })) {
875 return nullptr;
876 }
877 }
878 }
879
880 auto *NewBB =
881 BasicBlock::Create(Context&: BB->getContext(), Name: BBName, Parent: BB->getParent(), InsertBefore: Succ);
882 setUnwindEdgeTo(TI: BB->getTerminator(), Succ: NewBB);
883 updatePhiNodes(DestBB: Succ, OldPred: BB, NewPred: NewBB, Until: LandingPadReplacement);
884
885 if (LandingPadReplacement) {
886 auto *NewLP = OriginalPad->clone();
887 auto *Terminator = BranchInst::Create(IfTrue: Succ, InsertBefore: NewBB);
888 NewLP->insertBefore(InsertPos: Terminator);
889 LandingPadReplacement->addIncoming(V: NewLP, BB: NewBB);
890 } else {
891 Value *ParentPad = nullptr;
892 if (auto *FuncletPad = dyn_cast<FuncletPadInst>(Val: PadInst))
893 ParentPad = FuncletPad->getParentPad();
894 else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Val: PadInst))
895 ParentPad = CatchSwitch->getParentPad();
896 else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(Val: PadInst))
897 ParentPad = CleanupPad->getParentPad();
898 else if (auto *LandingPad = dyn_cast<LandingPadInst>(Val: PadInst))
899 ParentPad = LandingPad->getParent();
900 else
901 llvm_unreachable("handling for other EHPads not implemented yet");
902
903 auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, Args: {}, NameStr: BBName, InsertBefore: NewBB);
904 CleanupReturnInst::Create(CleanupPad: NewCleanupPad, UnwindBB: Succ, InsertBefore: NewBB);
905 }
906
907 auto *DT = Options.DT;
908 auto *MSSAU = Options.MSSAU;
909 if (!DT && !LI)
910 return NewBB;
911
912 if (DT) {
913 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
914 SmallVector<DominatorTree::UpdateType, 3> Updates;
915
916 Updates.push_back(Elt: {DominatorTree::Insert, BB, NewBB});
917 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, Succ});
918 Updates.push_back(Elt: {DominatorTree::Delete, BB, Succ});
919
920 DTU.applyUpdates(Updates);
921 DTU.flush();
922
923 if (MSSAU) {
924 MSSAU->applyUpdates(Updates, DT&: *DT);
925 if (VerifyMemorySSA)
926 MSSAU->getMemorySSA()->verifyMemorySSA();
927 }
928 }
929
930 if (LI) {
931 if (Loop *BBLoop = LI->getLoopFor(BB)) {
932 // If one or the other blocks were not in a loop, the new block is not
933 // either, and thus LI doesn't need to be updated.
934 if (Loop *SuccLoop = LI->getLoopFor(BB: Succ)) {
935 if (BBLoop == SuccLoop) {
936 // Both in the same loop, the NewBB joins loop.
937 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
938 } else if (BBLoop->contains(L: SuccLoop)) {
939 // Edge from an outer loop to an inner loop. Add to the outer loop.
940 BBLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
941 } else if (SuccLoop->contains(L: BBLoop)) {
942 // Edge from an inner loop to an outer loop. Add to the outer loop.
943 SuccLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
944 } else {
945 // Edge from two loops with no containment relation. Because these
946 // are natural loops, we know that the destination block must be the
947 // header of its loop (adding a branch into a loop elsewhere would
948 // create an irreducible loop).
949 assert(SuccLoop->getHeader() == Succ &&
950 "Should not create irreducible loops!");
951 if (Loop *P = SuccLoop->getParentLoop())
952 P->addBasicBlockToLoop(NewBB, LI&: *LI);
953 }
954 }
955
956 // If BB is in a loop and Succ is outside of that loop, we may need to
957 // update LoopSimplify form and LCSSA form.
958 if (!BBLoop->contains(BB: Succ)) {
959 assert(!BBLoop->contains(NewBB) &&
960 "Split point for loop exit is contained in loop!");
961
962 // Update LCSSA form in the newly created exit block.
963 if (Options.PreserveLCSSA) {
964 createPHIsForSplitLoopExit(Preds: BB, SplitBB: NewBB, DestBB: Succ);
965 }
966
967 if (!LoopPreds.empty()) {
968 BasicBlock *NewExitBB = SplitBlockPredecessors(
969 BB: Succ, Preds: LoopPreds, Suffix: "split", DT, LI, MSSAU, PreserveLCSSA: Options.PreserveLCSSA);
970 if (Options.PreserveLCSSA)
971 createPHIsForSplitLoopExit(Preds: LoopPreds, SplitBB: NewExitBB, DestBB: Succ);
972 }
973 }
974 }
975 }
976
977 return NewBB;
978}
979
980void llvm::createPHIsForSplitLoopExit(ArrayRef<BasicBlock *> Preds,
981 BasicBlock *SplitBB, BasicBlock *DestBB) {
982 // SplitBB shouldn't have anything non-trivial in it yet.
983 assert((SplitBB->getFirstNonPHI() == SplitBB->getTerminator() ||
984 SplitBB->isLandingPad()) &&
985 "SplitBB has non-PHI nodes!");
986
987 // For each PHI in the destination block.
988 for (PHINode &PN : DestBB->phis()) {
989 int Idx = PN.getBasicBlockIndex(BB: SplitBB);
990 assert(Idx >= 0 && "Invalid Block Index");
991 Value *V = PN.getIncomingValue(i: Idx);
992
993 // If the input is a PHI which already satisfies LCSSA, don't create
994 // a new one.
995 if (const PHINode *VP = dyn_cast<PHINode>(Val: V))
996 if (VP->getParent() == SplitBB)
997 continue;
998
999 // Otherwise a new PHI is needed. Create one and populate it.
1000 PHINode *NewPN = PHINode::Create(Ty: PN.getType(), NumReservedValues: Preds.size(), NameStr: "split");
1001 BasicBlock::iterator InsertPos =
1002 SplitBB->isLandingPad() ? SplitBB->begin()
1003 : SplitBB->getTerminator()->getIterator();
1004 NewPN->insertBefore(InsertPos);
1005 for (BasicBlock *BB : Preds)
1006 NewPN->addIncoming(V, BB);
1007
1008 // Update the original PHI.
1009 PN.setIncomingValue(i: Idx, V: NewPN);
1010 }
1011}
1012
1013unsigned
1014llvm::SplitAllCriticalEdges(Function &F,
1015 const CriticalEdgeSplittingOptions &Options) {
1016 unsigned NumBroken = 0;
1017 for (BasicBlock &BB : F) {
1018 Instruction *TI = BB.getTerminator();
1019 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(Val: TI))
1020 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
1021 if (SplitCriticalEdge(TI, SuccNum: i, Options))
1022 ++NumBroken;
1023 }
1024 return NumBroken;
1025}
1026
1027static BasicBlock *SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt,
1028 DomTreeUpdater *DTU, DominatorTree *DT,
1029 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1030 const Twine &BBName, bool Before) {
1031 if (Before) {
1032 DomTreeUpdater LocalDTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
1033 return splitBlockBefore(Old, SplitPt,
1034 DTU: DTU ? DTU : (DT ? &LocalDTU : nullptr), LI, MSSAU,
1035 BBName);
1036 }
1037 BasicBlock::iterator SplitIt = SplitPt;
1038 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad()) {
1039 ++SplitIt;
1040 assert(SplitIt != SplitPt->getParent()->end());
1041 }
1042 std::string Name = BBName.str();
1043 BasicBlock *New = Old->splitBasicBlock(
1044 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name);
1045
1046 // The new block lives in whichever loop the old one did. This preserves
1047 // LCSSA as well, because we force the split point to be after any PHI nodes.
1048 if (LI)
1049 if (Loop *L = LI->getLoopFor(BB: Old))
1050 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
1051
1052 if (DTU) {
1053 SmallVector<DominatorTree::UpdateType, 8> Updates;
1054 // Old dominates New. New node dominates all other nodes dominated by Old.
1055 SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
1056 Updates.push_back(Elt: {DominatorTree::Insert, Old, New});
1057 Updates.reserve(N: Updates.size() + 2 * succ_size(BB: New));
1058 for (BasicBlock *SuccessorOfOld : successors(BB: New))
1059 if (UniqueSuccessorsOfOld.insert(Ptr: SuccessorOfOld).second) {
1060 Updates.push_back(Elt: {DominatorTree::Insert, New, SuccessorOfOld});
1061 Updates.push_back(Elt: {DominatorTree::Delete, Old, SuccessorOfOld});
1062 }
1063
1064 DTU->applyUpdates(Updates);
1065 } else if (DT)
1066 // Old dominates New. New node dominates all other nodes dominated by Old.
1067 if (DomTreeNode *OldNode = DT->getNode(BB: Old)) {
1068 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1069
1070 DomTreeNode *NewNode = DT->addNewBlock(BB: New, DomBB: Old);
1071 for (DomTreeNode *I : Children)
1072 DT->changeImmediateDominator(N: I, NewIDom: NewNode);
1073 }
1074
1075 // Move MemoryAccesses still tracked in Old, but part of New now.
1076 // Update accesses in successor blocks accordingly.
1077 if (MSSAU)
1078 MSSAU->moveAllAfterSpliceBlocks(From: Old, To: New, Start: &*(New->begin()));
1079
1080 return New;
1081}
1082
1083BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1084 DominatorTree *DT, LoopInfo *LI,
1085 MemorySSAUpdater *MSSAU, const Twine &BBName,
1086 bool Before) {
1087 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName,
1088 Before);
1089}
1090BasicBlock *llvm::SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt,
1091 DomTreeUpdater *DTU, LoopInfo *LI,
1092 MemorySSAUpdater *MSSAU, const Twine &BBName,
1093 bool Before) {
1094 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName,
1095 Before);
1096}
1097
1098BasicBlock *llvm::splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt,
1099 DomTreeUpdater *DTU, LoopInfo *LI,
1100 MemorySSAUpdater *MSSAU,
1101 const Twine &BBName) {
1102
1103 BasicBlock::iterator SplitIt = SplitPt;
1104 while (isa<PHINode>(Val: SplitIt) || SplitIt->isEHPad())
1105 ++SplitIt;
1106 std::string Name = BBName.str();
1107 BasicBlock *New = Old->splitBasicBlock(
1108 I: SplitIt, BBName: Name.empty() ? Old->getName() + ".split" : Name,
1109 /* Before=*/true);
1110
1111 // The new block lives in whichever loop the old one did. This preserves
1112 // LCSSA as well, because we force the split point to be after any PHI nodes.
1113 if (LI)
1114 if (Loop *L = LI->getLoopFor(BB: Old))
1115 L->addBasicBlockToLoop(NewBB: New, LI&: *LI);
1116
1117 if (DTU) {
1118 SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
1119 // New dominates Old. The predecessor nodes of the Old node dominate
1120 // New node.
1121 SmallPtrSet<BasicBlock *, 8> UniquePredecessorsOfOld;
1122 DTUpdates.push_back(Elt: {DominatorTree::Insert, New, Old});
1123 DTUpdates.reserve(N: DTUpdates.size() + 2 * pred_size(BB: New));
1124 for (BasicBlock *PredecessorOfOld : predecessors(BB: New))
1125 if (UniquePredecessorsOfOld.insert(Ptr: PredecessorOfOld).second) {
1126 DTUpdates.push_back(Elt: {DominatorTree::Insert, PredecessorOfOld, New});
1127 DTUpdates.push_back(Elt: {DominatorTree::Delete, PredecessorOfOld, Old});
1128 }
1129
1130 DTU->applyUpdates(Updates: DTUpdates);
1131
1132 // Move MemoryAccesses still tracked in Old, but part of New now.
1133 // Update accesses in successor blocks accordingly.
1134 if (MSSAU) {
1135 MSSAU->applyUpdates(Updates: DTUpdates, DT&: DTU->getDomTree());
1136 if (VerifyMemorySSA)
1137 MSSAU->getMemorySSA()->verifyMemorySSA();
1138 }
1139 }
1140 return New;
1141}
1142
1143/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
1144/// Invalidates DFS Numbering when DTU or DT is provided.
1145static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
1146 ArrayRef<BasicBlock *> Preds,
1147 DomTreeUpdater *DTU, DominatorTree *DT,
1148 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1149 bool PreserveLCSSA, bool &HasLoopExit) {
1150 // Update dominator tree if available.
1151 if (DTU) {
1152 // Recalculation of DomTree is needed when updating a forward DomTree and
1153 // the Entry BB is replaced.
1154 if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1155 // The entry block was removed and there is no external interface for
1156 // the dominator tree to be notified of this change. In this corner-case
1157 // we recalculate the entire tree.
1158 DTU->recalculate(F&: *NewBB->getParent());
1159 } else {
1160 // Split block expects NewBB to have a non-empty set of predecessors.
1161 SmallVector<DominatorTree::UpdateType, 8> Updates;
1162 SmallPtrSet<BasicBlock *, 8> UniquePreds;
1163 Updates.push_back(Elt: {DominatorTree::Insert, NewBB, OldBB});
1164 Updates.reserve(N: Updates.size() + 2 * Preds.size());
1165 for (auto *Pred : Preds)
1166 if (UniquePreds.insert(Ptr: Pred).second) {
1167 Updates.push_back(Elt: {DominatorTree::Insert, Pred, NewBB});
1168 Updates.push_back(Elt: {DominatorTree::Delete, Pred, OldBB});
1169 }
1170 DTU->applyUpdates(Updates);
1171 }
1172 } else if (DT) {
1173 if (OldBB == DT->getRootNode()->getBlock()) {
1174 assert(NewBB->isEntryBlock());
1175 DT->setNewRoot(NewBB);
1176 } else {
1177 // Split block expects NewBB to have a non-empty set of predecessors.
1178 DT->splitBlock(NewBB);
1179 }
1180 }
1181
1182 // Update MemoryPhis after split if MemorySSA is available
1183 if (MSSAU)
1184 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(Old: OldBB, New: NewBB, Preds);
1185
1186 // The rest of the logic is only relevant for updating the loop structures.
1187 if (!LI)
1188 return;
1189
1190 if (DTU && DTU->hasDomTree())
1191 DT = &DTU->getDomTree();
1192 assert(DT && "DT should be available to update LoopInfo!");
1193 Loop *L = LI->getLoopFor(BB: OldBB);
1194
1195 // If we need to preserve loop analyses, collect some information about how
1196 // this split will affect loops.
1197 bool IsLoopEntry = !!L;
1198 bool SplitMakesNewLoopHeader = false;
1199 for (BasicBlock *Pred : Preds) {
1200 // Preds that are not reachable from entry should not be used to identify if
1201 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1202 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1203 // as true and make the NewBB the header of some loop. This breaks LI.
1204 if (!DT->isReachableFromEntry(A: Pred))
1205 continue;
1206 // If we need to preserve LCSSA, determine if any of the preds is a loop
1207 // exit.
1208 if (PreserveLCSSA)
1209 if (Loop *PL = LI->getLoopFor(BB: Pred))
1210 if (!PL->contains(BB: OldBB))
1211 HasLoopExit = true;
1212
1213 // If we need to preserve LoopInfo, note whether any of the preds crosses
1214 // an interesting loop boundary.
1215 if (!L)
1216 continue;
1217 if (L->contains(BB: Pred))
1218 IsLoopEntry = false;
1219 else
1220 SplitMakesNewLoopHeader = true;
1221 }
1222
1223 // Unless we have a loop for OldBB, nothing else to do here.
1224 if (!L)
1225 return;
1226
1227 if (IsLoopEntry) {
1228 // Add the new block to the nearest enclosing loop (and not an adjacent
1229 // loop). To find this, examine each of the predecessors and determine which
1230 // loops enclose them, and select the most-nested loop which contains the
1231 // loop containing the block being split.
1232 Loop *InnermostPredLoop = nullptr;
1233 for (BasicBlock *Pred : Preds) {
1234 if (Loop *PredLoop = LI->getLoopFor(BB: Pred)) {
1235 // Seek a loop which actually contains the block being split (to avoid
1236 // adjacent loops).
1237 while (PredLoop && !PredLoop->contains(BB: OldBB))
1238 PredLoop = PredLoop->getParentLoop();
1239
1240 // Select the most-nested of these loops which contains the block.
1241 if (PredLoop && PredLoop->contains(BB: OldBB) &&
1242 (!InnermostPredLoop ||
1243 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1244 InnermostPredLoop = PredLoop;
1245 }
1246 }
1247
1248 if (InnermostPredLoop)
1249 InnermostPredLoop->addBasicBlockToLoop(NewBB, LI&: *LI);
1250 } else {
1251 L->addBasicBlockToLoop(NewBB, LI&: *LI);
1252 if (SplitMakesNewLoopHeader)
1253 L->moveToHeader(BB: NewBB);
1254 }
1255}
1256
1257/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1258/// This also updates AliasAnalysis, if available.
1259static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1260 ArrayRef<BasicBlock *> Preds, BranchInst *BI,
1261 bool HasLoopExit) {
1262 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1263 SmallPtrSet<BasicBlock *, 16> PredSet(Preds.begin(), Preds.end());
1264 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(Val: I); ) {
1265 PHINode *PN = cast<PHINode>(Val: I++);
1266
1267 // Check to see if all of the values coming in are the same. If so, we
1268 // don't need to create a new PHI node, unless it's needed for LCSSA.
1269 Value *InVal = nullptr;
1270 if (!HasLoopExit) {
1271 InVal = PN->getIncomingValueForBlock(BB: Preds[0]);
1272 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1273 if (!PredSet.count(Ptr: PN->getIncomingBlock(i)))
1274 continue;
1275 if (!InVal)
1276 InVal = PN->getIncomingValue(i);
1277 else if (InVal != PN->getIncomingValue(i)) {
1278 InVal = nullptr;
1279 break;
1280 }
1281 }
1282 }
1283
1284 if (InVal) {
1285 // If all incoming values for the new PHI would be the same, just don't
1286 // make a new PHI. Instead, just remove the incoming values from the old
1287 // PHI.
1288 PN->removeIncomingValueIf(
1289 Predicate: [&](unsigned Idx) {
1290 return PredSet.contains(Ptr: PN->getIncomingBlock(i: Idx));
1291 },
1292 /* DeletePHIIfEmpty */ false);
1293
1294 // Add an incoming value to the PHI node in the loop for the preheader
1295 // edge.
1296 PN->addIncoming(V: InVal, BB: NewBB);
1297 continue;
1298 }
1299
1300 // If the values coming into the block are not the same, we need a new
1301 // PHI.
1302 // Create the new PHI node, insert it into NewBB at the end of the block
1303 PHINode *NewPHI =
1304 PHINode::Create(Ty: PN->getType(), NumReservedValues: Preds.size(), NameStr: PN->getName() + ".ph", InsertBefore: BI->getIterator());
1305
1306 // NOTE! This loop walks backwards for a reason! First off, this minimizes
1307 // the cost of removal if we end up removing a large number of values, and
1308 // second off, this ensures that the indices for the incoming values aren't
1309 // invalidated when we remove one.
1310 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1311 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1312 if (PredSet.count(Ptr: IncomingBB)) {
1313 Value *V = PN->removeIncomingValue(Idx: i, DeletePHIIfEmpty: false);
1314 NewPHI->addIncoming(V, BB: IncomingBB);
1315 }
1316 }
1317
1318 PN->addIncoming(V: NewPHI, BB: NewBB);
1319 }
1320}
1321
1322static void SplitLandingPadPredecessorsImpl(
1323 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1324 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1325 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1326 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1327
1328static BasicBlock *
1329SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef<BasicBlock *> Preds,
1330 const char *Suffix, DomTreeUpdater *DTU,
1331 DominatorTree *DT, LoopInfo *LI,
1332 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1333 // Do not attempt to split that which cannot be split.
1334 if (!BB->canSplitPredecessors())
1335 return nullptr;
1336
1337 // For the landingpads we need to act a bit differently.
1338 // Delegate this work to the SplitLandingPadPredecessors.
1339 if (BB->isLandingPad()) {
1340 SmallVector<BasicBlock*, 2> NewBBs;
1341 std::string NewName = std::string(Suffix) + ".split-lp";
1342
1343 SplitLandingPadPredecessorsImpl(OrigBB: BB, Preds, Suffix1: Suffix, Suffix2: NewName.c_str(), NewBBs,
1344 DTU, DT, LI, MSSAU, PreserveLCSSA);
1345 return NewBBs[0];
1346 }
1347
1348 // Create new basic block, insert right before the original block.
1349 BasicBlock *NewBB = BasicBlock::Create(
1350 Context&: BB->getContext(), Name: BB->getName() + Suffix, Parent: BB->getParent(), InsertBefore: BB);
1351
1352 // The new block unconditionally branches to the old block.
1353 BranchInst *BI = BranchInst::Create(IfTrue: BB, InsertBefore: NewBB);
1354
1355 Loop *L = nullptr;
1356 BasicBlock *OldLatch = nullptr;
1357 // Splitting the predecessors of a loop header creates a preheader block.
1358 if (LI && LI->isLoopHeader(BB)) {
1359 L = LI->getLoopFor(BB);
1360 // Using the loop start line number prevents debuggers stepping into the
1361 // loop body for this instruction.
1362 BI->setDebugLoc(L->getStartLoc());
1363
1364 // If BB is the header of the Loop, it is possible that the loop is
1365 // modified, such that the current latch does not remain the latch of the
1366 // loop. If that is the case, the loop metadata from the current latch needs
1367 // to be applied to the new latch.
1368 OldLatch = L->getLoopLatch();
1369 } else
1370 BI->setDebugLoc(BB->getFirstNonPHIOrDbg()->getDebugLoc());
1371
1372 // Move the edges from Preds to point to NewBB instead of BB.
1373 for (BasicBlock *Pred : Preds) {
1374 // This is slightly more strict than necessary; the minimum requirement
1375 // is that there be no more than one indirectbr branching to BB. And
1376 // all BlockAddress uses would need to be updated.
1377 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1378 "Cannot split an edge from an IndirectBrInst");
1379 Pred->getTerminator()->replaceSuccessorWith(OldBB: BB, NewBB);
1380 }
1381
1382 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1383 // node becomes an incoming value for BB's phi node. However, if the Preds
1384 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1385 // account for the newly created predecessor.
1386 if (Preds.empty()) {
1387 // Insert dummy values as the incoming value.
1388 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(Val: I); ++I)
1389 cast<PHINode>(Val&: I)->addIncoming(V: PoisonValue::get(T: I->getType()), BB: NewBB);
1390 }
1391
1392 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1393 bool HasLoopExit = false;
1394 UpdateAnalysisInformation(OldBB: BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1395 HasLoopExit);
1396
1397 if (!Preds.empty()) {
1398 // Update the PHI nodes in BB with the values coming from NewBB.
1399 UpdatePHINodes(OrigBB: BB, NewBB, Preds, BI, HasLoopExit);
1400 }
1401
1402 if (OldLatch) {
1403 BasicBlock *NewLatch = L->getLoopLatch();
1404 if (NewLatch != OldLatch) {
1405 MDNode *MD = OldLatch->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
1406 NewLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: MD);
1407 // It's still possible that OldLatch is the latch of another inner loop,
1408 // in which case we do not remove the metadata.
1409 Loop *IL = LI->getLoopFor(BB: OldLatch);
1410 if (IL && IL->getLoopLatch() != OldLatch)
1411 OldLatch->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: nullptr);
1412 }
1413 }
1414
1415 return NewBB;
1416}
1417
1418BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1419 ArrayRef<BasicBlock *> Preds,
1420 const char *Suffix, DominatorTree *DT,
1421 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1422 bool PreserveLCSSA) {
1423 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1424 MSSAU, PreserveLCSSA);
1425}
1426BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
1427 ArrayRef<BasicBlock *> Preds,
1428 const char *Suffix,
1429 DomTreeUpdater *DTU, LoopInfo *LI,
1430 MemorySSAUpdater *MSSAU,
1431 bool PreserveLCSSA) {
1432 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1433 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1434}
1435
1436static void SplitLandingPadPredecessorsImpl(
1437 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1438 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1439 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1440 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1441 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1442
1443 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1444 // it right before the original block.
1445 BasicBlock *NewBB1 = BasicBlock::Create(Context&: OrigBB->getContext(),
1446 Name: OrigBB->getName() + Suffix1,
1447 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1448 NewBBs.push_back(Elt: NewBB1);
1449
1450 // The new block unconditionally branches to the old block.
1451 BranchInst *BI1 = BranchInst::Create(IfTrue: OrigBB, InsertBefore: NewBB1);
1452 BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1453
1454 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1455 for (BasicBlock *Pred : Preds) {
1456 // This is slightly more strict than necessary; the minimum requirement
1457 // is that there be no more than one indirectbr branching to BB. And
1458 // all BlockAddress uses would need to be updated.
1459 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1460 "Cannot split an edge from an IndirectBrInst");
1461 Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB1);
1462 }
1463
1464 bool HasLoopExit = false;
1465 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB1, Preds, DTU, DT, LI, MSSAU,
1466 PreserveLCSSA, HasLoopExit);
1467
1468 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1469 UpdatePHINodes(OrigBB, NewBB: NewBB1, Preds, BI: BI1, HasLoopExit);
1470
1471 // Move the remaining edges from OrigBB to point to NewBB2.
1472 SmallVector<BasicBlock*, 8> NewBB2Preds;
1473 for (pred_iterator i = pred_begin(BB: OrigBB), e = pred_end(BB: OrigBB);
1474 i != e; ) {
1475 BasicBlock *Pred = *i++;
1476 if (Pred == NewBB1) continue;
1477 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1478 "Cannot split an edge from an IndirectBrInst");
1479 NewBB2Preds.push_back(Elt: Pred);
1480 e = pred_end(BB: OrigBB);
1481 }
1482
1483 BasicBlock *NewBB2 = nullptr;
1484 if (!NewBB2Preds.empty()) {
1485 // Create another basic block for the rest of OrigBB's predecessors.
1486 NewBB2 = BasicBlock::Create(Context&: OrigBB->getContext(),
1487 Name: OrigBB->getName() + Suffix2,
1488 Parent: OrigBB->getParent(), InsertBefore: OrigBB);
1489 NewBBs.push_back(Elt: NewBB2);
1490
1491 // The new block unconditionally branches to the old block.
1492 BranchInst *BI2 = BranchInst::Create(IfTrue: OrigBB, InsertBefore: NewBB2);
1493 BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1494
1495 // Move the remaining edges from OrigBB to point to NewBB2.
1496 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1497 NewBB2Pred->getTerminator()->replaceUsesOfWith(From: OrigBB, To: NewBB2);
1498
1499 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1500 HasLoopExit = false;
1501 UpdateAnalysisInformation(OldBB: OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, DTU, DT, LI, MSSAU,
1502 PreserveLCSSA, HasLoopExit);
1503
1504 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1505 UpdatePHINodes(OrigBB, NewBB: NewBB2, Preds: NewBB2Preds, BI: BI2, HasLoopExit);
1506 }
1507
1508 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1509 Instruction *Clone1 = LPad->clone();
1510 Clone1->setName(Twine("lpad") + Suffix1);
1511 Clone1->insertInto(ParentBB: NewBB1, It: NewBB1->getFirstInsertionPt());
1512
1513 if (NewBB2) {
1514 Instruction *Clone2 = LPad->clone();
1515 Clone2->setName(Twine("lpad") + Suffix2);
1516 Clone2->insertInto(ParentBB: NewBB2, It: NewBB2->getFirstInsertionPt());
1517
1518 // Create a PHI node for the two cloned landingpad instructions only
1519 // if the original landingpad instruction has some uses.
1520 if (!LPad->use_empty()) {
1521 assert(!LPad->getType()->isTokenTy() &&
1522 "Split cannot be applied if LPad is token type. Otherwise an "
1523 "invalid PHINode of token type would be created.");
1524 PHINode *PN = PHINode::Create(Ty: LPad->getType(), NumReservedValues: 2, NameStr: "lpad.phi", InsertBefore: LPad->getIterator());
1525 PN->addIncoming(V: Clone1, BB: NewBB1);
1526 PN->addIncoming(V: Clone2, BB: NewBB2);
1527 LPad->replaceAllUsesWith(V: PN);
1528 }
1529 LPad->eraseFromParent();
1530 } else {
1531 // There is no second clone. Just replace the landing pad with the first
1532 // clone.
1533 LPad->replaceAllUsesWith(V: Clone1);
1534 LPad->eraseFromParent();
1535 }
1536}
1537
1538void llvm::SplitLandingPadPredecessors(BasicBlock *OrigBB,
1539 ArrayRef<BasicBlock *> Preds,
1540 const char *Suffix1, const char *Suffix2,
1541 SmallVectorImpl<BasicBlock *> &NewBBs,
1542 DomTreeUpdater *DTU, LoopInfo *LI,
1543 MemorySSAUpdater *MSSAU,
1544 bool PreserveLCSSA) {
1545 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1546 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1547 PreserveLCSSA);
1548}
1549
1550ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
1551 BasicBlock *Pred,
1552 DomTreeUpdater *DTU) {
1553 Instruction *UncondBranch = Pred->getTerminator();
1554 // Clone the return and add it to the end of the predecessor.
1555 Instruction *NewRet = RI->clone();
1556 NewRet->insertInto(ParentBB: Pred, It: Pred->end());
1557
1558 // If the return instruction returns a value, and if the value was a
1559 // PHI node in "BB", propagate the right value into the return.
1560 for (Use &Op : NewRet->operands()) {
1561 Value *V = Op;
1562 Instruction *NewBC = nullptr;
1563 if (BitCastInst *BCI = dyn_cast<BitCastInst>(Val: V)) {
1564 // Return value might be bitcasted. Clone and insert it before the
1565 // return instruction.
1566 V = BCI->getOperand(i_nocapture: 0);
1567 NewBC = BCI->clone();
1568 NewBC->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1569 Op = NewBC;
1570 }
1571
1572 Instruction *NewEV = nullptr;
1573 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val: V)) {
1574 V = EVI->getOperand(i_nocapture: 0);
1575 NewEV = EVI->clone();
1576 if (NewBC) {
1577 NewBC->setOperand(i: 0, Val: NewEV);
1578 NewEV->insertInto(ParentBB: Pred, It: NewBC->getIterator());
1579 } else {
1580 NewEV->insertInto(ParentBB: Pred, It: NewRet->getIterator());
1581 Op = NewEV;
1582 }
1583 }
1584
1585 if (PHINode *PN = dyn_cast<PHINode>(Val: V)) {
1586 if (PN->getParent() == BB) {
1587 if (NewEV) {
1588 NewEV->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1589 } else if (NewBC)
1590 NewBC->setOperand(i: 0, Val: PN->getIncomingValueForBlock(BB: Pred));
1591 else
1592 Op = PN->getIncomingValueForBlock(BB: Pred);
1593 }
1594 }
1595 }
1596
1597 // Update any PHI nodes in the returning block to realize that we no
1598 // longer branch to them.
1599 BB->removePredecessor(Pred);
1600 UncondBranch->eraseFromParent();
1601
1602 if (DTU)
1603 DTU->applyUpdates(Updates: {{DominatorTree::Delete, Pred, BB}});
1604
1605 return cast<ReturnInst>(Val: NewRet);
1606}
1607
1608Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
1609 BasicBlock::iterator SplitBefore,
1610 bool Unreachable,
1611 MDNode *BranchWeights,
1612 DomTreeUpdater *DTU, LoopInfo *LI,
1613 BasicBlock *ThenBlock) {
1614 SplitBlockAndInsertIfThenElse(
1615 Cond, SplitBefore, ThenBlock: &ThenBlock, /* ElseBlock */ nullptr,
1616 /* UnreachableThen */ Unreachable,
1617 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1618 return ThenBlock->getTerminator();
1619}
1620
1621Instruction *llvm::SplitBlockAndInsertIfElse(Value *Cond,
1622 BasicBlock::iterator SplitBefore,
1623 bool Unreachable,
1624 MDNode *BranchWeights,
1625 DomTreeUpdater *DTU, LoopInfo *LI,
1626 BasicBlock *ElseBlock) {
1627 SplitBlockAndInsertIfThenElse(
1628 Cond, SplitBefore, /* ThenBlock */ nullptr, ElseBlock: &ElseBlock,
1629 /* UnreachableThen */ false,
1630 /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1631 return ElseBlock->getTerminator();
1632}
1633
1634void llvm::SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore,
1635 Instruction **ThenTerm,
1636 Instruction **ElseTerm,
1637 MDNode *BranchWeights,
1638 DomTreeUpdater *DTU, LoopInfo *LI) {
1639 BasicBlock *ThenBlock = nullptr;
1640 BasicBlock *ElseBlock = nullptr;
1641 SplitBlockAndInsertIfThenElse(
1642 Cond, SplitBefore, ThenBlock: &ThenBlock, ElseBlock: &ElseBlock, /* UnreachableThen */ false,
1643 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1644
1645 *ThenTerm = ThenBlock->getTerminator();
1646 *ElseTerm = ElseBlock->getTerminator();
1647}
1648
1649void llvm::SplitBlockAndInsertIfThenElse(
1650 Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
1651 BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1652 MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1653 assert((ThenBlock || ElseBlock) &&
1654 "At least one branch block must be created");
1655 assert((!UnreachableThen || !UnreachableElse) &&
1656 "Split block tail must be reachable");
1657
1658 SmallVector<DominatorTree::UpdateType, 8> Updates;
1659 SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1660 BasicBlock *Head = SplitBefore->getParent();
1661 if (DTU) {
1662 UniqueOrigSuccessors.insert(I: succ_begin(BB: Head), E: succ_end(BB: Head));
1663 Updates.reserve(N: 4 + 2 * UniqueOrigSuccessors.size());
1664 }
1665
1666 LLVMContext &C = Head->getContext();
1667 BasicBlock *Tail = Head->splitBasicBlock(I: SplitBefore);
1668 BasicBlock *TrueBlock = Tail;
1669 BasicBlock *FalseBlock = Tail;
1670 bool ThenToTailEdge = false;
1671 bool ElseToTailEdge = false;
1672
1673 // Encapsulate the logic around creation/insertion/etc of a new block.
1674 auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1675 bool &ToTailEdge) {
1676 if (PBB == nullptr)
1677 return; // Do not create/insert a block.
1678
1679 if (*PBB)
1680 BB = *PBB; // Caller supplied block, use it.
1681 else {
1682 // Create a new block.
1683 BB = BasicBlock::Create(Context&: C, Name: "", Parent: Head->getParent(), InsertBefore: Tail);
1684 if (Unreachable)
1685 (void)new UnreachableInst(C, BB);
1686 else {
1687 (void)BranchInst::Create(IfTrue: Tail, InsertBefore: BB);
1688 ToTailEdge = true;
1689 }
1690 BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1691 // Pass the new block back to the caller.
1692 *PBB = BB;
1693 }
1694 };
1695
1696 handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1697 handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1698
1699 Instruction *HeadOldTerm = Head->getTerminator();
1700 BranchInst *HeadNewTerm =
1701 BranchInst::Create(/*ifTrue*/ IfTrue: TrueBlock, /*ifFalse*/ IfFalse: FalseBlock, Cond);
1702 HeadNewTerm->setMetadata(KindID: LLVMContext::MD_prof, Node: BranchWeights);
1703 ReplaceInstWithInst(From: HeadOldTerm, To: HeadNewTerm);
1704
1705 if (DTU) {
1706 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: TrueBlock);
1707 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Head, Args&: FalseBlock);
1708 if (ThenToTailEdge)
1709 Updates.emplace_back(Args: DominatorTree::Insert, Args&: TrueBlock, Args&: Tail);
1710 if (ElseToTailEdge)
1711 Updates.emplace_back(Args: DominatorTree::Insert, Args&: FalseBlock, Args&: Tail);
1712 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1713 Updates.emplace_back(Args: DominatorTree::Insert, Args&: Tail, Args&: UniqueOrigSuccessor);
1714 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1715 Updates.emplace_back(Args: DominatorTree::Delete, Args&: Head, Args&: UniqueOrigSuccessor);
1716 DTU->applyUpdates(Updates);
1717 }
1718
1719 if (LI) {
1720 if (Loop *L = LI->getLoopFor(BB: Head); L) {
1721 if (ThenToTailEdge)
1722 L->addBasicBlockToLoop(NewBB: TrueBlock, LI&: *LI);
1723 if (ElseToTailEdge)
1724 L->addBasicBlockToLoop(NewBB: FalseBlock, LI&: *LI);
1725 L->addBasicBlockToLoop(NewBB: Tail, LI&: *LI);
1726 }
1727 }
1728}
1729
1730std::pair<Instruction*, Value*>
1731llvm::SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore) {
1732 BasicBlock *LoopPred = SplitBefore->getParent();
1733 BasicBlock *LoopBody = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1734 BasicBlock *LoopExit = SplitBlock(Old: SplitBefore->getParent(), SplitPt: SplitBefore);
1735
1736 auto *Ty = End->getType();
1737 auto &DL = SplitBefore->getDataLayout();
1738 const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1739
1740 IRBuilder<> Builder(LoopBody->getTerminator());
1741 auto *IV = Builder.CreatePHI(Ty, NumReservedValues: 2, Name: "iv");
1742 auto *IVNext =
1743 Builder.CreateAdd(LHS: IV, RHS: ConstantInt::get(Ty, V: 1), Name: IV->getName() + ".next",
1744 /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1745 auto *IVCheck = Builder.CreateICmpEQ(LHS: IVNext, RHS: End,
1746 Name: IV->getName() + ".check");
1747 Builder.CreateCondBr(Cond: IVCheck, True: LoopExit, False: LoopBody);
1748 LoopBody->getTerminator()->eraseFromParent();
1749
1750 // Populate the IV PHI.
1751 IV->addIncoming(V: ConstantInt::get(Ty, V: 0), BB: LoopPred);
1752 IV->addIncoming(V: IVNext, BB: LoopBody);
1753
1754 return std::make_pair(x: LoopBody->getFirstNonPHI(), y&: IV);
1755}
1756
1757void llvm::SplitBlockAndInsertForEachLane(ElementCount EC,
1758 Type *IndexTy, Instruction *InsertBefore,
1759 std::function<void(IRBuilderBase&, Value*)> Func) {
1760
1761 IRBuilder<> IRB(InsertBefore);
1762
1763 if (EC.isScalable()) {
1764 Value *NumElements = IRB.CreateElementCount(DstType: IndexTy, EC);
1765
1766 auto [BodyIP, Index] =
1767 SplitBlockAndInsertSimpleForLoop(End: NumElements, SplitBefore: InsertBefore);
1768
1769 IRB.SetInsertPoint(BodyIP);
1770 Func(IRB, Index);
1771 return;
1772 }
1773
1774 unsigned Num = EC.getFixedValue();
1775 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1776 IRB.SetInsertPoint(InsertBefore);
1777 Func(IRB, ConstantInt::get(Ty: IndexTy, V: Idx));
1778 }
1779}
1780
1781void llvm::SplitBlockAndInsertForEachLane(
1782 Value *EVL, Instruction *InsertBefore,
1783 std::function<void(IRBuilderBase &, Value *)> Func) {
1784
1785 IRBuilder<> IRB(InsertBefore);
1786 Type *Ty = EVL->getType();
1787
1788 if (!isa<ConstantInt>(Val: EVL)) {
1789 auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(End: EVL, SplitBefore: InsertBefore);
1790 IRB.SetInsertPoint(BodyIP);
1791 Func(IRB, Index);
1792 return;
1793 }
1794
1795 unsigned Num = cast<ConstantInt>(Val: EVL)->getZExtValue();
1796 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1797 IRB.SetInsertPoint(InsertBefore);
1798 Func(IRB, ConstantInt::get(Ty, V: Idx));
1799 }
1800}
1801
1802BranchInst *llvm::GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
1803 BasicBlock *&IfFalse) {
1804 PHINode *SomePHI = dyn_cast<PHINode>(Val: BB->begin());
1805 BasicBlock *Pred1 = nullptr;
1806 BasicBlock *Pred2 = nullptr;
1807
1808 if (SomePHI) {
1809 if (SomePHI->getNumIncomingValues() != 2)
1810 return nullptr;
1811 Pred1 = SomePHI->getIncomingBlock(i: 0);
1812 Pred2 = SomePHI->getIncomingBlock(i: 1);
1813 } else {
1814 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1815 if (PI == PE) // No predecessor
1816 return nullptr;
1817 Pred1 = *PI++;
1818 if (PI == PE) // Only one predecessor
1819 return nullptr;
1820 Pred2 = *PI++;
1821 if (PI != PE) // More than two predecessors
1822 return nullptr;
1823 }
1824
1825 // We can only handle branches. Other control flow will be lowered to
1826 // branches if possible anyway.
1827 BranchInst *Pred1Br = dyn_cast<BranchInst>(Val: Pred1->getTerminator());
1828 BranchInst *Pred2Br = dyn_cast<BranchInst>(Val: Pred2->getTerminator());
1829 if (!Pred1Br || !Pred2Br)
1830 return nullptr;
1831
1832 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1833 // either are.
1834 if (Pred2Br->isConditional()) {
1835 // If both branches are conditional, we don't have an "if statement". In
1836 // reality, we could transform this case, but since the condition will be
1837 // required anyway, we stand no chance of eliminating it, so the xform is
1838 // probably not profitable.
1839 if (Pred1Br->isConditional())
1840 return nullptr;
1841
1842 std::swap(a&: Pred1, b&: Pred2);
1843 std::swap(a&: Pred1Br, b&: Pred2Br);
1844 }
1845
1846 if (Pred1Br->isConditional()) {
1847 // The only thing we have to watch out for here is to make sure that Pred2
1848 // doesn't have incoming edges from other blocks. If it does, the condition
1849 // doesn't dominate BB.
1850 if (!Pred2->getSinglePredecessor())
1851 return nullptr;
1852
1853 // If we found a conditional branch predecessor, make sure that it branches
1854 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1855 if (Pred1Br->getSuccessor(i: 0) == BB &&
1856 Pred1Br->getSuccessor(i: 1) == Pred2) {
1857 IfTrue = Pred1;
1858 IfFalse = Pred2;
1859 } else if (Pred1Br->getSuccessor(i: 0) == Pred2 &&
1860 Pred1Br->getSuccessor(i: 1) == BB) {
1861 IfTrue = Pred2;
1862 IfFalse = Pred1;
1863 } else {
1864 // We know that one arm of the conditional goes to BB, so the other must
1865 // go somewhere unrelated, and this must not be an "if statement".
1866 return nullptr;
1867 }
1868
1869 return Pred1Br;
1870 }
1871
1872 // Ok, if we got here, both predecessors end with an unconditional branch to
1873 // BB. Don't panic! If both blocks only have a single (identical)
1874 // predecessor, and THAT is a conditional branch, then we're all ok!
1875 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1876 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1877 return nullptr;
1878
1879 // Otherwise, if this is a conditional branch, then we can use it!
1880 BranchInst *BI = dyn_cast<BranchInst>(Val: CommonPred->getTerminator());
1881 if (!BI) return nullptr;
1882
1883 assert(BI->isConditional() && "Two successors but not conditional?");
1884 if (BI->getSuccessor(i: 0) == Pred1) {
1885 IfTrue = Pred1;
1886 IfFalse = Pred2;
1887 } else {
1888 IfTrue = Pred2;
1889 IfFalse = Pred1;
1890 }
1891 return BI;
1892}
1893
1894// After creating a control flow hub, the operands of PHINodes in an outgoing
1895// block Out no longer match the predecessors of that block. Predecessors of Out
1896// that are incoming blocks to the hub are now replaced by just one edge from
1897// the hub. To match this new control flow, the corresponding values from each
1898// PHINode must now be moved a new PHINode in the first guard block of the hub.
1899//
1900// This operation cannot be performed with SSAUpdater, because it involves one
1901// new use: If the block Out is in the list of Incoming blocks, then the newly
1902// created PHI in the Hub will use itself along that edge from Out to Hub.
1903static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
1904 const SetVector<BasicBlock *> &Incoming,
1905 BasicBlock *FirstGuardBlock) {
1906 auto I = Out->begin();
1907 while (I != Out->end() && isa<PHINode>(Val: I)) {
1908 auto Phi = cast<PHINode>(Val&: I);
1909 auto NewPhi =
1910 PHINode::Create(Ty: Phi->getType(), NumReservedValues: Incoming.size(),
1911 NameStr: Phi->getName() + ".moved", InsertBefore: FirstGuardBlock->begin());
1912 for (auto *In : Incoming) {
1913 Value *V = UndefValue::get(T: Phi->getType());
1914 if (In == Out) {
1915 V = NewPhi;
1916 } else if (Phi->getBasicBlockIndex(BB: In) != -1) {
1917 V = Phi->removeIncomingValue(BB: In, DeletePHIIfEmpty: false);
1918 }
1919 NewPhi->addIncoming(V, BB: In);
1920 }
1921 assert(NewPhi->getNumIncomingValues() == Incoming.size());
1922 if (Phi->getNumOperands() == 0) {
1923 Phi->replaceAllUsesWith(V: NewPhi);
1924 I = Phi->eraseFromParent();
1925 continue;
1926 }
1927 Phi->addIncoming(V: NewPhi, BB: GuardBlock);
1928 ++I;
1929 }
1930}
1931
1932using BBPredicates = DenseMap<BasicBlock *, Instruction *>;
1933using BBSetVector = SetVector<BasicBlock *>;
1934
1935// Redirects the terminator of the incoming block to the first guard
1936// block in the hub. The condition of the original terminator (if it
1937// was conditional) and its original successors are returned as a
1938// tuple <condition, succ0, succ1>. The function additionally filters
1939// out successors that are not in the set of outgoing blocks.
1940//
1941// - condition is non-null iff the branch is conditional.
1942// - Succ1 is non-null iff the sole/taken target is an outgoing block.
1943// - Succ2 is non-null iff condition is non-null and the fallthrough
1944// target is an outgoing block.
1945static std::tuple<Value *, BasicBlock *, BasicBlock *>
1946redirectToHub(BasicBlock *BB, BasicBlock *FirstGuardBlock,
1947 const BBSetVector &Outgoing) {
1948 assert(isa<BranchInst>(BB->getTerminator()) &&
1949 "Only support branch terminator.");
1950 auto Branch = cast<BranchInst>(Val: BB->getTerminator());
1951 auto Condition = Branch->isConditional() ? Branch->getCondition() : nullptr;
1952
1953 BasicBlock *Succ0 = Branch->getSuccessor(i: 0);
1954 BasicBlock *Succ1 = nullptr;
1955 Succ0 = Outgoing.count(key: Succ0) ? Succ0 : nullptr;
1956
1957 if (Branch->isUnconditional()) {
1958 Branch->setSuccessor(idx: 0, NewSucc: FirstGuardBlock);
1959 assert(Succ0);
1960 } else {
1961 Succ1 = Branch->getSuccessor(i: 1);
1962 Succ1 = Outgoing.count(key: Succ1) ? Succ1 : nullptr;
1963 assert(Succ0 || Succ1);
1964 if (Succ0 && !Succ1) {
1965 Branch->setSuccessor(idx: 0, NewSucc: FirstGuardBlock);
1966 } else if (Succ1 && !Succ0) {
1967 Branch->setSuccessor(idx: 1, NewSucc: FirstGuardBlock);
1968 } else {
1969 Branch->eraseFromParent();
1970 BranchInst::Create(IfTrue: FirstGuardBlock, InsertBefore: BB);
1971 }
1972 }
1973
1974 assert(Succ0 || Succ1);
1975 return std::make_tuple(args&: Condition, args&: Succ0, args&: Succ1);
1976}
1977// Setup the branch instructions for guard blocks.
1978//
1979// Each guard block terminates in a conditional branch that transfers
1980// control to the corresponding outgoing block or the next guard
1981// block. The last guard block has two outgoing blocks as successors
1982// since the condition for the final outgoing block is trivially
1983// true. So we create one less block (including the first guard block)
1984// than the number of outgoing blocks.
1985static void setupBranchForGuard(SmallVectorImpl<BasicBlock *> &GuardBlocks,
1986 const BBSetVector &Outgoing,
1987 BBPredicates &GuardPredicates) {
1988 // To help keep the loop simple, temporarily append the last
1989 // outgoing block to the list of guard blocks.
1990 GuardBlocks.push_back(Elt: Outgoing.back());
1991
1992 for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
1993 auto Out = Outgoing[i];
1994 assert(GuardPredicates.count(Out));
1995 BranchInst::Create(IfTrue: Out, IfFalse: GuardBlocks[i + 1], Cond: GuardPredicates[Out],
1996 InsertBefore: GuardBlocks[i]);
1997 }
1998
1999 // Remove the last block from the guard list.
2000 GuardBlocks.pop_back();
2001}
2002
2003/// We are using one integer to represent the block we are branching to. Then at
2004/// each guard block, the predicate was calcuated using a simple `icmp eq`.
2005static void calcPredicateUsingInteger(
2006 const BBSetVector &Incoming, const BBSetVector &Outgoing,
2007 SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates) {
2008 auto &Context = Incoming.front()->getContext();
2009 auto FirstGuardBlock = GuardBlocks.front();
2010
2011 auto Phi = PHINode::Create(Ty: Type::getInt32Ty(C&: Context), NumReservedValues: Incoming.size(),
2012 NameStr: "merged.bb.idx", InsertBefore: FirstGuardBlock);
2013
2014 for (auto In : Incoming) {
2015 Value *Condition;
2016 BasicBlock *Succ0;
2017 BasicBlock *Succ1;
2018 std::tie(args&: Condition, args&: Succ0, args&: Succ1) =
2019 redirectToHub(BB: In, FirstGuardBlock, Outgoing);
2020 Value *IncomingId = nullptr;
2021 if (Succ0 && Succ1) {
2022 // target_bb_index = Condition ? index_of_succ0 : index_of_succ1.
2023 auto Succ0Iter = find(Range: Outgoing, Val: Succ0);
2024 auto Succ1Iter = find(Range: Outgoing, Val: Succ1);
2025 Value *Id0 = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context),
2026 V: std::distance(first: Outgoing.begin(), last: Succ0Iter));
2027 Value *Id1 = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context),
2028 V: std::distance(first: Outgoing.begin(), last: Succ1Iter));
2029 IncomingId = SelectInst::Create(C: Condition, S1: Id0, S2: Id1, NameStr: "target.bb.idx",
2030 InsertBefore: In->getTerminator()->getIterator());
2031 } else {
2032 // Get the index of the non-null successor.
2033 auto SuccIter = Succ0 ? find(Range: Outgoing, Val: Succ0) : find(Range: Outgoing, Val: Succ1);
2034 IncomingId = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context),
2035 V: std::distance(first: Outgoing.begin(), last: SuccIter));
2036 }
2037 Phi->addIncoming(V: IncomingId, BB: In);
2038 }
2039
2040 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
2041 auto Out = Outgoing[i];
2042 auto Cmp = ICmpInst::Create(Op: Instruction::ICmp, Pred: ICmpInst::ICMP_EQ, S1: Phi,
2043 S2: ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: i),
2044 Name: Out->getName() + ".predicate", InsertBefore: GuardBlocks[i]);
2045 GuardPredicates[Out] = Cmp;
2046 }
2047}
2048
2049/// We record the predicate of each outgoing block using a phi of boolean.
2050static void calcPredicateUsingBooleans(
2051 const BBSetVector &Incoming, const BBSetVector &Outgoing,
2052 SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates,
2053 SmallVectorImpl<WeakVH> &DeletionCandidates) {
2054 auto &Context = Incoming.front()->getContext();
2055 auto BoolTrue = ConstantInt::getTrue(Context);
2056 auto BoolFalse = ConstantInt::getFalse(Context);
2057 auto FirstGuardBlock = GuardBlocks.front();
2058
2059 // The predicate for the last outgoing is trivially true, and so we
2060 // process only the first N-1 successors.
2061 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
2062 auto Out = Outgoing[i];
2063 LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
2064
2065 auto Phi =
2066 PHINode::Create(Ty: Type::getInt1Ty(C&: Context), NumReservedValues: Incoming.size(),
2067 NameStr: StringRef("Guard.") + Out->getName(), InsertBefore: FirstGuardBlock);
2068 GuardPredicates[Out] = Phi;
2069 }
2070
2071 for (auto *In : Incoming) {
2072 Value *Condition;
2073 BasicBlock *Succ0;
2074 BasicBlock *Succ1;
2075 std::tie(args&: Condition, args&: Succ0, args&: Succ1) =
2076 redirectToHub(BB: In, FirstGuardBlock, Outgoing);
2077
2078 // Optimization: Consider an incoming block A with both successors
2079 // Succ0 and Succ1 in the set of outgoing blocks. The predicates
2080 // for Succ0 and Succ1 complement each other. If Succ0 is visited
2081 // first in the loop below, control will branch to Succ0 using the
2082 // corresponding predicate. But if that branch is not taken, then
2083 // control must reach Succ1, which means that the incoming value of
2084 // the predicate from `In` is true for Succ1.
2085 bool OneSuccessorDone = false;
2086 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
2087 auto Out = Outgoing[i];
2088 PHINode *Phi = cast<PHINode>(Val: GuardPredicates[Out]);
2089 if (Out != Succ0 && Out != Succ1) {
2090 Phi->addIncoming(V: BoolFalse, BB: In);
2091 } else if (!Succ0 || !Succ1 || OneSuccessorDone) {
2092 // Optimization: When only one successor is an outgoing block,
2093 // the incoming predicate from `In` is always true.
2094 Phi->addIncoming(V: BoolTrue, BB: In);
2095 } else {
2096 assert(Succ0 && Succ1);
2097 if (Out == Succ0) {
2098 Phi->addIncoming(V: Condition, BB: In);
2099 } else {
2100 auto Inverted = invertCondition(Condition);
2101 DeletionCandidates.push_back(Elt: Condition);
2102 Phi->addIncoming(V: Inverted, BB: In);
2103 }
2104 OneSuccessorDone = true;
2105 }
2106 }
2107 }
2108}
2109
2110// Capture the existing control flow as guard predicates, and redirect
2111// control flow from \p Incoming block through the \p GuardBlocks to the
2112// \p Outgoing blocks.
2113//
2114// There is one guard predicate for each outgoing block OutBB. The
2115// predicate represents whether the hub should transfer control flow
2116// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates
2117// them in the same order as the Outgoing set-vector, and control
2118// branches to the first outgoing block whose predicate evaluates to true.
2119static void
2120convertToGuardPredicates(SmallVectorImpl<BasicBlock *> &GuardBlocks,
2121 SmallVectorImpl<WeakVH> &DeletionCandidates,
2122 const BBSetVector &Incoming,
2123 const BBSetVector &Outgoing, const StringRef Prefix,
2124 std::optional<unsigned> MaxControlFlowBooleans) {
2125 BBPredicates GuardPredicates;
2126 auto F = Incoming.front()->getParent();
2127
2128 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i)
2129 GuardBlocks.push_back(
2130 Elt: BasicBlock::Create(Context&: F->getContext(), Name: Prefix + ".guard", Parent: F));
2131
2132 // When we are using an integer to record which target block to jump to, we
2133 // are creating less live values, actually we are using one single integer to
2134 // store the index of the target block. When we are using booleans to store
2135 // the branching information, we need (N-1) boolean values, where N is the
2136 // number of outgoing block.
2137 if (!MaxControlFlowBooleans || Outgoing.size() <= *MaxControlFlowBooleans)
2138 calcPredicateUsingBooleans(Incoming, Outgoing, GuardBlocks, GuardPredicates,
2139 DeletionCandidates);
2140 else
2141 calcPredicateUsingInteger(Incoming, Outgoing, GuardBlocks, GuardPredicates);
2142
2143 setupBranchForGuard(GuardBlocks, Outgoing, GuardPredicates);
2144}
2145
2146BasicBlock *llvm::CreateControlFlowHub(
2147 DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
2148 const BBSetVector &Incoming, const BBSetVector &Outgoing,
2149 const StringRef Prefix, std::optional<unsigned> MaxControlFlowBooleans) {
2150 if (Outgoing.size() < 2)
2151 return Outgoing.front();
2152
2153 SmallVector<DominatorTree::UpdateType, 16> Updates;
2154 if (DTU) {
2155 for (auto *In : Incoming) {
2156 for (auto Succ : successors(BB: In))
2157 if (Outgoing.count(key: Succ))
2158 Updates.push_back(Elt: {DominatorTree::Delete, In, Succ});
2159 }
2160 }
2161
2162 SmallVector<WeakVH, 8> DeletionCandidates;
2163 convertToGuardPredicates(GuardBlocks, DeletionCandidates, Incoming, Outgoing,
2164 Prefix, MaxControlFlowBooleans);
2165 auto FirstGuardBlock = GuardBlocks.front();
2166
2167 // Update the PHINodes in each outgoing block to match the new control flow.
2168 for (int i = 0, e = GuardBlocks.size(); i != e; ++i)
2169 reconnectPhis(Out: Outgoing[i], GuardBlock: GuardBlocks[i], Incoming, FirstGuardBlock);
2170
2171 reconnectPhis(Out: Outgoing.back(), GuardBlock: GuardBlocks.back(), Incoming, FirstGuardBlock);
2172
2173 if (DTU) {
2174 int NumGuards = GuardBlocks.size();
2175 assert((int)Outgoing.size() == NumGuards + 1);
2176
2177 for (auto In : Incoming)
2178 Updates.push_back(Elt: {DominatorTree::Insert, In, FirstGuardBlock});
2179
2180 for (int i = 0; i != NumGuards - 1; ++i) {
2181 Updates.push_back(Elt: {DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
2182 Updates.push_back(
2183 Elt: {DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
2184 }
2185 Updates.push_back(Elt: {DominatorTree::Insert, GuardBlocks[NumGuards - 1],
2186 Outgoing[NumGuards - 1]});
2187 Updates.push_back(Elt: {DominatorTree::Insert, GuardBlocks[NumGuards - 1],
2188 Outgoing[NumGuards]});
2189 DTU->applyUpdates(Updates);
2190 }
2191
2192 for (auto I : DeletionCandidates) {
2193 if (I->use_empty())
2194 if (auto Inst = dyn_cast_or_null<Instruction>(Val&: I))
2195 Inst->eraseFromParent();
2196 }
2197
2198 return FirstGuardBlock;
2199}
2200
2201void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) {
2202 Value *NewCond = PBI->getCondition();
2203 // If this is a "cmp" instruction, only used for branching (and nowhere
2204 // else), then we can simply invert the predicate.
2205 if (NewCond->hasOneUse() && isa<CmpInst>(Val: NewCond)) {
2206 CmpInst *CI = cast<CmpInst>(Val: NewCond);
2207 CI->setPredicate(CI->getInversePredicate());
2208 } else
2209 NewCond = Builder.CreateNot(V: NewCond, Name: NewCond->getName() + ".not");
2210
2211 PBI->setCondition(NewCond);
2212 PBI->swapSuccessors();
2213}
2214
2215bool llvm::hasOnlySimpleTerminator(const Function &F) {
2216 for (auto &BB : F) {
2217 auto *Term = BB.getTerminator();
2218 if (!(isa<ReturnInst>(Val: Term) || isa<UnreachableInst>(Val: Term) ||
2219 isa<BranchInst>(Val: Term)))
2220 return false;
2221 }
2222 return true;
2223}
2224
2225bool llvm::isPresplitCoroSuspendExitEdge(const BasicBlock &Src,
2226 const BasicBlock &Dest) {
2227 assert(Src.getParent() == Dest.getParent());
2228 if (!Src.getParent()->isPresplitCoroutine())
2229 return false;
2230 if (auto *SW = dyn_cast<SwitchInst>(Val: Src.getTerminator()))
2231 if (auto *Intr = dyn_cast<IntrinsicInst>(Val: SW->getCondition()))
2232 return Intr->getIntrinsicID() == Intrinsic::coro_suspend &&
2233 SW->getDefaultDest() == &Dest;
2234 return false;
2235}
2236