| 1 | //===-- LCSSA.cpp - Convert loops into loop-closed SSA form ---------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass transforms loops by placing phi nodes at the end of the loops for |
| 10 | // all values that are live across the loop boundary. For example, it turns |
| 11 | // the left into the right code: |
| 12 | // |
| 13 | // for (...) for (...) |
| 14 | // if (c) if (c) |
| 15 | // X1 = ... X1 = ... |
| 16 | // else else |
| 17 | // X2 = ... X2 = ... |
| 18 | // X3 = phi(X1, X2) X3 = phi(X1, X2) |
| 19 | // ... = X3 + 4 X4 = phi(X3) |
| 20 | // ... = X4 + 4 |
| 21 | // |
| 22 | // This is still valid LLVM; the extra phi nodes are purely redundant, and will |
| 23 | // be trivially eliminated by InstCombine. The major benefit of this |
| 24 | // transformation is that it makes many other loop optimizations, such as |
| 25 | // LoopUnswitching, simpler. |
| 26 | // |
| 27 | //===----------------------------------------------------------------------===// |
| 28 | |
| 29 | #include "llvm/Transforms/Utils/LCSSA.h" |
| 30 | #include "llvm/ADT/STLExtras.h" |
| 31 | #include "llvm/ADT/Statistic.h" |
| 32 | #include "llvm/Analysis/AliasAnalysis.h" |
| 33 | #include "llvm/Analysis/BasicAliasAnalysis.h" |
| 34 | #include "llvm/Analysis/BranchProbabilityInfo.h" |
| 35 | #include "llvm/Analysis/GlobalsModRef.h" |
| 36 | #include "llvm/Analysis/LoopInfo.h" |
| 37 | #include "llvm/Analysis/LoopPass.h" |
| 38 | #include "llvm/Analysis/MemorySSA.h" |
| 39 | #include "llvm/Analysis/ScalarEvolution.h" |
| 40 | #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" |
| 41 | #include "llvm/IR/DebugInfo.h" |
| 42 | #include "llvm/IR/Dominators.h" |
| 43 | #include "llvm/IR/Instructions.h" |
| 44 | #include "llvm/IR/PredIteratorCache.h" |
| 45 | #include "llvm/InitializePasses.h" |
| 46 | #include "llvm/Pass.h" |
| 47 | #include "llvm/Support/CommandLine.h" |
| 48 | #include "llvm/Transforms/Utils.h" |
| 49 | #include "llvm/Transforms/Utils/LoopUtils.h" |
| 50 | #include "llvm/Transforms/Utils/SSAUpdater.h" |
| 51 | using namespace llvm; |
| 52 | |
| 53 | #define DEBUG_TYPE "lcssa" |
| 54 | |
| 55 | STATISTIC(NumLCSSA, "Number of live out of a loop variables" ); |
| 56 | |
| 57 | #ifdef EXPENSIVE_CHECKS |
| 58 | static bool VerifyLoopLCSSA = true; |
| 59 | #else |
| 60 | static bool VerifyLoopLCSSA = false; |
| 61 | #endif |
| 62 | static cl::opt<bool, true> |
| 63 | VerifyLoopLCSSAFlag("verify-loop-lcssa" , cl::location(L&: VerifyLoopLCSSA), |
| 64 | cl::Hidden, |
| 65 | cl::desc("Verify loop lcssa form (time consuming)" )); |
| 66 | |
| 67 | /// Return true if the specified block is in the list. |
| 68 | static bool isExitBlock(BasicBlock *BB, |
| 69 | const SmallVectorImpl<BasicBlock *> &ExitBlocks) { |
| 70 | return is_contained(Range: ExitBlocks, Element: BB); |
| 71 | } |
| 72 | |
| 73 | // Cache the Loop ExitBlocks computed during the analysis. We expect to get a |
| 74 | // lot of instructions within the same loops, computing the exit blocks is |
| 75 | // expensive, and we're not mutating the loop structure. |
| 76 | using LoopExitBlocksTy = SmallDenseMap<Loop *, SmallVector<BasicBlock *, 1>>; |
| 77 | |
| 78 | /// For every instruction from the worklist, check to see if it has any uses |
| 79 | /// that are outside the current loop. If so, insert LCSSA PHI nodes and |
| 80 | /// rewrite the uses. |
| 81 | static bool |
| 82 | formLCSSAForInstructionsImpl(SmallVectorImpl<Instruction *> &Worklist, |
| 83 | const DominatorTree &DT, const LoopInfo &LI, |
| 84 | ScalarEvolution *SE, |
| 85 | SmallVectorImpl<PHINode *> *PHIsToRemove, |
| 86 | SmallVectorImpl<PHINode *> *InsertedPHIs, |
| 87 | LoopExitBlocksTy &LoopExitBlocks) { |
| 88 | SmallVector<Use *, 16> UsesToRewrite; |
| 89 | SmallSetVector<PHINode *, 16> LocalPHIsToRemove; |
| 90 | PredIteratorCache PredCache; |
| 91 | bool Changed = false; |
| 92 | |
| 93 | while (!Worklist.empty()) { |
| 94 | UsesToRewrite.clear(); |
| 95 | |
| 96 | Instruction *I = Worklist.pop_back_val(); |
| 97 | assert(!I->getType()->isTokenLikeTy() && |
| 98 | "Token-like values shouldn't be in the worklist" ); |
| 99 | BasicBlock *InstBB = I->getParent(); |
| 100 | Loop *L = LI.getLoopFor(BB: InstBB); |
| 101 | assert(L && "Instruction belongs to a BB that's not part of a loop" ); |
| 102 | auto [It, Inserted] = LoopExitBlocks.try_emplace(Key: L); |
| 103 | if (Inserted) |
| 104 | L->getExitBlocks(ExitBlocks&: It->second); |
| 105 | const SmallVectorImpl<BasicBlock *> &ExitBlocks = It->second; |
| 106 | |
| 107 | if (ExitBlocks.empty()) |
| 108 | continue; |
| 109 | |
| 110 | SmallVector<Instruction *> LifetimeMarkers; |
| 111 | bool DropLifetimeMarkers = false; |
| 112 | for (Use &U : make_early_inc_range(Range: I->uses())) { |
| 113 | Instruction *User = cast<Instruction>(Val: U.getUser()); |
| 114 | BasicBlock *UserBB = User->getParent(); |
| 115 | |
| 116 | // Lifetime markers must refer directly to an alloca. Rewriting their |
| 117 | // operands through LCSSA PHIs would produce invalid IR, so conservatively |
| 118 | // drop all lifetime markers when one crosses the loop boundary. |
| 119 | if (User->isLifetimeStartOrEnd()) { |
| 120 | LifetimeMarkers.push_back(Elt: User); |
| 121 | if (InstBB != UserBB && !L->contains(BB: UserBB)) |
| 122 | DropLifetimeMarkers = true; |
| 123 | continue; |
| 124 | } |
| 125 | |
| 126 | // Skip uses in unreachable blocks. |
| 127 | if (!DT.isReachableFromEntry(A: UserBB)) { |
| 128 | U.set(PoisonValue::get(T: I->getType())); |
| 129 | continue; |
| 130 | } |
| 131 | |
| 132 | // For practical purposes, we consider that the use in a PHI |
| 133 | // occurs in the respective predecessor block. For more info, |
| 134 | // see the `phi` doc in LangRef and the LCSSA doc. |
| 135 | if (auto *PN = dyn_cast<PHINode>(Val: User)) |
| 136 | UserBB = PN->getIncomingBlock(U); |
| 137 | |
| 138 | if (InstBB != UserBB && !L->contains(BB: UserBB)) |
| 139 | UsesToRewrite.push_back(Elt: &U); |
| 140 | } |
| 141 | |
| 142 | if (DropLifetimeMarkers) { |
| 143 | // Use-list order is arbitrary, so wait until all markers are collected. |
| 144 | for (Instruction *Marker : LifetimeMarkers) |
| 145 | Marker->eraseFromParent(); |
| 146 | Changed = true; |
| 147 | } |
| 148 | |
| 149 | // If there are no uses outside the loop, exit with no change. |
| 150 | if (UsesToRewrite.empty()) |
| 151 | continue; |
| 152 | |
| 153 | ++NumLCSSA; // We are applying the transformation |
| 154 | |
| 155 | // Invoke instructions are special in that their result value is not |
| 156 | // available along their unwind edge. The code below tests to see whether |
| 157 | // DomBB dominates the value, so adjust DomBB to the normal destination |
| 158 | // block, which is effectively where the value is first usable. |
| 159 | BasicBlock *DomBB = InstBB; |
| 160 | if (auto *Inv = dyn_cast<InvokeInst>(Val: I)) |
| 161 | DomBB = Inv->getNormalDest(); |
| 162 | |
| 163 | const DomTreeNode *DomNode = DT.getNode(BB: DomBB); |
| 164 | |
| 165 | SmallVector<PHINode *, 16> AddedPHIs; |
| 166 | SmallVector<PHINode *, 8> PostProcessPHIs; |
| 167 | |
| 168 | SmallVector<PHINode *, 4> LocalInsertedPHIs; |
| 169 | SSAUpdater SSAUpdate(&LocalInsertedPHIs); |
| 170 | SSAUpdate.Initialize(Ty: I->getType(), Name: I->getName()); |
| 171 | |
| 172 | // Insert the LCSSA phi's into all of the exit blocks dominated by the |
| 173 | // value, and add them to the Phi's map. |
| 174 | bool HasSCEV = SE && SE->isSCEVable(Ty: I->getType()) && |
| 175 | SE->getExistingSCEV(V: I) != nullptr; |
| 176 | for (BasicBlock *ExitBB : ExitBlocks) { |
| 177 | if (!DT.dominates(A: DomNode, B: DT.getNode(BB: ExitBB))) |
| 178 | continue; |
| 179 | |
| 180 | // If we already inserted something for this BB, don't reprocess it. |
| 181 | if (SSAUpdate.HasValueForBlock(BB: ExitBB)) |
| 182 | continue; |
| 183 | PHINode *PN = PHINode::Create(Ty: I->getType(), NumReservedValues: PredCache.size(BB: ExitBB), |
| 184 | NameStr: I->getName() + ".lcssa" ); |
| 185 | PN->insertBefore(InsertPos: ExitBB->begin()); |
| 186 | if (InsertedPHIs) |
| 187 | InsertedPHIs->push_back(Elt: PN); |
| 188 | // Get the debug location from the original instruction. |
| 189 | PN->setDebugLoc(I->getDebugLoc()); |
| 190 | |
| 191 | // Add inputs from inside the loop for this PHI. This is valid |
| 192 | // because `I` dominates `ExitBB` (checked above). This implies |
| 193 | // that every incoming block/edge is dominated by `I` as well, |
| 194 | // i.e. we can add uses of `I` to those incoming edges/append to the incoming |
| 195 | // blocks without violating the SSA dominance property. |
| 196 | for (BasicBlock *Pred : PredCache.get(BB: ExitBB)) { |
| 197 | PN->addIncoming(V: I, BB: Pred); |
| 198 | |
| 199 | // If the exit block has a predecessor not within the loop, arrange for |
| 200 | // the incoming value use corresponding to that predecessor to be |
| 201 | // rewritten in terms of a different LCSSA PHI. |
| 202 | if (!L->contains(BB: Pred)) |
| 203 | UsesToRewrite.push_back( |
| 204 | Elt: &PN->getOperandUse(i: PN->getOperandNumForIncomingValue( |
| 205 | i: PN->getNumIncomingValues() - 1))); |
| 206 | } |
| 207 | |
| 208 | AddedPHIs.push_back(Elt: PN); |
| 209 | |
| 210 | // Remember that this phi makes the value alive in this block. |
| 211 | SSAUpdate.AddAvailableValue(BB: ExitBB, V: PN); |
| 212 | |
| 213 | // LoopSimplify might fail to simplify some loops (e.g. when indirect |
| 214 | // branches are involved). In such situations, it might happen that an |
| 215 | // exit for Loop L1 is the header of a disjoint Loop L2. Thus, when we |
| 216 | // create PHIs in such an exit block, we are also inserting PHIs into L2's |
| 217 | // header. This could break LCSSA form for L2 because these inserted PHIs |
| 218 | // can also have uses outside of L2. Remember all PHIs in such situation |
| 219 | // as to revisit than later on. FIXME: Remove this if indirectbr support |
| 220 | // into LoopSimplify gets improved. |
| 221 | if (auto *OtherLoop = LI.getLoopFor(BB: ExitBB)) |
| 222 | if (!L->contains(L: OtherLoop)) |
| 223 | PostProcessPHIs.push_back(Elt: PN); |
| 224 | |
| 225 | // If we have a cached SCEV for the original instruction, make sure the |
| 226 | // new LCSSA phi node is also cached. This makes sures that BECounts |
| 227 | // based on it will be invalidated when the LCSSA phi node is invalidated, |
| 228 | // which some passes rely on. |
| 229 | if (HasSCEV) |
| 230 | SE->getSCEV(V: PN); |
| 231 | } |
| 232 | |
| 233 | // Rewrite all uses outside the loop in terms of the new PHIs we just |
| 234 | // inserted. |
| 235 | for (Use *UseToRewrite : UsesToRewrite) { |
| 236 | Instruction *User = cast<Instruction>(Val: UseToRewrite->getUser()); |
| 237 | BasicBlock *UserBB = User->getParent(); |
| 238 | |
| 239 | // For practical purposes, we consider that the use in a PHI |
| 240 | // occurs in the respective predecessor block. For more info, |
| 241 | // see the `phi` doc in LangRef and the LCSSA doc. |
| 242 | if (auto *PN = dyn_cast<PHINode>(Val: User)) |
| 243 | UserBB = PN->getIncomingBlock(U: *UseToRewrite); |
| 244 | |
| 245 | // If this use is in an exit block, rewrite to use the newly inserted PHI. |
| 246 | // This is required for correctness because SSAUpdate doesn't handle uses |
| 247 | // in the same block. It assumes the PHI we inserted is at the end of the |
| 248 | // block. |
| 249 | if (isa<PHINode>(Val: UserBB->begin()) && isExitBlock(BB: UserBB, ExitBlocks)) { |
| 250 | UseToRewrite->set(&UserBB->front()); |
| 251 | continue; |
| 252 | } |
| 253 | |
| 254 | // If we added a single PHI, it must dominate all uses and we can directly |
| 255 | // rename it. |
| 256 | if (AddedPHIs.size() == 1) { |
| 257 | UseToRewrite->set(AddedPHIs[0]); |
| 258 | continue; |
| 259 | } |
| 260 | |
| 261 | // Otherwise, do full PHI insertion. |
| 262 | SSAUpdate.RewriteUse(U&: *UseToRewrite); |
| 263 | } |
| 264 | |
| 265 | SmallVector<DbgVariableRecord *, 4> DbgVariableRecords; |
| 266 | llvm::findDbgValues(V: I, DbgVariableRecords); |
| 267 | |
| 268 | // Update pre-existing debug value uses that reside outside the loop. |
| 269 | for (DbgVariableRecord *DVR : DbgVariableRecords) { |
| 270 | BasicBlock *UserBB = DVR->getMarker()->getParent(); |
| 271 | if (InstBB == UserBB || L->contains(BB: UserBB)) |
| 272 | continue; |
| 273 | // We currently only handle debug values residing in blocks that were |
| 274 | // traversed while rewriting the uses. If we inserted just a single PHI, |
| 275 | // we will handle all relevant debug values. |
| 276 | Value *V = AddedPHIs.size() == 1 ? AddedPHIs[0] |
| 277 | : SSAUpdate.FindValueForBlock(BB: UserBB); |
| 278 | if (V) |
| 279 | DVR->replaceVariableLocationOp(OldValue: I, NewValue: V); |
| 280 | } |
| 281 | |
| 282 | // SSAUpdater might have inserted phi-nodes inside other loops. We'll need |
| 283 | // to post-process them to keep LCSSA form. |
| 284 | for (PHINode *InsertedPN : LocalInsertedPHIs) { |
| 285 | if (auto *OtherLoop = LI.getLoopFor(BB: InsertedPN->getParent())) |
| 286 | if (!L->contains(L: OtherLoop)) |
| 287 | PostProcessPHIs.push_back(Elt: InsertedPN); |
| 288 | if (InsertedPHIs) |
| 289 | InsertedPHIs->push_back(Elt: InsertedPN); |
| 290 | } |
| 291 | |
| 292 | // Post process PHI instructions that were inserted into another disjoint |
| 293 | // loop and update their exits properly. |
| 294 | for (auto *PostProcessPN : PostProcessPHIs) |
| 295 | if (!PostProcessPN->use_empty()) |
| 296 | Worklist.push_back(Elt: PostProcessPN); |
| 297 | |
| 298 | // Keep track of PHI nodes that we want to remove because they did not have |
| 299 | // any uses rewritten. |
| 300 | for (PHINode *PN : AddedPHIs) |
| 301 | if (PN->use_empty()) |
| 302 | LocalPHIsToRemove.insert(X: PN); |
| 303 | |
| 304 | Changed = true; |
| 305 | } |
| 306 | |
| 307 | // Remove PHI nodes that did not have any uses rewritten or add them to |
| 308 | // PHIsToRemove, so the caller can remove them after some additional cleanup. |
| 309 | // We need to redo the use_empty() check here, because even if the PHI node |
| 310 | // wasn't used when added to LocalPHIsToRemove, later added PHI nodes can be |
| 311 | // using it. This cleanup is not guaranteed to handle trees/cycles of PHI |
| 312 | // nodes that only are used by each other. Such situations has only been |
| 313 | // noticed when the input IR contains unreachable code, and leaving some extra |
| 314 | // redundant PHI nodes in such situations is considered a minor problem. |
| 315 | if (PHIsToRemove) { |
| 316 | PHIsToRemove->append(in_start: LocalPHIsToRemove.begin(), in_end: LocalPHIsToRemove.end()); |
| 317 | } else { |
| 318 | for (PHINode *PN : LocalPHIsToRemove) |
| 319 | if (PN->use_empty()) |
| 320 | PN->eraseFromParent(); |
| 321 | } |
| 322 | return Changed; |
| 323 | } |
| 324 | |
| 325 | /// For every instruction from the worklist, check to see if it has any uses |
| 326 | /// that are outside the current loop. If so, insert LCSSA PHI nodes and |
| 327 | /// rewrite the uses. |
| 328 | bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist, |
| 329 | const DominatorTree &DT, const LoopInfo &LI, |
| 330 | ScalarEvolution *SE, |
| 331 | SmallVectorImpl<PHINode *> *PHIsToRemove, |
| 332 | SmallVectorImpl<PHINode *> *InsertedPHIs) { |
| 333 | LoopExitBlocksTy LoopExitBlocks; |
| 334 | |
| 335 | return formLCSSAForInstructionsImpl(Worklist, DT, LI, SE, PHIsToRemove, |
| 336 | InsertedPHIs, LoopExitBlocks); |
| 337 | } |
| 338 | |
| 339 | // Compute the set of BasicBlocks in the loop `L` dominating at least one exit. |
| 340 | static void computeBlocksDominatingExits( |
| 341 | Loop &L, const DominatorTree &DT, ArrayRef<BasicBlock *> ExitBlocks, |
| 342 | SmallSetVector<BasicBlock *, 8> &BlocksDominatingExits) { |
| 343 | // We start from the exit blocks, as every block trivially dominates itself |
| 344 | // (not strictly). |
| 345 | SmallVector<BasicBlock *, 8> BBWorklist(ExitBlocks); |
| 346 | |
| 347 | while (!BBWorklist.empty()) { |
| 348 | BasicBlock *BB = BBWorklist.pop_back_val(); |
| 349 | |
| 350 | // Check if this is a loop header. If this is the case, we're done. |
| 351 | if (L.getHeader() == BB) |
| 352 | continue; |
| 353 | |
| 354 | // Otherwise, add its immediate predecessor in the dominator tree to the |
| 355 | // worklist, unless we visited it already. |
| 356 | BasicBlock *IDomBB = DT.getNode(BB)->getIDom()->getBlock(); |
| 357 | |
| 358 | // Exit blocks can have an immediate dominator not belonging to the |
| 359 | // loop. For an exit block to be immediately dominated by another block |
| 360 | // outside the loop, it implies not all paths from that dominator, to the |
| 361 | // exit block, go through the loop. |
| 362 | // Example: |
| 363 | // |
| 364 | // |---- A |
| 365 | // | | |
| 366 | // | B<-- |
| 367 | // | | | |
| 368 | // |---> C -- |
| 369 | // | |
| 370 | // D |
| 371 | // |
| 372 | // C is the exit block of the loop and it's immediately dominated by A, |
| 373 | // which doesn't belong to the loop. |
| 374 | if (!L.contains(BB: IDomBB)) |
| 375 | continue; |
| 376 | |
| 377 | if (BlocksDominatingExits.insert(X: IDomBB)) |
| 378 | BBWorklist.push_back(Elt: IDomBB); |
| 379 | } |
| 380 | } |
| 381 | |
| 382 | static bool formLCSSAImpl(Loop &L, const DominatorTree &DT, const LoopInfo *LI, |
| 383 | ScalarEvolution *SE, |
| 384 | LoopExitBlocksTy &LoopExitBlocks) { |
| 385 | bool Changed = false; |
| 386 | |
| 387 | #ifdef EXPENSIVE_CHECKS |
| 388 | // Verify all sub-loops are in LCSSA form already. |
| 389 | for (Loop *SubLoop: L) { |
| 390 | (void)SubLoop; // Silence unused variable warning. |
| 391 | assert(SubLoop->isRecursivelyLCSSAForm(DT, *LI) && "Subloop not in LCSSA!" ); |
| 392 | } |
| 393 | #endif |
| 394 | |
| 395 | auto [It, Inserted] = LoopExitBlocks.try_emplace(Key: &L); |
| 396 | if (Inserted) |
| 397 | L.getExitBlocks(ExitBlocks&: It->second); |
| 398 | const SmallVectorImpl<BasicBlock *> &ExitBlocks = It->second; |
| 399 | if (ExitBlocks.empty()) |
| 400 | return false; |
| 401 | |
| 402 | SmallSetVector<BasicBlock *, 8> BlocksDominatingExits; |
| 403 | |
| 404 | // We want to avoid use-scanning leveraging dominance informations. |
| 405 | // If a block doesn't dominate any of the loop exits, the none of the values |
| 406 | // defined in the loop can be used outside. |
| 407 | // We compute the set of blocks fullfilling the conditions in advance |
| 408 | // walking the dominator tree upwards until we hit a loop header. |
| 409 | computeBlocksDominatingExits(L, DT, ExitBlocks, BlocksDominatingExits); |
| 410 | |
| 411 | SmallVector<Instruction *, 8> Worklist; |
| 412 | |
| 413 | // Look at all the instructions in the loop, checking to see if they have uses |
| 414 | // outside the loop. If so, put them into the worklist to rewrite those uses. |
| 415 | for (BasicBlock *BB : BlocksDominatingExits) { |
| 416 | // Skip blocks that are part of any sub-loops, they must be in LCSSA |
| 417 | // already. |
| 418 | if (LI->getLoopFor(BB) != &L) |
| 419 | continue; |
| 420 | for (Instruction &I : *BB) { |
| 421 | // Reject two common cases fast: instructions with no uses (like stores) |
| 422 | // and instructions with one use that is in the same block as this. |
| 423 | if (I.use_empty() || |
| 424 | (I.hasOneUse() && I.user_back()->getParent() == BB && |
| 425 | !isa<PHINode>(Val: I.user_back()))) |
| 426 | continue; |
| 427 | |
| 428 | // Token-like values cannot be used in PHI nodes, so we skip over them. |
| 429 | // We can run into tokens which are live out of a loop with catchswitch |
| 430 | // instructions in Windows EH if the catchswitch has one catchpad which |
| 431 | // is inside the loop and another which is not. |
| 432 | if (I.getType()->isTokenLikeTy()) |
| 433 | continue; |
| 434 | |
| 435 | Worklist.push_back(Elt: &I); |
| 436 | } |
| 437 | } |
| 438 | |
| 439 | Changed = formLCSSAForInstructionsImpl(Worklist, DT, LI: *LI, SE, PHIsToRemove: nullptr, |
| 440 | InsertedPHIs: nullptr, LoopExitBlocks); |
| 441 | |
| 442 | assert(L.isLCSSAForm(DT)); |
| 443 | |
| 444 | return Changed; |
| 445 | } |
| 446 | |
| 447 | bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI, |
| 448 | ScalarEvolution *SE) { |
| 449 | LoopExitBlocksTy LoopExitBlocks; |
| 450 | |
| 451 | return formLCSSAImpl(L, DT, LI, SE, LoopExitBlocks); |
| 452 | } |
| 453 | |
| 454 | /// Process a loop nest depth first. |
| 455 | static bool formLCSSARecursivelyImpl(Loop &L, const DominatorTree &DT, |
| 456 | const LoopInfo *LI, ScalarEvolution *SE, |
| 457 | LoopExitBlocksTy &LoopExitBlocks) { |
| 458 | bool Changed = false; |
| 459 | |
| 460 | // Recurse depth-first through inner loops. |
| 461 | for (Loop *SubLoop : L.getSubLoops()) |
| 462 | Changed |= formLCSSARecursivelyImpl(L&: *SubLoop, DT, LI, SE, LoopExitBlocks); |
| 463 | |
| 464 | Changed |= formLCSSAImpl(L, DT, LI, SE, LoopExitBlocks); |
| 465 | return Changed; |
| 466 | } |
| 467 | |
| 468 | /// Process a loop nest depth first. |
| 469 | bool llvm::formLCSSARecursively(Loop &L, const DominatorTree &DT, |
| 470 | const LoopInfo *LI, ScalarEvolution *SE) { |
| 471 | LoopExitBlocksTy LoopExitBlocks; |
| 472 | |
| 473 | return formLCSSARecursivelyImpl(L, DT, LI, SE, LoopExitBlocks); |
| 474 | } |
| 475 | |
| 476 | /// Process all loops in the function, inner-most out. |
| 477 | static bool formLCSSAOnAllLoops(const LoopInfo *LI, const DominatorTree &DT, |
| 478 | ScalarEvolution *SE) { |
| 479 | bool Changed = false; |
| 480 | for (const auto &L : *LI) |
| 481 | Changed |= formLCSSARecursively(L&: *L, DT, LI, SE); |
| 482 | return Changed; |
| 483 | } |
| 484 | |
| 485 | namespace { |
| 486 | struct LCSSAWrapperPass : public FunctionPass { |
| 487 | static char ID; // Pass identification, replacement for typeid |
| 488 | LCSSAWrapperPass() : FunctionPass(ID) { |
| 489 | initializeLCSSAWrapperPassPass(*PassRegistry::getPassRegistry()); |
| 490 | } |
| 491 | |
| 492 | // Cached analysis information for the current function. |
| 493 | DominatorTree *DT; |
| 494 | LoopInfo *LI; |
| 495 | ScalarEvolution *SE; |
| 496 | |
| 497 | bool runOnFunction(Function &F) override; |
| 498 | void verifyAnalysis() const override { |
| 499 | // This check is very expensive. On the loop intensive compiles it may cause |
| 500 | // up to 10x slowdown. Currently it's disabled by default. LPPassManager |
| 501 | // always does limited form of the LCSSA verification. Similar reasoning |
| 502 | // was used for the LoopInfo verifier. |
| 503 | if (VerifyLoopLCSSA) { |
| 504 | assert(all_of(*LI, |
| 505 | [&](Loop *L) { |
| 506 | return L->isRecursivelyLCSSAForm(*DT, *LI); |
| 507 | }) && |
| 508 | "LCSSA form is broken!" ); |
| 509 | } |
| 510 | }; |
| 511 | |
| 512 | /// This transformation requires natural loop information & requires that |
| 513 | /// loop preheaders be inserted into the CFG. It maintains both of these, |
| 514 | /// as well as the CFG. It also requires dominator information. |
| 515 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 516 | AU.setPreservesCFG(); |
| 517 | |
| 518 | AU.addRequired<DominatorTreeWrapperPass>(); |
| 519 | AU.addRequired<LoopInfoWrapperPass>(); |
| 520 | AU.addPreservedID(ID&: LoopSimplifyID); |
| 521 | AU.addPreserved<AAResultsWrapperPass>(); |
| 522 | AU.addPreserved<GlobalsAAWrapperPass>(); |
| 523 | AU.addPreserved<ScalarEvolutionWrapperPass>(); |
| 524 | AU.addPreserved<SCEVAAWrapperPass>(); |
| 525 | AU.addPreserved<BranchProbabilityInfoWrapperPass>(); |
| 526 | AU.addPreserved<MemorySSAWrapperPass>(); |
| 527 | |
| 528 | // This is needed to perform LCSSA verification inside LPPassManager |
| 529 | AU.addRequired<LCSSAVerificationPass>(); |
| 530 | AU.addPreserved<LCSSAVerificationPass>(); |
| 531 | } |
| 532 | }; |
| 533 | } |
| 534 | |
| 535 | char LCSSAWrapperPass::ID = 0; |
| 536 | INITIALIZE_PASS_BEGIN(LCSSAWrapperPass, "lcssa" , "Loop-Closed SSA Form Pass" , |
| 537 | false, false) |
| 538 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) |
| 539 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) |
| 540 | INITIALIZE_PASS_DEPENDENCY(LCSSAVerificationPass) |
| 541 | INITIALIZE_PASS_END(LCSSAWrapperPass, "lcssa" , "Loop-Closed SSA Form Pass" , |
| 542 | false, false) |
| 543 | |
| 544 | Pass *llvm::createLCSSAPass() { return new LCSSAWrapperPass(); } |
| 545 | char &llvm::LCSSAID = LCSSAWrapperPass::ID; |
| 546 | |
| 547 | /// Transform \p F into loop-closed SSA form. |
| 548 | bool LCSSAWrapperPass::runOnFunction(Function &F) { |
| 549 | LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); |
| 550 | DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); |
| 551 | auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>(); |
| 552 | SE = SEWP ? &SEWP->getSE() : nullptr; |
| 553 | |
| 554 | return formLCSSAOnAllLoops(LI, DT: *DT, SE); |
| 555 | } |
| 556 | |
| 557 | PreservedAnalyses LCSSAPass::run(Function &F, FunctionAnalysisManager &AM) { |
| 558 | auto &LI = AM.getResult<LoopAnalysis>(IR&: F); |
| 559 | auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F); |
| 560 | auto *SE = AM.getCachedResult<ScalarEvolutionAnalysis>(IR&: F); |
| 561 | if (!formLCSSAOnAllLoops(LI: &LI, DT, SE)) |
| 562 | return PreservedAnalyses::all(); |
| 563 | |
| 564 | PreservedAnalyses PA; |
| 565 | PA.preserveSet<CFGAnalyses>(); |
| 566 | PA.preserve<ScalarEvolutionAnalysis>(); |
| 567 | PA.preserve<MemorySSAAnalysis>(); |
| 568 | return PA; |
| 569 | } |
| 570 | |