| 1 | //===-- LCSSA.cpp - Convert loops into loop-closed SSA form ---------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass transforms loops by placing phi nodes at the end of the loops for |
| 10 | // all values that are live across the loop boundary. For example, it turns |
| 11 | // the left into the right code: |
| 12 | // |
| 13 | // for (...) for (...) |
| 14 | // if (c) if (c) |
| 15 | // X1 = ... X1 = ... |
| 16 | // else else |
| 17 | // X2 = ... X2 = ... |
| 18 | // X3 = phi(X1, X2) X3 = phi(X1, X2) |
| 19 | // ... = X3 + 4 X4 = phi(X3) |
| 20 | // ... = X4 + 4 |
| 21 | // |
| 22 | // This is still valid LLVM; the extra phi nodes are purely redundant, and will |
| 23 | // be trivially eliminated by InstCombine. The major benefit of this |
| 24 | // transformation is that it makes many other loop optimizations, such as |
| 25 | // LoopUnswitching, simpler. |
| 26 | // |
| 27 | //===----------------------------------------------------------------------===// |
| 28 | |
| 29 | #include "llvm/Transforms/Utils/LCSSA.h" |
| 30 | #include "llvm/ADT/STLExtras.h" |
| 31 | #include "llvm/ADT/Statistic.h" |
| 32 | #include "llvm/Analysis/AliasAnalysis.h" |
| 33 | #include "llvm/Analysis/BasicAliasAnalysis.h" |
| 34 | #include "llvm/Analysis/BranchProbabilityInfo.h" |
| 35 | #include "llvm/Analysis/GlobalsModRef.h" |
| 36 | #include "llvm/Analysis/LoopInfo.h" |
| 37 | #include "llvm/Analysis/LoopPass.h" |
| 38 | #include "llvm/Analysis/MemorySSA.h" |
| 39 | #include "llvm/Analysis/ScalarEvolution.h" |
| 40 | #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h" |
| 41 | #include "llvm/IR/DebugInfo.h" |
| 42 | #include "llvm/IR/Dominators.h" |
| 43 | #include "llvm/IR/Instructions.h" |
| 44 | #include "llvm/IR/IntrinsicInst.h" |
| 45 | #include "llvm/IR/PredIteratorCache.h" |
| 46 | #include "llvm/InitializePasses.h" |
| 47 | #include "llvm/Pass.h" |
| 48 | #include "llvm/Support/CommandLine.h" |
| 49 | #include "llvm/Transforms/Utils.h" |
| 50 | #include "llvm/Transforms/Utils/LoopUtils.h" |
| 51 | #include "llvm/Transforms/Utils/SSAUpdater.h" |
| 52 | using namespace llvm; |
| 53 | |
| 54 | #define DEBUG_TYPE "lcssa" |
| 55 | |
| 56 | STATISTIC(NumLCSSA, "Number of live out of a loop variables" ); |
| 57 | |
| 58 | #ifdef EXPENSIVE_CHECKS |
| 59 | static bool VerifyLoopLCSSA = true; |
| 60 | #else |
| 61 | static bool VerifyLoopLCSSA = false; |
| 62 | #endif |
| 63 | static cl::opt<bool, true> |
| 64 | VerifyLoopLCSSAFlag("verify-loop-lcssa" , cl::location(L&: VerifyLoopLCSSA), |
| 65 | cl::Hidden, |
| 66 | cl::desc("Verify loop lcssa form (time consuming)" )); |
| 67 | |
| 68 | /// Return true if the specified block is in the list. |
| 69 | static bool isExitBlock(BasicBlock *BB, |
| 70 | const SmallVectorImpl<BasicBlock *> &ExitBlocks) { |
| 71 | return is_contained(Range: ExitBlocks, Element: BB); |
| 72 | } |
| 73 | |
| 74 | // Cache the Loop ExitBlocks computed during the analysis. We expect to get a |
| 75 | // lot of instructions within the same loops, computing the exit blocks is |
| 76 | // expensive, and we're not mutating the loop structure. |
| 77 | using LoopExitBlocksTy = SmallDenseMap<Loop *, SmallVector<BasicBlock *, 1>>; |
| 78 | |
| 79 | /// For every instruction from the worklist, check to see if it has any uses |
| 80 | /// that are outside the current loop. If so, insert LCSSA PHI nodes and |
| 81 | /// rewrite the uses. |
| 82 | static bool |
| 83 | formLCSSAForInstructionsImpl(SmallVectorImpl<Instruction *> &Worklist, |
| 84 | const DominatorTree &DT, const LoopInfo &LI, |
| 85 | ScalarEvolution *SE, |
| 86 | SmallVectorImpl<PHINode *> *PHIsToRemove, |
| 87 | SmallVectorImpl<PHINode *> *InsertedPHIs, |
| 88 | LoopExitBlocksTy &LoopExitBlocks) { |
| 89 | SmallVector<Use *, 16> UsesToRewrite; |
| 90 | SmallSetVector<PHINode *, 16> LocalPHIsToRemove; |
| 91 | PredIteratorCache PredCache; |
| 92 | bool Changed = false; |
| 93 | |
| 94 | while (!Worklist.empty()) { |
| 95 | UsesToRewrite.clear(); |
| 96 | |
| 97 | Instruction *I = Worklist.pop_back_val(); |
| 98 | assert(!I->getType()->isTokenTy() && "Tokens shouldn't be in the worklist" ); |
| 99 | BasicBlock *InstBB = I->getParent(); |
| 100 | Loop *L = LI.getLoopFor(BB: InstBB); |
| 101 | assert(L && "Instruction belongs to a BB that's not part of a loop" ); |
| 102 | auto [It, Inserted] = LoopExitBlocks.try_emplace(Key: L); |
| 103 | if (Inserted) |
| 104 | L->getExitBlocks(ExitBlocks&: It->second); |
| 105 | const SmallVectorImpl<BasicBlock *> &ExitBlocks = It->second; |
| 106 | |
| 107 | if (ExitBlocks.empty()) |
| 108 | continue; |
| 109 | |
| 110 | for (Use &U : make_early_inc_range(Range: I->uses())) { |
| 111 | Instruction *User = cast<Instruction>(Val: U.getUser()); |
| 112 | BasicBlock *UserBB = User->getParent(); |
| 113 | |
| 114 | // Skip uses in unreachable blocks. |
| 115 | if (!DT.isReachableFromEntry(A: UserBB)) { |
| 116 | U.set(PoisonValue::get(T: I->getType())); |
| 117 | continue; |
| 118 | } |
| 119 | |
| 120 | // For practical purposes, we consider that the use in a PHI |
| 121 | // occurs in the respective predecessor block. For more info, |
| 122 | // see the `phi` doc in LangRef and the LCSSA doc. |
| 123 | if (auto *PN = dyn_cast<PHINode>(Val: User)) |
| 124 | UserBB = PN->getIncomingBlock(U); |
| 125 | |
| 126 | if (InstBB != UserBB && !L->contains(BB: UserBB)) |
| 127 | UsesToRewrite.push_back(Elt: &U); |
| 128 | } |
| 129 | |
| 130 | // If there are no uses outside the loop, exit with no change. |
| 131 | if (UsesToRewrite.empty()) |
| 132 | continue; |
| 133 | |
| 134 | ++NumLCSSA; // We are applying the transformation |
| 135 | |
| 136 | // Invoke instructions are special in that their result value is not |
| 137 | // available along their unwind edge. The code below tests to see whether |
| 138 | // DomBB dominates the value, so adjust DomBB to the normal destination |
| 139 | // block, which is effectively where the value is first usable. |
| 140 | BasicBlock *DomBB = InstBB; |
| 141 | if (auto *Inv = dyn_cast<InvokeInst>(Val: I)) |
| 142 | DomBB = Inv->getNormalDest(); |
| 143 | |
| 144 | const DomTreeNode *DomNode = DT.getNode(BB: DomBB); |
| 145 | |
| 146 | SmallVector<PHINode *, 16> AddedPHIs; |
| 147 | SmallVector<PHINode *, 8> PostProcessPHIs; |
| 148 | |
| 149 | SmallVector<PHINode *, 4> LocalInsertedPHIs; |
| 150 | SSAUpdater SSAUpdate(&LocalInsertedPHIs); |
| 151 | SSAUpdate.Initialize(Ty: I->getType(), Name: I->getName()); |
| 152 | |
| 153 | // Insert the LCSSA phi's into all of the exit blocks dominated by the |
| 154 | // value, and add them to the Phi's map. |
| 155 | bool HasSCEV = SE && SE->isSCEVable(Ty: I->getType()) && |
| 156 | SE->getExistingSCEV(V: I) != nullptr; |
| 157 | for (BasicBlock *ExitBB : ExitBlocks) { |
| 158 | if (!DT.dominates(A: DomNode, B: DT.getNode(BB: ExitBB))) |
| 159 | continue; |
| 160 | |
| 161 | // If we already inserted something for this BB, don't reprocess it. |
| 162 | if (SSAUpdate.HasValueForBlock(BB: ExitBB)) |
| 163 | continue; |
| 164 | PHINode *PN = PHINode::Create(Ty: I->getType(), NumReservedValues: PredCache.size(BB: ExitBB), |
| 165 | NameStr: I->getName() + ".lcssa" ); |
| 166 | PN->insertBefore(InsertPos: ExitBB->begin()); |
| 167 | if (InsertedPHIs) |
| 168 | InsertedPHIs->push_back(Elt: PN); |
| 169 | // Get the debug location from the original instruction. |
| 170 | PN->setDebugLoc(I->getDebugLoc()); |
| 171 | |
| 172 | // Add inputs from inside the loop for this PHI. This is valid |
| 173 | // because `I` dominates `ExitBB` (checked above). This implies |
| 174 | // that every incoming block/edge is dominated by `I` as well, |
| 175 | // i.e. we can add uses of `I` to those incoming edges/append to the incoming |
| 176 | // blocks without violating the SSA dominance property. |
| 177 | for (BasicBlock *Pred : PredCache.get(BB: ExitBB)) { |
| 178 | PN->addIncoming(V: I, BB: Pred); |
| 179 | |
| 180 | // If the exit block has a predecessor not within the loop, arrange for |
| 181 | // the incoming value use corresponding to that predecessor to be |
| 182 | // rewritten in terms of a different LCSSA PHI. |
| 183 | if (!L->contains(BB: Pred)) |
| 184 | UsesToRewrite.push_back( |
| 185 | Elt: &PN->getOperandUse(i: PN->getOperandNumForIncomingValue( |
| 186 | i: PN->getNumIncomingValues() - 1))); |
| 187 | } |
| 188 | |
| 189 | AddedPHIs.push_back(Elt: PN); |
| 190 | |
| 191 | // Remember that this phi makes the value alive in this block. |
| 192 | SSAUpdate.AddAvailableValue(BB: ExitBB, V: PN); |
| 193 | |
| 194 | // LoopSimplify might fail to simplify some loops (e.g. when indirect |
| 195 | // branches are involved). In such situations, it might happen that an |
| 196 | // exit for Loop L1 is the header of a disjoint Loop L2. Thus, when we |
| 197 | // create PHIs in such an exit block, we are also inserting PHIs into L2's |
| 198 | // header. This could break LCSSA form for L2 because these inserted PHIs |
| 199 | // can also have uses outside of L2. Remember all PHIs in such situation |
| 200 | // as to revisit than later on. FIXME: Remove this if indirectbr support |
| 201 | // into LoopSimplify gets improved. |
| 202 | if (auto *OtherLoop = LI.getLoopFor(BB: ExitBB)) |
| 203 | if (!L->contains(L: OtherLoop)) |
| 204 | PostProcessPHIs.push_back(Elt: PN); |
| 205 | |
| 206 | // If we have a cached SCEV for the original instruction, make sure the |
| 207 | // new LCSSA phi node is also cached. This makes sures that BECounts |
| 208 | // based on it will be invalidated when the LCSSA phi node is invalidated, |
| 209 | // which some passes rely on. |
| 210 | if (HasSCEV) |
| 211 | SE->getSCEV(V: PN); |
| 212 | } |
| 213 | |
| 214 | // Rewrite all uses outside the loop in terms of the new PHIs we just |
| 215 | // inserted. |
| 216 | for (Use *UseToRewrite : UsesToRewrite) { |
| 217 | Instruction *User = cast<Instruction>(Val: UseToRewrite->getUser()); |
| 218 | BasicBlock *UserBB = User->getParent(); |
| 219 | |
| 220 | // For practical purposes, we consider that the use in a PHI |
| 221 | // occurs in the respective predecessor block. For more info, |
| 222 | // see the `phi` doc in LangRef and the LCSSA doc. |
| 223 | if (auto *PN = dyn_cast<PHINode>(Val: User)) |
| 224 | UserBB = PN->getIncomingBlock(U: *UseToRewrite); |
| 225 | |
| 226 | // If this use is in an exit block, rewrite to use the newly inserted PHI. |
| 227 | // This is required for correctness because SSAUpdate doesn't handle uses |
| 228 | // in the same block. It assumes the PHI we inserted is at the end of the |
| 229 | // block. |
| 230 | if (isa<PHINode>(Val: UserBB->begin()) && isExitBlock(BB: UserBB, ExitBlocks)) { |
| 231 | UseToRewrite->set(&UserBB->front()); |
| 232 | continue; |
| 233 | } |
| 234 | |
| 235 | // If we added a single PHI, it must dominate all uses and we can directly |
| 236 | // rename it. |
| 237 | if (AddedPHIs.size() == 1) { |
| 238 | UseToRewrite->set(AddedPHIs[0]); |
| 239 | continue; |
| 240 | } |
| 241 | |
| 242 | // Otherwise, do full PHI insertion. |
| 243 | SSAUpdate.RewriteUse(U&: *UseToRewrite); |
| 244 | } |
| 245 | |
| 246 | SmallVector<DbgValueInst *, 4> DbgValues; |
| 247 | SmallVector<DbgVariableRecord *, 4> DbgVariableRecords; |
| 248 | llvm::findDbgValues(DbgValues, V: I, DbgVariableRecords: &DbgVariableRecords); |
| 249 | |
| 250 | // Update pre-existing debug value uses that reside outside the loop. |
| 251 | for (auto *DVI : DbgValues) { |
| 252 | BasicBlock *UserBB = DVI->getParent(); |
| 253 | if (InstBB == UserBB || L->contains(BB: UserBB)) |
| 254 | continue; |
| 255 | // We currently only handle debug values residing in blocks that were |
| 256 | // traversed while rewriting the uses. If we inserted just a single PHI, |
| 257 | // we will handle all relevant debug values. |
| 258 | Value *V = AddedPHIs.size() == 1 ? AddedPHIs[0] |
| 259 | : SSAUpdate.FindValueForBlock(BB: UserBB); |
| 260 | if (V) |
| 261 | DVI->replaceVariableLocationOp(OldValue: I, NewValue: V); |
| 262 | } |
| 263 | |
| 264 | // RemoveDIs: copy-paste of block above, using non-instruction debug-info |
| 265 | // records. |
| 266 | for (DbgVariableRecord *DVR : DbgVariableRecords) { |
| 267 | BasicBlock *UserBB = DVR->getMarker()->getParent(); |
| 268 | if (InstBB == UserBB || L->contains(BB: UserBB)) |
| 269 | continue; |
| 270 | // We currently only handle debug values residing in blocks that were |
| 271 | // traversed while rewriting the uses. If we inserted just a single PHI, |
| 272 | // we will handle all relevant debug values. |
| 273 | Value *V = AddedPHIs.size() == 1 ? AddedPHIs[0] |
| 274 | : SSAUpdate.FindValueForBlock(BB: UserBB); |
| 275 | if (V) |
| 276 | DVR->replaceVariableLocationOp(OldValue: I, NewValue: V); |
| 277 | } |
| 278 | |
| 279 | // SSAUpdater might have inserted phi-nodes inside other loops. We'll need |
| 280 | // to post-process them to keep LCSSA form. |
| 281 | for (PHINode *InsertedPN : LocalInsertedPHIs) { |
| 282 | if (auto *OtherLoop = LI.getLoopFor(BB: InsertedPN->getParent())) |
| 283 | if (!L->contains(L: OtherLoop)) |
| 284 | PostProcessPHIs.push_back(Elt: InsertedPN); |
| 285 | if (InsertedPHIs) |
| 286 | InsertedPHIs->push_back(Elt: InsertedPN); |
| 287 | } |
| 288 | |
| 289 | // Post process PHI instructions that were inserted into another disjoint |
| 290 | // loop and update their exits properly. |
| 291 | for (auto *PostProcessPN : PostProcessPHIs) |
| 292 | if (!PostProcessPN->use_empty()) |
| 293 | Worklist.push_back(Elt: PostProcessPN); |
| 294 | |
| 295 | // Keep track of PHI nodes that we want to remove because they did not have |
| 296 | // any uses rewritten. |
| 297 | for (PHINode *PN : AddedPHIs) |
| 298 | if (PN->use_empty()) |
| 299 | LocalPHIsToRemove.insert(X: PN); |
| 300 | |
| 301 | Changed = true; |
| 302 | } |
| 303 | |
| 304 | // Remove PHI nodes that did not have any uses rewritten or add them to |
| 305 | // PHIsToRemove, so the caller can remove them after some additional cleanup. |
| 306 | // We need to redo the use_empty() check here, because even if the PHI node |
| 307 | // wasn't used when added to LocalPHIsToRemove, later added PHI nodes can be |
| 308 | // using it. This cleanup is not guaranteed to handle trees/cycles of PHI |
| 309 | // nodes that only are used by each other. Such situations has only been |
| 310 | // noticed when the input IR contains unreachable code, and leaving some extra |
| 311 | // redundant PHI nodes in such situations is considered a minor problem. |
| 312 | if (PHIsToRemove) { |
| 313 | PHIsToRemove->append(in_start: LocalPHIsToRemove.begin(), in_end: LocalPHIsToRemove.end()); |
| 314 | } else { |
| 315 | for (PHINode *PN : LocalPHIsToRemove) |
| 316 | if (PN->use_empty()) |
| 317 | PN->eraseFromParent(); |
| 318 | } |
| 319 | return Changed; |
| 320 | } |
| 321 | |
| 322 | /// For every instruction from the worklist, check to see if it has any uses |
| 323 | /// that are outside the current loop. If so, insert LCSSA PHI nodes and |
| 324 | /// rewrite the uses. |
| 325 | bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist, |
| 326 | const DominatorTree &DT, const LoopInfo &LI, |
| 327 | ScalarEvolution *SE, |
| 328 | SmallVectorImpl<PHINode *> *PHIsToRemove, |
| 329 | SmallVectorImpl<PHINode *> *InsertedPHIs) { |
| 330 | LoopExitBlocksTy LoopExitBlocks; |
| 331 | |
| 332 | return formLCSSAForInstructionsImpl(Worklist, DT, LI, SE, PHIsToRemove, |
| 333 | InsertedPHIs, LoopExitBlocks); |
| 334 | } |
| 335 | |
| 336 | // Compute the set of BasicBlocks in the loop `L` dominating at least one exit. |
| 337 | static void computeBlocksDominatingExits( |
| 338 | Loop &L, const DominatorTree &DT, ArrayRef<BasicBlock *> ExitBlocks, |
| 339 | SmallSetVector<BasicBlock *, 8> &BlocksDominatingExits) { |
| 340 | // We start from the exit blocks, as every block trivially dominates itself |
| 341 | // (not strictly). |
| 342 | SmallVector<BasicBlock *, 8> BBWorklist(ExitBlocks); |
| 343 | |
| 344 | while (!BBWorklist.empty()) { |
| 345 | BasicBlock *BB = BBWorklist.pop_back_val(); |
| 346 | |
| 347 | // Check if this is a loop header. If this is the case, we're done. |
| 348 | if (L.getHeader() == BB) |
| 349 | continue; |
| 350 | |
| 351 | // Otherwise, add its immediate predecessor in the dominator tree to the |
| 352 | // worklist, unless we visited it already. |
| 353 | BasicBlock *IDomBB = DT.getNode(BB)->getIDom()->getBlock(); |
| 354 | |
| 355 | // Exit blocks can have an immediate dominator not belonging to the |
| 356 | // loop. For an exit block to be immediately dominated by another block |
| 357 | // outside the loop, it implies not all paths from that dominator, to the |
| 358 | // exit block, go through the loop. |
| 359 | // Example: |
| 360 | // |
| 361 | // |---- A |
| 362 | // | | |
| 363 | // | B<-- |
| 364 | // | | | |
| 365 | // |---> C -- |
| 366 | // | |
| 367 | // D |
| 368 | // |
| 369 | // C is the exit block of the loop and it's immediately dominated by A, |
| 370 | // which doesn't belong to the loop. |
| 371 | if (!L.contains(BB: IDomBB)) |
| 372 | continue; |
| 373 | |
| 374 | if (BlocksDominatingExits.insert(X: IDomBB)) |
| 375 | BBWorklist.push_back(Elt: IDomBB); |
| 376 | } |
| 377 | } |
| 378 | |
| 379 | static bool formLCSSAImpl(Loop &L, const DominatorTree &DT, const LoopInfo *LI, |
| 380 | ScalarEvolution *SE, |
| 381 | LoopExitBlocksTy &LoopExitBlocks) { |
| 382 | bool Changed = false; |
| 383 | |
| 384 | #ifdef EXPENSIVE_CHECKS |
| 385 | // Verify all sub-loops are in LCSSA form already. |
| 386 | for (Loop *SubLoop: L) { |
| 387 | (void)SubLoop; // Silence unused variable warning. |
| 388 | assert(SubLoop->isRecursivelyLCSSAForm(DT, *LI) && "Subloop not in LCSSA!" ); |
| 389 | } |
| 390 | #endif |
| 391 | |
| 392 | auto [It, Inserted] = LoopExitBlocks.try_emplace(Key: &L); |
| 393 | if (Inserted) |
| 394 | L.getExitBlocks(ExitBlocks&: It->second); |
| 395 | const SmallVectorImpl<BasicBlock *> &ExitBlocks = It->second; |
| 396 | if (ExitBlocks.empty()) |
| 397 | return false; |
| 398 | |
| 399 | SmallSetVector<BasicBlock *, 8> BlocksDominatingExits; |
| 400 | |
| 401 | // We want to avoid use-scanning leveraging dominance informations. |
| 402 | // If a block doesn't dominate any of the loop exits, the none of the values |
| 403 | // defined in the loop can be used outside. |
| 404 | // We compute the set of blocks fullfilling the conditions in advance |
| 405 | // walking the dominator tree upwards until we hit a loop header. |
| 406 | computeBlocksDominatingExits(L, DT, ExitBlocks, BlocksDominatingExits); |
| 407 | |
| 408 | SmallVector<Instruction *, 8> Worklist; |
| 409 | |
| 410 | // Look at all the instructions in the loop, checking to see if they have uses |
| 411 | // outside the loop. If so, put them into the worklist to rewrite those uses. |
| 412 | for (BasicBlock *BB : BlocksDominatingExits) { |
| 413 | // Skip blocks that are part of any sub-loops, they must be in LCSSA |
| 414 | // already. |
| 415 | if (LI->getLoopFor(BB) != &L) |
| 416 | continue; |
| 417 | for (Instruction &I : *BB) { |
| 418 | // Reject two common cases fast: instructions with no uses (like stores) |
| 419 | // and instructions with one use that is in the same block as this. |
| 420 | if (I.use_empty() || |
| 421 | (I.hasOneUse() && I.user_back()->getParent() == BB && |
| 422 | !isa<PHINode>(Val: I.user_back()))) |
| 423 | continue; |
| 424 | |
| 425 | // Tokens cannot be used in PHI nodes, so we skip over them. |
| 426 | // We can run into tokens which are live out of a loop with catchswitch |
| 427 | // instructions in Windows EH if the catchswitch has one catchpad which |
| 428 | // is inside the loop and another which is not. |
| 429 | if (I.getType()->isTokenTy()) |
| 430 | continue; |
| 431 | |
| 432 | Worklist.push_back(Elt: &I); |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | Changed = formLCSSAForInstructionsImpl(Worklist, DT, LI: *LI, SE, PHIsToRemove: nullptr, |
| 437 | InsertedPHIs: nullptr, LoopExitBlocks); |
| 438 | |
| 439 | assert(L.isLCSSAForm(DT)); |
| 440 | |
| 441 | return Changed; |
| 442 | } |
| 443 | |
| 444 | bool llvm::formLCSSA(Loop &L, const DominatorTree &DT, const LoopInfo *LI, |
| 445 | ScalarEvolution *SE) { |
| 446 | LoopExitBlocksTy LoopExitBlocks; |
| 447 | |
| 448 | return formLCSSAImpl(L, DT, LI, SE, LoopExitBlocks); |
| 449 | } |
| 450 | |
| 451 | /// Process a loop nest depth first. |
| 452 | static bool formLCSSARecursivelyImpl(Loop &L, const DominatorTree &DT, |
| 453 | const LoopInfo *LI, ScalarEvolution *SE, |
| 454 | LoopExitBlocksTy &LoopExitBlocks) { |
| 455 | bool Changed = false; |
| 456 | |
| 457 | // Recurse depth-first through inner loops. |
| 458 | for (Loop *SubLoop : L.getSubLoops()) |
| 459 | Changed |= formLCSSARecursivelyImpl(L&: *SubLoop, DT, LI, SE, LoopExitBlocks); |
| 460 | |
| 461 | Changed |= formLCSSAImpl(L, DT, LI, SE, LoopExitBlocks); |
| 462 | return Changed; |
| 463 | } |
| 464 | |
| 465 | /// Process a loop nest depth first. |
| 466 | bool llvm::formLCSSARecursively(Loop &L, const DominatorTree &DT, |
| 467 | const LoopInfo *LI, ScalarEvolution *SE) { |
| 468 | LoopExitBlocksTy LoopExitBlocks; |
| 469 | |
| 470 | return formLCSSARecursivelyImpl(L, DT, LI, SE, LoopExitBlocks); |
| 471 | } |
| 472 | |
| 473 | /// Process all loops in the function, inner-most out. |
| 474 | static bool formLCSSAOnAllLoops(const LoopInfo *LI, const DominatorTree &DT, |
| 475 | ScalarEvolution *SE) { |
| 476 | bool Changed = false; |
| 477 | for (const auto &L : *LI) |
| 478 | Changed |= formLCSSARecursively(L&: *L, DT, LI, SE); |
| 479 | return Changed; |
| 480 | } |
| 481 | |
| 482 | namespace { |
| 483 | struct LCSSAWrapperPass : public FunctionPass { |
| 484 | static char ID; // Pass identification, replacement for typeid |
| 485 | LCSSAWrapperPass() : FunctionPass(ID) { |
| 486 | initializeLCSSAWrapperPassPass(*PassRegistry::getPassRegistry()); |
| 487 | } |
| 488 | |
| 489 | // Cached analysis information for the current function. |
| 490 | DominatorTree *DT; |
| 491 | LoopInfo *LI; |
| 492 | ScalarEvolution *SE; |
| 493 | |
| 494 | bool runOnFunction(Function &F) override; |
| 495 | void verifyAnalysis() const override { |
| 496 | // This check is very expensive. On the loop intensive compiles it may cause |
| 497 | // up to 10x slowdown. Currently it's disabled by default. LPPassManager |
| 498 | // always does limited form of the LCSSA verification. Similar reasoning |
| 499 | // was used for the LoopInfo verifier. |
| 500 | if (VerifyLoopLCSSA) { |
| 501 | assert(all_of(*LI, |
| 502 | [&](Loop *L) { |
| 503 | return L->isRecursivelyLCSSAForm(*DT, *LI); |
| 504 | }) && |
| 505 | "LCSSA form is broken!" ); |
| 506 | } |
| 507 | }; |
| 508 | |
| 509 | /// This transformation requires natural loop information & requires that |
| 510 | /// loop preheaders be inserted into the CFG. It maintains both of these, |
| 511 | /// as well as the CFG. It also requires dominator information. |
| 512 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 513 | AU.setPreservesCFG(); |
| 514 | |
| 515 | AU.addRequired<DominatorTreeWrapperPass>(); |
| 516 | AU.addRequired<LoopInfoWrapperPass>(); |
| 517 | AU.addPreservedID(ID&: LoopSimplifyID); |
| 518 | AU.addPreserved<AAResultsWrapperPass>(); |
| 519 | AU.addPreserved<BasicAAWrapperPass>(); |
| 520 | AU.addPreserved<GlobalsAAWrapperPass>(); |
| 521 | AU.addPreserved<ScalarEvolutionWrapperPass>(); |
| 522 | AU.addPreserved<SCEVAAWrapperPass>(); |
| 523 | AU.addPreserved<BranchProbabilityInfoWrapperPass>(); |
| 524 | AU.addPreserved<MemorySSAWrapperPass>(); |
| 525 | |
| 526 | // This is needed to perform LCSSA verification inside LPPassManager |
| 527 | AU.addRequired<LCSSAVerificationPass>(); |
| 528 | AU.addPreserved<LCSSAVerificationPass>(); |
| 529 | } |
| 530 | }; |
| 531 | } |
| 532 | |
| 533 | char LCSSAWrapperPass::ID = 0; |
| 534 | INITIALIZE_PASS_BEGIN(LCSSAWrapperPass, "lcssa" , "Loop-Closed SSA Form Pass" , |
| 535 | false, false) |
| 536 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) |
| 537 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) |
| 538 | INITIALIZE_PASS_DEPENDENCY(LCSSAVerificationPass) |
| 539 | INITIALIZE_PASS_END(LCSSAWrapperPass, "lcssa" , "Loop-Closed SSA Form Pass" , |
| 540 | false, false) |
| 541 | |
| 542 | Pass *llvm::createLCSSAPass() { return new LCSSAWrapperPass(); } |
| 543 | char &llvm::LCSSAID = LCSSAWrapperPass::ID; |
| 544 | |
| 545 | /// Transform \p F into loop-closed SSA form. |
| 546 | bool LCSSAWrapperPass::runOnFunction(Function &F) { |
| 547 | LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); |
| 548 | DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); |
| 549 | auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>(); |
| 550 | SE = SEWP ? &SEWP->getSE() : nullptr; |
| 551 | |
| 552 | return formLCSSAOnAllLoops(LI, DT: *DT, SE); |
| 553 | } |
| 554 | |
| 555 | PreservedAnalyses LCSSAPass::run(Function &F, FunctionAnalysisManager &AM) { |
| 556 | auto &LI = AM.getResult<LoopAnalysis>(IR&: F); |
| 557 | auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F); |
| 558 | auto *SE = AM.getCachedResult<ScalarEvolutionAnalysis>(IR&: F); |
| 559 | if (!formLCSSAOnAllLoops(LI: &LI, DT, SE)) |
| 560 | return PreservedAnalyses::all(); |
| 561 | |
| 562 | PreservedAnalyses PA; |
| 563 | PA.preserveSet<CFGAnalyses>(); |
| 564 | PA.preserve<ScalarEvolutionAnalysis>(); |
| 565 | // BPI maps terminators to probabilities, since we don't modify the CFG, no |
| 566 | // updates are needed to preserve it. |
| 567 | PA.preserve<BranchProbabilityAnalysis>(); |
| 568 | PA.preserve<MemorySSAAnalysis>(); |
| 569 | return PA; |
| 570 | } |
| 571 | |