1//===- NewGVN.cpp - Global Value Numbering Pass ---------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file implements the new LLVM's Global Value Numbering pass.
11/// GVN partitions values computed by a function into congruence classes.
12/// Values ending up in the same congruence class are guaranteed to be the same
13/// for every execution of the program. In that respect, congruency is a
14/// compile-time approximation of equivalence of values at runtime.
15/// The algorithm implemented here uses a sparse formulation and it's based
16/// on the ideas described in the paper:
17/// "A Sparse Algorithm for Predicated Global Value Numbering" from
18/// Karthik Gargi.
19///
20/// A brief overview of the algorithm: The algorithm is essentially the same as
21/// the standard RPO value numbering algorithm (a good reference is the paper
22/// "SCC based value numbering" by L. Taylor Simpson) with one major difference:
23/// The RPO algorithm proceeds, on every iteration, to process every reachable
24/// block and every instruction in that block. This is because the standard RPO
25/// algorithm does not track what things have the same value number, it only
26/// tracks what the value number of a given operation is (the mapping is
27/// operation -> value number). Thus, when a value number of an operation
28/// changes, it must reprocess everything to ensure all uses of a value number
29/// get updated properly. In constrast, the sparse algorithm we use *also*
30/// tracks what operations have a given value number (IE it also tracks the
31/// reverse mapping from value number -> operations with that value number), so
32/// that it only needs to reprocess the instructions that are affected when
33/// something's value number changes. The vast majority of complexity and code
34/// in this file is devoted to tracking what value numbers could change for what
35/// instructions when various things happen. The rest of the algorithm is
36/// devoted to performing symbolic evaluation, forward propagation, and
37/// simplification of operations based on the value numbers deduced so far
38///
39/// In order to make the GVN mostly-complete, we use a technique derived from
40/// "Detection of Redundant Expressions: A Complete and Polynomial-time
41/// Algorithm in SSA" by R.R. Pai. The source of incompleteness in most SSA
42/// based GVN algorithms is related to their inability to detect equivalence
43/// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)).
44/// We resolve this issue by generating the equivalent "phi of ops" form for
45/// each op of phis we see, in a way that only takes polynomial time to resolve.
46///
47/// We also do not perform elimination by using any published algorithm. All
48/// published algorithms are O(Instructions). Instead, we use a technique that
49/// is O(number of operations with the same value number), enabling us to skip
50/// trying to eliminate things that have unique value numbers.
51//
52//===----------------------------------------------------------------------===//
53
54#include "llvm/Transforms/Scalar/NewGVN.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/BitVector.h"
57#include "llvm/ADT/DenseMap.h"
58#include "llvm/ADT/DenseMapInfo.h"
59#include "llvm/ADT/DenseSet.h"
60#include "llvm/ADT/DepthFirstIterator.h"
61#include "llvm/ADT/GraphTraits.h"
62#include "llvm/ADT/Hashing.h"
63#include "llvm/ADT/PointerIntPair.h"
64#include "llvm/ADT/PostOrderIterator.h"
65#include "llvm/ADT/SetOperations.h"
66#include "llvm/ADT/SmallPtrSet.h"
67#include "llvm/ADT/SmallVector.h"
68#include "llvm/ADT/SparseBitVector.h"
69#include "llvm/ADT/Statistic.h"
70#include "llvm/ADT/iterator_range.h"
71#include "llvm/Analysis/AliasAnalysis.h"
72#include "llvm/Analysis/AssumptionCache.h"
73#include "llvm/Analysis/CFGPrinter.h"
74#include "llvm/Analysis/ConstantFolding.h"
75#include "llvm/Analysis/GlobalsModRef.h"
76#include "llvm/Analysis/InstructionSimplify.h"
77#include "llvm/Analysis/MemoryBuiltins.h"
78#include "llvm/Analysis/MemorySSA.h"
79#include "llvm/Analysis/TargetLibraryInfo.h"
80#include "llvm/Analysis/ValueTracking.h"
81#include "llvm/IR/Argument.h"
82#include "llvm/IR/BasicBlock.h"
83#include "llvm/IR/Constant.h"
84#include "llvm/IR/Constants.h"
85#include "llvm/IR/DebugInfo.h"
86#include "llvm/IR/Dominators.h"
87#include "llvm/IR/Function.h"
88#include "llvm/IR/InstrTypes.h"
89#include "llvm/IR/Instruction.h"
90#include "llvm/IR/Instructions.h"
91#include "llvm/IR/IntrinsicInst.h"
92#include "llvm/IR/PatternMatch.h"
93#include "llvm/IR/Type.h"
94#include "llvm/IR/Use.h"
95#include "llvm/IR/User.h"
96#include "llvm/IR/Value.h"
97#include "llvm/Support/Allocator.h"
98#include "llvm/Support/ArrayRecycler.h"
99#include "llvm/Support/Casting.h"
100#include "llvm/Support/CommandLine.h"
101#include "llvm/Support/Debug.h"
102#include "llvm/Support/DebugCounter.h"
103#include "llvm/Support/ErrorHandling.h"
104#include "llvm/Support/PointerLikeTypeTraits.h"
105#include "llvm/Support/raw_ostream.h"
106#include "llvm/Transforms/Scalar/GVNExpression.h"
107#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
108#include "llvm/Transforms/Utils/Local.h"
109#include "llvm/Transforms/Utils/PredicateInfo.h"
110#include "llvm/Transforms/Utils/VNCoercion.h"
111#include <algorithm>
112#include <cassert>
113#include <cstdint>
114#include <iterator>
115#include <map>
116#include <memory>
117#include <set>
118#include <string>
119#include <tuple>
120#include <utility>
121#include <vector>
122
123using namespace llvm;
124using namespace llvm::GVNExpression;
125using namespace llvm::VNCoercion;
126using namespace llvm::PatternMatch;
127
128#define DEBUG_TYPE "newgvn"
129
130STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
131STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
132STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
133STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
134STATISTIC(NumGVNMaxIterations,
135 "Maximum Number of iterations it took to converge GVN");
136STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
137STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
138STATISTIC(NumGVNAvoidedSortedLeaderChanges,
139 "Number of avoided sorted leader changes");
140STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");
141STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created");
142STATISTIC(NumGVNPHIOfOpsEliminations,
143 "Number of things eliminated using PHI of ops");
144DEBUG_COUNTER(VNCounter, "newgvn-vn",
145 "Controls which instructions are value numbered");
146DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",
147 "Controls which instructions we create phi of ops for");
148// Currently store defining access refinement is too slow due to basicaa being
149// egregiously slow. This flag lets us keep it working while we work on this
150// issue.
151static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
152 cl::init(Val: false), cl::Hidden);
153
154/// Currently, the generation "phi of ops" can result in correctness issues.
155static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(Val: true),
156 cl::Hidden);
157
158//===----------------------------------------------------------------------===//
159// GVN Pass
160//===----------------------------------------------------------------------===//
161
162// Anchor methods.
163Expression::~Expression() = default;
164BasicExpression::~BasicExpression() = default;
165CallExpression::~CallExpression() = default;
166LoadExpression::~LoadExpression() = default;
167StoreExpression::~StoreExpression() = default;
168AggregateValueExpression::~AggregateValueExpression() = default;
169PHIExpression::~PHIExpression() = default;
170
171namespace {
172
173// Tarjan's SCC finding algorithm with Nuutila's improvements
174// SCCIterator is actually fairly complex for the simple thing we want.
175// It also wants to hand us SCC's that are unrelated to the phi node we ask
176// about, and have us process them there or risk redoing work.
177// Graph traits over a filter iterator also doesn't work that well here.
178// This SCC finder is specialized to walk use-def chains, and only follows
179// instructions,
180// not generic values (arguments, etc).
181struct TarjanSCC {
182 TarjanSCC() : Components(1) {}
183
184 void Start(const Instruction *Start) {
185 if (Root.lookup(Val: Start) == 0)
186 FindSCC(I: Start);
187 }
188
189 const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {
190 unsigned ComponentID = ValueToComponent.lookup(Val: V);
191
192 assert(ComponentID > 0 &&
193 "Asking for a component for a value we never processed");
194 return Components[ComponentID];
195 }
196
197private:
198 void FindSCC(const Instruction *I) {
199 Root[I] = ++DFSNum;
200 // Store the DFS Number we had before it possibly gets incremented.
201 unsigned int OurDFS = DFSNum;
202 for (const auto &Op : I->operands()) {
203 if (auto *InstOp = dyn_cast<Instruction>(Val: Op)) {
204 if (Root.lookup(Val: Op) == 0)
205 FindSCC(I: InstOp);
206 if (!InComponent.count(Ptr: Op))
207 Root[I] = std::min(a: Root.lookup(Val: I), b: Root.lookup(Val: Op));
208 }
209 }
210 // See if we really were the root of a component, by seeing if we still have
211 // our DFSNumber. If we do, we are the root of the component, and we have
212 // completed a component. If we do not, we are not the root of a component,
213 // and belong on the component stack.
214 if (Root.lookup(Val: I) == OurDFS) {
215 unsigned ComponentID = Components.size();
216 Components.resize(N: Components.size() + 1);
217 auto &Component = Components.back();
218 Component.insert(Ptr: I);
219 LLVM_DEBUG(dbgs() << "Component root is " << *I << "\n");
220 InComponent.insert(Ptr: I);
221 ValueToComponent[I] = ComponentID;
222 // Pop a component off the stack and label it.
223 while (!Stack.empty() && Root.lookup(Val: Stack.back()) >= OurDFS) {
224 auto *Member = Stack.back();
225 LLVM_DEBUG(dbgs() << "Component member is " << *Member << "\n");
226 Component.insert(Ptr: Member);
227 InComponent.insert(Ptr: Member);
228 ValueToComponent[Member] = ComponentID;
229 Stack.pop_back();
230 }
231 } else {
232 // Part of a component, push to stack
233 Stack.push_back(Elt: I);
234 }
235 }
236
237 unsigned int DFSNum = 1;
238 SmallPtrSet<const Value *, 8> InComponent;
239 DenseMap<const Value *, unsigned int> Root;
240 SmallVector<const Value *, 8> Stack;
241
242 // Store the components as vector of ptr sets, because we need the topo order
243 // of SCC's, but not individual member order
244 SmallVector<SmallPtrSet<const Value *, 8>, 8> Components;
245
246 DenseMap<const Value *, unsigned> ValueToComponent;
247};
248
249// Congruence classes represent the set of expressions/instructions
250// that are all the same *during some scope in the function*.
251// That is, because of the way we perform equality propagation, and
252// because of memory value numbering, it is not correct to assume
253// you can willy-nilly replace any member with any other at any
254// point in the function.
255//
256// For any Value in the Member set, it is valid to replace any dominated member
257// with that Value.
258//
259// Every congruence class has a leader, and the leader is used to symbolize
260// instructions in a canonical way (IE every operand of an instruction that is a
261// member of the same congruence class will always be replaced with leader
262// during symbolization). To simplify symbolization, we keep the leader as a
263// constant if class can be proved to be a constant value. Otherwise, the
264// leader is the member of the value set with the smallest DFS number. Each
265// congruence class also has a defining expression, though the expression may be
266// null. If it exists, it can be used for forward propagation and reassociation
267// of values.
268
269// For memory, we also track a representative MemoryAccess, and a set of memory
270// members for MemoryPhis (which have no real instructions). Note that for
271// memory, it seems tempting to try to split the memory members into a
272// MemoryCongruenceClass or something. Unfortunately, this does not work
273// easily. The value numbering of a given memory expression depends on the
274// leader of the memory congruence class, and the leader of memory congruence
275// class depends on the value numbering of a given memory expression. This
276// leads to wasted propagation, and in some cases, missed optimization. For
277// example: If we had value numbered two stores together before, but now do not,
278// we move them to a new value congruence class. This in turn will move at one
279// of the memorydefs to a new memory congruence class. Which in turn, affects
280// the value numbering of the stores we just value numbered (because the memory
281// congruence class is part of the value number). So while theoretically
282// possible to split them up, it turns out to be *incredibly* complicated to get
283// it to work right, because of the interdependency. While structurally
284// slightly messier, it is algorithmically much simpler and faster to do what we
285// do here, and track them both at once in the same class.
286// Note: The default iterators for this class iterate over values
287class CongruenceClass {
288public:
289 using MemberType = Value;
290 using MemberSet = SmallPtrSet<MemberType *, 4>;
291 using MemoryMemberType = MemoryPhi;
292 using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;
293
294 explicit CongruenceClass(unsigned ID) : ID(ID) {}
295 CongruenceClass(unsigned ID, std::pair<Value *, unsigned int> Leader,
296 const Expression *E)
297 : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
298
299 unsigned getID() const { return ID; }
300
301 // True if this class has no members left. This is mainly used for assertion
302 // purposes, and for skipping empty classes.
303 bool isDead() const {
304 // If it's both dead from a value perspective, and dead from a memory
305 // perspective, it's really dead.
306 return empty() && memory_empty();
307 }
308
309 // Leader functions
310 Value *getLeader() const { return RepLeader.first; }
311 void setLeader(std::pair<Value *, unsigned int> Leader) {
312 RepLeader = std::move(Leader);
313 }
314 const std::pair<Value *, unsigned int> &getNextLeader() const {
315 return NextLeader;
316 }
317 void resetNextLeader() { NextLeader = {nullptr, ~0}; }
318 bool addPossibleLeader(std::pair<Value *, unsigned int> LeaderPair) {
319 if (LeaderPair.second < RepLeader.second) {
320 NextLeader = RepLeader;
321 RepLeader = std::move(LeaderPair);
322 return true;
323 } else if (LeaderPair.second < NextLeader.second) {
324 NextLeader = std::move(LeaderPair);
325 }
326 return false;
327 }
328
329 Value *getStoredValue() const { return RepStoredValue; }
330 void setStoredValue(Value *Leader) { RepStoredValue = Leader; }
331 const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }
332 void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }
333
334 // Forward propagation info
335 const Expression *getDefiningExpr() const { return DefiningExpr; }
336
337 // Value member set
338 bool empty() const { return Members.empty(); }
339 unsigned size() const { return Members.size(); }
340 MemberSet::const_iterator begin() const { return Members.begin(); }
341 MemberSet::const_iterator end() const { return Members.end(); }
342 void insert(MemberType *M) { Members.insert(Ptr: M); }
343 void erase(MemberType *M) { Members.erase(Ptr: M); }
344 void swap(MemberSet &Other) { Members.swap(RHS&: Other); }
345
346 // Memory member set
347 bool memory_empty() const { return MemoryMembers.empty(); }
348 unsigned memory_size() const { return MemoryMembers.size(); }
349 MemoryMemberSet::const_iterator memory_begin() const {
350 return MemoryMembers.begin();
351 }
352 MemoryMemberSet::const_iterator memory_end() const {
353 return MemoryMembers.end();
354 }
355 iterator_range<MemoryMemberSet::const_iterator> memory() const {
356 return make_range(x: memory_begin(), y: memory_end());
357 }
358
359 void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(Ptr: M); }
360 void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(Ptr: M); }
361
362 // Store count
363 unsigned getStoreCount() const { return StoreCount; }
364 void incStoreCount() { ++StoreCount; }
365 void decStoreCount() {
366 assert(StoreCount != 0 && "Store count went negative");
367 --StoreCount;
368 }
369
370 // True if this class has no memory members.
371 bool definesNoMemory() const { return StoreCount == 0 && memory_empty(); }
372
373 // Return true if two congruence classes are equivalent to each other. This
374 // means that every field but the ID number and the dead field are equivalent.
375 bool isEquivalentTo(const CongruenceClass *Other) const {
376 if (!Other)
377 return false;
378 if (this == Other)
379 return true;
380
381 if (std::tie(args: StoreCount, args: RepLeader, args: RepStoredValue, args: RepMemoryAccess) !=
382 std::tie(args: Other->StoreCount, args: Other->RepLeader, args: Other->RepStoredValue,
383 args: Other->RepMemoryAccess))
384 return false;
385 if (DefiningExpr != Other->DefiningExpr)
386 if (!DefiningExpr || !Other->DefiningExpr ||
387 *DefiningExpr != *Other->DefiningExpr)
388 return false;
389
390 if (Members.size() != Other->Members.size())
391 return false;
392
393 return llvm::set_is_subset(S1: Members, S2: Other->Members);
394 }
395
396private:
397 unsigned ID;
398
399 // Representative leader and its corresponding RPO number.
400 // The leader must have the lowest RPO number.
401 std::pair<Value *, unsigned int> RepLeader = {nullptr, ~0U};
402
403 // The most dominating leader after our current leader (given by the RPO
404 // number), because the member set is not sorted and is expensive to keep
405 // sorted all the time.
406 std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
407
408 // If this is represented by a store, the value of the store.
409 Value *RepStoredValue = nullptr;
410
411 // If this class contains MemoryDefs or MemoryPhis, this is the leading memory
412 // access.
413 const MemoryAccess *RepMemoryAccess = nullptr;
414
415 // Defining Expression.
416 const Expression *DefiningExpr = nullptr;
417
418 // Actual members of this class.
419 MemberSet Members;
420
421 // This is the set of MemoryPhis that exist in the class. MemoryDefs and
422 // MemoryUses have real instructions representing them, so we only need to
423 // track MemoryPhis here.
424 MemoryMemberSet MemoryMembers;
425
426 // Number of stores in this congruence class.
427 // This is used so we can detect store equivalence changes properly.
428 int StoreCount = 0;
429};
430
431struct ExactEqualsExpression {
432 const Expression &E;
433
434 explicit ExactEqualsExpression(const Expression &E) : E(E) {}
435
436 hash_code getComputedHash() const { return E.getComputedHash(); }
437
438 bool operator==(const Expression &Other) const {
439 return E.exactlyEquals(Other);
440 }
441};
442} // end anonymous namespace
443
444template <> struct llvm::DenseMapInfo<const Expression *> {
445 static unsigned getHashValue(const Expression *E) {
446 return E->getComputedHash();
447 }
448
449 static unsigned getHashValue(const ExactEqualsExpression &E) {
450 return E.getComputedHash();
451 }
452
453 static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS) {
454 return LHS == *RHS;
455 }
456
457 static bool isEqual(const Expression *LHS, const Expression *RHS) {
458 if (LHS == RHS)
459 return true;
460 // Compare hashes before equality. This is *not* what the hashtable does,
461 // since it is computing it modulo the number of buckets, whereas we are
462 // using the full hash keyspace. Since the hashes are precomputed, this
463 // check is *much* faster than equality.
464 if (LHS->getComputedHash() != RHS->getComputedHash())
465 return false;
466 return *LHS == *RHS;
467 }
468};
469
470namespace {
471
472class NewGVN {
473 Function &F;
474 DominatorTree *DT = nullptr;
475 const TargetLibraryInfo *TLI = nullptr;
476 AliasAnalysis *AA = nullptr;
477 MemorySSA *MSSA = nullptr;
478 MemorySSAWalker *MSSAWalker = nullptr;
479 AssumptionCache *AC = nullptr;
480 const DataLayout &DL;
481
482 // These are the only two things the create* functions should have
483 // side-effects on due to allocating memory.
484 mutable BumpPtrAllocator ExpressionAllocator;
485 mutable ArrayRecycler<Value *> ArgRecycler;
486 mutable TarjanSCC SCCFinder;
487
488 std::unique_ptr<PredicateInfo> PredInfo;
489 const SimplifyQuery SQ;
490
491 // Number of function arguments, used by ranking
492 unsigned int NumFuncArgs = 0;
493
494 // RPOOrdering of basic blocks
495 DenseMap<const DomTreeNode *, unsigned> RPOOrdering;
496
497 // Congruence class info.
498
499 // This class is called INITIAL in the paper. It is the class everything
500 // startsout in, and represents any value. Being an optimistic analysis,
501 // anything in the TOP class has the value TOP, which is indeterminate and
502 // equivalent to everything.
503 CongruenceClass *TOPClass = nullptr;
504 std::vector<CongruenceClass *> CongruenceClasses;
505 unsigned NextCongruenceNum = 0;
506
507 // Value Mappings.
508 DenseMap<Value *, CongruenceClass *> ValueToClass;
509 DenseMap<Value *, const Expression *> ValueToExpression;
510
511 // Value PHI handling, used to make equivalence between phi(op, op) and
512 // op(phi, phi).
513 // These mappings just store various data that would normally be part of the
514 // IR.
515 SmallPtrSet<const Instruction *, 8> PHINodeUses;
516
517 // The cached results, in general, are only valid for the specific block where
518 // they were computed. The unsigned part of the key is a unique block
519 // identifier
520 DenseMap<std::pair<const Value *, unsigned>, bool> OpSafeForPHIOfOps;
521 unsigned CacheIdx;
522
523 // Map a temporary instruction we created to a parent block.
524 DenseMap<const Value *, BasicBlock *> TempToBlock;
525
526 // Map between the already in-program instructions and the temporary phis we
527 // created that they are known equivalent to.
528 DenseMap<const Value *, PHINode *> RealToTemp;
529
530 // In order to know when we should re-process instructions that have
531 // phi-of-ops, we track the set of expressions that they needed as
532 // leaders. When we discover new leaders for those expressions, we process the
533 // associated phi-of-op instructions again in case they have changed. The
534 // other way they may change is if they had leaders, and those leaders
535 // disappear. However, at the point they have leaders, there are uses of the
536 // relevant operands in the created phi node, and so they will get reprocessed
537 // through the normal user marking we perform.
538 mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers;
539 DenseMap<const Expression *, SmallPtrSet<Instruction *, 2>>
540 ExpressionToPhiOfOps;
541
542 // Map from temporary operation to MemoryAccess.
543 DenseMap<const Instruction *, MemoryUseOrDef *> TempToMemory;
544
545 // Set of all temporary instructions we created.
546 // Note: This will include instructions that were just created during value
547 // numbering. The way to test if something is using them is to check
548 // RealToTemp.
549 DenseSet<Instruction *> AllTempInstructions;
550
551 // This is the set of instructions to revisit on a reachability change. At
552 // the end of the main iteration loop it will contain at least all the phi of
553 // ops instructions that will be changed to phis, as well as regular phis.
554 // During the iteration loop, it may contain other things, such as phi of ops
555 // instructions that used edge reachability to reach a result, and so need to
556 // be revisited when the edge changes, independent of whether the phi they
557 // depended on changes.
558 DenseMap<BasicBlock *, SparseBitVector<>> RevisitOnReachabilityChange;
559
560 // Mapping from predicate info we used to the instructions we used it with.
561 // In order to correctly ensure propagation, we must keep track of what
562 // comparisons we used, so that when the values of the comparisons change, we
563 // propagate the information to the places we used the comparison.
564 mutable DenseMap<const Value *, SmallPtrSet<Instruction *, 2>>
565 PredicateToUsers;
566
567 // the same reasoning as PredicateToUsers. When we skip MemoryAccesses for
568 // stores, we no longer can rely solely on the def-use chains of MemorySSA.
569 mutable DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>>
570 MemoryToUsers;
571
572 // A table storing which memorydefs/phis represent a memory state provably
573 // equivalent to another memory state.
574 // We could use the congruence class machinery, but the MemoryAccess's are
575 // abstract memory states, so they can only ever be equivalent to each other,
576 // and not to constants, etc.
577 DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass;
578
579 // We could, if we wanted, build MemoryPhiExpressions and
580 // MemoryVariableExpressions, etc, and value number them the same way we value
581 // number phi expressions. For the moment, this seems like overkill. They
582 // can only exist in one of three states: they can be TOP (equal to
583 // everything), Equivalent to something else, or unique. Because we do not
584 // create expressions for them, we need to simulate leader change not just
585 // when they change class, but when they change state. Note: We can do the
586 // same thing for phis, and avoid having phi expressions if we wanted, We
587 // should eventually unify in one direction or the other, so this is a little
588 // bit of an experiment in which turns out easier to maintain.
589 enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };
590 DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState;
591
592 enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle };
593 mutable DenseMap<const Instruction *, InstCycleState> InstCycleState;
594
595 // Expression to class mapping.
596 using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
597 ExpressionClassMap ExpressionToClass;
598
599 // We have a single expression that represents currently DeadExpressions.
600 // For dead expressions we can prove will stay dead, we mark them with
601 // DFS number zero. However, it's possible in the case of phi nodes
602 // for us to assume/prove all arguments are dead during fixpointing.
603 // We use DeadExpression for that case.
604 DeadExpression *SingletonDeadExpression = nullptr;
605
606 // Which values have changed as a result of leader changes.
607 SmallPtrSet<Value *, 8> LeaderChanges;
608
609 // Reachability info.
610 using BlockEdge = BasicBlockEdge;
611 DenseSet<BlockEdge> ReachableEdges;
612 SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
613
614 // This is a bitvector because, on larger functions, we may have
615 // thousands of touched instructions at once (entire blocks,
616 // instructions with hundreds of uses, etc). Even with optimization
617 // for when we mark whole blocks as touched, when this was a
618 // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
619 // the time in GVN just managing this list. The bitvector, on the
620 // other hand, efficiently supports test/set/clear of both
621 // individual and ranges, as well as "find next element" This
622 // enables us to use it as a worklist with essentially 0 cost.
623 BitVector TouchedInstructions;
624
625 DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
626 mutable DenseMap<const BitCastInst *, const Value *> PredicateSwapChoice;
627
628#ifndef NDEBUG
629 // Debugging for how many times each block and instruction got processed.
630 DenseMap<const Value *, unsigned> ProcessedCount;
631#endif
632
633 // DFS info.
634 // This contains a mapping from Instructions to DFS numbers.
635 // The numbering starts at 1. An instruction with DFS number zero
636 // means that the instruction is dead.
637 DenseMap<const Value *, unsigned> InstrDFS;
638
639 // This contains the mapping DFS numbers to instructions.
640 SmallVector<Value *, 32> DFSToInstr;
641
642 // Deletion info.
643 SmallPtrSet<Instruction *, 8> InstructionsToErase;
644
645public:
646 NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
647 TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
648 const DataLayout &DL)
649 : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), AC(AC), DL(DL),
650 // Reuse ExpressionAllocator for PredicateInfo as well.
651 PredInfo(
652 std::make_unique<PredicateInfo>(args&: F, args&: *DT, args&: *AC, args&: ExpressionAllocator)),
653 SQ(DL, TLI, DT, AC, /*CtxI=*/nullptr, /*UseInstrInfo=*/false,
654 /*CanUseUndef=*/false) {}
655
656 bool runGVN();
657
658private:
659 /// Helper struct return a Expression with an optional extra dependency.
660 struct ExprResult {
661 const Expression *Expr;
662 Value *ExtraDep;
663 const PredicateBase *PredDep;
664
665 ExprResult(const Expression *Expr, Value *ExtraDep = nullptr,
666 const PredicateBase *PredDep = nullptr)
667 : Expr(Expr), ExtraDep(ExtraDep), PredDep(PredDep) {}
668 ExprResult(const ExprResult &) = delete;
669 ExprResult(ExprResult &&Other)
670 : Expr(Other.Expr), ExtraDep(Other.ExtraDep), PredDep(Other.PredDep) {
671 Other.Expr = nullptr;
672 Other.ExtraDep = nullptr;
673 Other.PredDep = nullptr;
674 }
675 ExprResult &operator=(const ExprResult &Other) = delete;
676 ExprResult &operator=(ExprResult &&Other) = delete;
677
678 ~ExprResult() { assert(!ExtraDep && "unhandled ExtraDep"); }
679
680 operator bool() const { return Expr; }
681
682 static ExprResult none() { return {nullptr, nullptr, nullptr}; }
683 static ExprResult some(const Expression *Expr, Value *ExtraDep = nullptr) {
684 return {Expr, ExtraDep, nullptr};
685 }
686 static ExprResult some(const Expression *Expr,
687 const PredicateBase *PredDep) {
688 return {Expr, nullptr, PredDep};
689 }
690 static ExprResult some(const Expression *Expr, Value *ExtraDep,
691 const PredicateBase *PredDep) {
692 return {Expr, ExtraDep, PredDep};
693 }
694 };
695
696 // Expression handling.
697 ExprResult createExpression(Instruction *) const;
698 const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,
699 Instruction *) const;
700
701 // Our canonical form for phi arguments is a pair of incoming value, incoming
702 // basic block.
703 using ValPair = std::pair<Value *, BasicBlock *>;
704
705 PHIExpression *createPHIExpression(ArrayRef<ValPair>, const Instruction *,
706 BasicBlock *, bool &HasBackEdge,
707 bool &OriginalOpsConstant) const;
708 const DeadExpression *createDeadExpression() const;
709 const VariableExpression *createVariableExpression(Value *) const;
710 const ConstantExpression *createConstantExpression(Constant *) const;
711 const Expression *createVariableOrConstant(Value *V) const;
712 const UnknownExpression *createUnknownExpression(Instruction *) const;
713 const StoreExpression *createStoreExpression(StoreInst *,
714 const MemoryAccess *) const;
715 LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
716 const MemoryAccess *) const;
717 const CallExpression *createCallExpression(CallInst *,
718 const MemoryAccess *) const;
719 const AggregateValueExpression *
720 createAggregateValueExpression(Instruction *) const;
721 bool setBasicExpressionInfo(Instruction *, BasicExpression *) const;
722
723 // Congruence class handling.
724 CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
725 // Set RPO to 0 for values that are always available (constants and function
726 // args). These should always be made leader.
727 unsigned LeaderDFS = 0;
728
729 // If Leader is not specified, either we have a memory class or the leader
730 // will be set later. Otherwise, if Leader is an Instruction, set LeaderDFS
731 // to its RPO number.
732 if (!Leader)
733 LeaderDFS = ~0;
734 else if (auto *I = dyn_cast<Instruction>(Val: Leader))
735 LeaderDFS = InstrToDFSNum(V: I);
736 auto *result =
737 new CongruenceClass(NextCongruenceNum++, {Leader, LeaderDFS}, E);
738 CongruenceClasses.emplace_back(args&: result);
739 return result;
740 }
741
742 CongruenceClass *createMemoryClass(MemoryAccess *MA) {
743 auto *CC = createCongruenceClass(Leader: nullptr, E: nullptr);
744 CC->setMemoryLeader(MA);
745 return CC;
746 }
747
748 CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {
749 auto *CC = getMemoryClass(MA);
750 if (CC->getMemoryLeader() != MA)
751 CC = createMemoryClass(MA);
752 return CC;
753 }
754
755 CongruenceClass *createSingletonCongruenceClass(Value *Member) {
756 CongruenceClass *CClass = createCongruenceClass(Leader: Member, E: nullptr);
757 CClass->insert(M: Member);
758 ValueToClass[Member] = CClass;
759 return CClass;
760 }
761
762 void initializeCongruenceClasses(Function &F);
763 const Expression *makePossiblePHIOfOps(Instruction *,
764 SmallPtrSetImpl<Value *> &);
765 Value *findLeaderForInst(Instruction *ValueOp,
766 SmallPtrSetImpl<Value *> &Visited,
767 MemoryAccess *MemAccess, Instruction *OrigInst,
768 BasicBlock *PredBB);
769 bool OpIsSafeForPHIOfOps(Value *Op, const BasicBlock *PHIBlock,
770 SmallPtrSetImpl<const Value *> &);
771 void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue);
772 void removePhiOfOps(Instruction *I, PHINode *PHITemp);
773
774 // Value number an Instruction or MemoryPhi.
775 void valueNumberMemoryPhi(MemoryPhi *);
776 void valueNumberInstruction(Instruction *);
777
778 // Symbolic evaluation.
779 ExprResult checkExprResults(Expression *, Instruction *, Value *) const;
780 ExprResult performSymbolicEvaluation(Instruction *,
781 SmallPtrSetImpl<Value *> &) const;
782 const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
783 Instruction *,
784 MemoryAccess *) const;
785 const Expression *performSymbolicLoadEvaluation(Instruction *) const;
786 const Expression *performSymbolicStoreEvaluation(Instruction *) const;
787 ExprResult performSymbolicCallEvaluation(Instruction *) const;
788 void sortPHIOps(MutableArrayRef<ValPair> Ops) const;
789 const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>,
790 Instruction *I,
791 BasicBlock *PHIBlock) const;
792 const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
793 ExprResult performSymbolicCmpEvaluation(Instruction *) const;
794 ExprResult performSymbolicPredicateInfoEvaluation(BitCastInst *) const;
795
796 // Congruence finding.
797 bool someEquivalentDominates(const Instruction *, const Instruction *) const;
798 Value *lookupOperandLeader(Value *) const;
799 CongruenceClass *getClassForExpression(const Expression *E) const;
800 void performCongruenceFinding(Instruction *, const Expression *);
801 void moveValueToNewCongruenceClass(Instruction *, const Expression *,
802 CongruenceClass *, CongruenceClass *);
803 void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,
804 CongruenceClass *, CongruenceClass *);
805 Value *getNextValueLeader(CongruenceClass *) const;
806 const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;
807 bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);
808 CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;
809 const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;
810 bool isMemoryAccessTOP(const MemoryAccess *) const;
811
812 // Ranking
813 unsigned int getRank(const Value *) const;
814 bool shouldSwapOperands(const Value *, const Value *) const;
815 bool shouldSwapOperandsForPredicate(const Value *, const Value *,
816 const BitCastInst *I) const;
817
818 // Reachability handling.
819 void updateReachableEdge(BasicBlock *, BasicBlock *);
820 void processOutgoingEdges(Instruction *, BasicBlock *);
821 Value *findConditionEquivalence(Value *) const;
822
823 // Elimination.
824 struct ValueDFS;
825 void convertClassToDFSOrdered(const CongruenceClass &,
826 SmallVectorImpl<ValueDFS> &,
827 DenseMap<const Value *, unsigned int> &,
828 SmallPtrSetImpl<Instruction *> &) const;
829 void convertClassToLoadsAndStores(const CongruenceClass &,
830 SmallVectorImpl<ValueDFS> &) const;
831
832 bool eliminateInstructions(Function &);
833 void replaceInstruction(Instruction *, Value *);
834 void markInstructionForDeletion(Instruction *);
835 void deleteInstructionsInBlock(BasicBlock *);
836 Value *findPHIOfOpsLeader(const Expression *, const Instruction *,
837 const BasicBlock *) const;
838
839 // Various instruction touch utilities
840 template <typename Map, typename KeyType>
841 void touchAndErase(Map &, const KeyType &);
842 void markUsersTouched(Value *);
843 void markMemoryUsersTouched(const MemoryAccess *);
844 void markMemoryDefTouched(const MemoryAccess *);
845 void markPredicateUsersTouched(Instruction *);
846 void markValueLeaderChangeTouched(CongruenceClass *CC);
847 void markMemoryLeaderChangeTouched(CongruenceClass *CC);
848 void markPhiOfOpsChanged(const Expression *E);
849 void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;
850 void addAdditionalUsers(Value *To, Value *User) const;
851 void addAdditionalUsers(ExprResult &Res, Instruction *User) const;
852
853 // Main loop of value numbering
854 void iterateTouchedInstructions();
855
856 // Utilities.
857 void cleanupTables();
858 std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
859 void updateProcessedCount(const Value *V);
860 void verifyMemoryCongruency() const;
861 void verifyIterationSettled(Function &F);
862 void verifyStoreExpressions() const;
863 bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &,
864 const MemoryAccess *, const MemoryAccess *) const;
865 BasicBlock *getBlockForValue(Value *V) const;
866 void deleteExpression(const Expression *E) const;
867 MemoryUseOrDef *getMemoryAccess(const Instruction *) const;
868 MemoryPhi *getMemoryAccess(const BasicBlock *) const;
869 template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
870
871 unsigned InstrToDFSNum(const Value *V) const {
872 assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");
873 return InstrDFS.lookup(Val: V);
874 }
875
876 unsigned InstrToDFSNum(const MemoryAccess *MA) const {
877 return MemoryToDFSNum(MA);
878 }
879
880 Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }
881
882 // Given a MemoryAccess, return the relevant instruction DFS number. Note:
883 // This deliberately takes a value so it can be used with Use's, which will
884 // auto-convert to Value's but not to MemoryAccess's.
885 unsigned MemoryToDFSNum(const Value *MA) const {
886 assert(isa<MemoryAccess>(MA) &&
887 "This should not be used with instructions");
888 return isa<MemoryUseOrDef>(Val: MA)
889 ? InstrToDFSNum(V: cast<MemoryUseOrDef>(Val: MA)->getMemoryInst())
890 : InstrDFS.lookup(Val: MA);
891 }
892
893 bool isCycleFree(const Instruction *) const;
894 bool isBackedge(BasicBlock *From, BasicBlock *To) const;
895
896 // Debug counter info. When verifying, we have to reset the value numbering
897 // debug counter to the same state it started in to get the same results.
898 DebugCounter::CounterState StartingVNCounter;
899};
900
901} // end anonymous namespace
902
903template <typename T>
904static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
905 if (!isa<LoadExpression>(Val: RHS) && !isa<StoreExpression>(Val: RHS))
906 return false;
907 return LHS.MemoryExpression::equals(RHS);
908}
909
910bool LoadExpression::equals(const Expression &Other) const {
911 return equalsLoadStoreHelper(LHS: *this, RHS: Other);
912}
913
914bool StoreExpression::equals(const Expression &Other) const {
915 if (!equalsLoadStoreHelper(LHS: *this, RHS: Other))
916 return false;
917 // Make sure that store vs store includes the value operand.
918 if (const auto *S = dyn_cast<StoreExpression>(Val: &Other))
919 if (getStoredValue() != S->getStoredValue())
920 return false;
921 return true;
922}
923
924bool CallExpression::equals(const Expression &Other) const {
925 if (!MemoryExpression::equals(Other))
926 return false;
927
928 if (auto *RHS = dyn_cast<CallExpression>(Val: &Other))
929 return Call->getAttributes()
930 .intersectWith(C&: Call->getContext(), Other: RHS->Call->getAttributes())
931 .has_value();
932
933 return false;
934}
935
936// Determine if the edge From->To is a backedge
937bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const {
938 return From == To ||
939 RPOOrdering.lookup(Val: DT->getNode(BB: From)) >=
940 RPOOrdering.lookup(Val: DT->getNode(BB: To));
941}
942
943#ifndef NDEBUG
944static std::string getBlockName(const BasicBlock *B) {
945 return DOTGraphTraits<DOTFuncInfo *>::getSimpleNodeLabel(B, nullptr);
946}
947#endif
948
949// Get a MemoryAccess for an instruction, fake or real.
950MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const {
951 auto *Result = MSSA->getMemoryAccess(I);
952 return Result ? Result : TempToMemory.lookup(Val: I);
953}
954
955// Get a MemoryPhi for a basic block. These are all real.
956MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const {
957 return MSSA->getMemoryAccess(BB);
958}
959
960// Get the basic block from an instruction/memory value.
961BasicBlock *NewGVN::getBlockForValue(Value *V) const {
962 if (auto *I = dyn_cast<Instruction>(Val: V)) {
963 auto *Parent = I->getParent();
964 if (Parent)
965 return Parent;
966 Parent = TempToBlock.lookup(Val: V);
967 assert(Parent && "Every fake instruction should have a block");
968 return Parent;
969 }
970
971 auto *MP = dyn_cast<MemoryPhi>(Val: V);
972 assert(MP && "Should have been an instruction or a MemoryPhi");
973 return MP->getBlock();
974}
975
976// Delete a definitely dead expression, so it can be reused by the expression
977// allocator. Some of these are not in creation functions, so we have to accept
978// const versions.
979void NewGVN::deleteExpression(const Expression *E) const {
980 assert(isa<BasicExpression>(E));
981 auto *BE = cast<BasicExpression>(Val: E);
982 const_cast<BasicExpression *>(BE)->deallocateOperands(Recycler&: ArgRecycler);
983 ExpressionAllocator.Deallocate(Ptr: E);
984}
985
986// If V is a predicateinfo copy, get the thing it is a copy of.
987static Value *getCopyOf(const Value *V) {
988 if (auto *BC = dyn_cast<BitCastInst>(Val: V))
989 if (BC->getType() == BC->getOperand(i_nocapture: 0)->getType())
990 return BC->getOperand(i_nocapture: 0);
991 return nullptr;
992}
993
994// Return true if V is really PN, even accounting for predicateinfo copies.
995static bool isCopyOfPHI(const Value *V, const PHINode *PN) {
996 return V == PN || getCopyOf(V) == PN;
997}
998
999static bool isCopyOfAPHI(const Value *V) {
1000 auto *CO = getCopyOf(V);
1001 return CO && isa<PHINode>(Val: CO);
1002}
1003
1004// Sort PHI Operands into a canonical order. What we use here is an RPO
1005// order. The BlockInstRange numbers are generated in an RPO walk of the basic
1006// blocks.
1007void NewGVN::sortPHIOps(MutableArrayRef<ValPair> Ops) const {
1008 llvm::sort(C&: Ops, Comp: [&](const ValPair &P1, const ValPair &P2) {
1009 return BlockInstRange.lookup(Val: P1.second).first <
1010 BlockInstRange.lookup(Val: P2.second).first;
1011 });
1012}
1013
1014// Return true if V is a value that will always be available (IE can
1015// be placed anywhere) in the function. We don't do globals here
1016// because they are often worse to put in place.
1017static bool alwaysAvailable(Value *V) {
1018 return isa<Constant>(Val: V) || isa<Argument>(Val: V);
1019}
1020
1021// Create a PHIExpression from an array of {incoming edge, value} pairs. I is
1022// the original instruction we are creating a PHIExpression for (but may not be
1023// a phi node). We require, as an invariant, that all the PHIOperands in the
1024// same block are sorted the same way. sortPHIOps will sort them into a
1025// canonical order.
1026PHIExpression *NewGVN::createPHIExpression(ArrayRef<ValPair> PHIOperands,
1027 const Instruction *I,
1028 BasicBlock *PHIBlock,
1029 bool &HasBackedge,
1030 bool &OriginalOpsConstant) const {
1031 unsigned NumOps = PHIOperands.size();
1032 auto *E = new (ExpressionAllocator) PHIExpression(NumOps, PHIBlock);
1033
1034 E->allocateOperands(Recycler&: ArgRecycler, Allocator&: ExpressionAllocator);
1035 E->setType(PHIOperands.begin()->first->getType());
1036 E->setOpcode(Instruction::PHI);
1037
1038 // Filter out unreachable phi operands.
1039 auto Filtered = make_filter_range(Range&: PHIOperands, Pred: [&](const ValPair &P) {
1040 auto *BB = P.second;
1041 if (auto *PHIOp = dyn_cast<PHINode>(Val: I))
1042 if (isCopyOfPHI(V: P.first, PN: PHIOp))
1043 return false;
1044 if (!ReachableEdges.count(V: {BB, PHIBlock}))
1045 return false;
1046 // Things in TOPClass are equivalent to everything.
1047 if (ValueToClass.lookup(Val: P.first) == TOPClass)
1048 return false;
1049 OriginalOpsConstant = OriginalOpsConstant && isa<Constant>(Val: P.first);
1050 HasBackedge = HasBackedge || isBackedge(From: BB, To: PHIBlock);
1051 return lookupOperandLeader(P.first) != I;
1052 });
1053 llvm::transform(Range&: Filtered, d_first: op_inserter(E), F: [&](const ValPair &P) -> Value * {
1054 return lookupOperandLeader(P.first);
1055 });
1056 return E;
1057}
1058
1059// Set basic expression info (Arguments, type, opcode) for Expression
1060// E from Instruction I in block B.
1061bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const {
1062 bool AllConstant = true;
1063 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I))
1064 E->setType(GEP->getSourceElementType());
1065 else
1066 E->setType(I->getType());
1067 E->setOpcode(I->getOpcode());
1068 E->allocateOperands(Recycler&: ArgRecycler, Allocator&: ExpressionAllocator);
1069
1070 // Transform the operand array into an operand leader array, and keep track of
1071 // whether all members are constant.
1072 std::transform(first: I->op_begin(), last: I->op_end(), result: op_inserter(E), unary_op: [&](Value *O) {
1073 auto Operand = lookupOperandLeader(O);
1074 AllConstant = AllConstant && isa<Constant>(Val: Operand);
1075 return Operand;
1076 });
1077
1078 return AllConstant;
1079}
1080
1081const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
1082 Value *Arg1, Value *Arg2,
1083 Instruction *I) const {
1084 auto *E = new (ExpressionAllocator) BasicExpression(2);
1085 // TODO: we need to remove context instruction after Value Tracking
1086 // can run without context instruction
1087 const SimplifyQuery Q = SQ.getWithInstruction(I);
1088
1089 E->setType(T);
1090 E->setOpcode(Opcode);
1091 E->allocateOperands(Recycler&: ArgRecycler, Allocator&: ExpressionAllocator);
1092 if (Instruction::isCommutative(Opcode)) {
1093 // Ensure that commutative instructions that only differ by a permutation
1094 // of their operands get the same value number by sorting the operand value
1095 // numbers. Since all commutative instructions have two operands it is more
1096 // efficient to sort by hand rather than using, say, std::sort.
1097 if (shouldSwapOperands(Arg1, Arg2))
1098 std::swap(a&: Arg1, b&: Arg2);
1099 }
1100 E->op_push_back(Arg: lookupOperandLeader(Arg1));
1101 E->op_push_back(Arg: lookupOperandLeader(Arg2));
1102
1103 Value *V = simplifyBinOp(Opcode, LHS: E->getOperand(N: 0), RHS: E->getOperand(N: 1), Q);
1104 if (auto Simplified = checkExprResults(E, I, V)) {
1105 addAdditionalUsers(Res&: Simplified, User: I);
1106 return Simplified.Expr;
1107 }
1108 return E;
1109}
1110
1111// Take a Value returned by simplification of Expression E/Instruction
1112// I, and see if it resulted in a simpler expression. If so, return
1113// that expression.
1114NewGVN::ExprResult NewGVN::checkExprResults(Expression *E, Instruction *I,
1115 Value *V) const {
1116 if (!V)
1117 return ExprResult::none();
1118
1119 if (auto *C = dyn_cast<Constant>(Val: V)) {
1120 if (I)
1121 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1122 << " constant " << *C << "\n");
1123 NumGVNOpsSimplified++;
1124 assert(isa<BasicExpression>(E) &&
1125 "We should always have had a basic expression here");
1126 deleteExpression(E);
1127 return ExprResult::some(Expr: createConstantExpression(C));
1128 } else if (isa<Argument>(Val: V) || isa<GlobalVariable>(Val: V)) {
1129 if (I)
1130 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1131 << " variable " << *V << "\n");
1132 deleteExpression(E);
1133 return ExprResult::some(Expr: createVariableExpression(V));
1134 }
1135
1136 CongruenceClass *CC = ValueToClass.lookup(Val: V);
1137 if (CC) {
1138 if (CC->getLeader() && CC->getLeader() != I) {
1139 return ExprResult::some(Expr: createVariableOrConstant(V: CC->getLeader()), ExtraDep: V);
1140 }
1141 if (CC->getDefiningExpr()) {
1142 if (I)
1143 LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1144 << " expression " << *CC->getDefiningExpr() << "\n");
1145 NumGVNOpsSimplified++;
1146 deleteExpression(E);
1147 return ExprResult::some(Expr: CC->getDefiningExpr(), ExtraDep: V);
1148 }
1149 }
1150
1151 return ExprResult::none();
1152}
1153
1154// Create a value expression from the instruction I, replacing operands with
1155// their leaders.
1156
1157NewGVN::ExprResult NewGVN::createExpression(Instruction *I) const {
1158 auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
1159 // TODO: we need to remove context instruction after Value Tracking
1160 // can run without context instruction
1161 const SimplifyQuery Q = SQ.getWithInstruction(I);
1162
1163 bool AllConstant = setBasicExpressionInfo(I, E);
1164
1165 if (I->isCommutative()) {
1166 // Ensure that commutative instructions that only differ by a permutation
1167 // of their operands get the same value number by sorting the operand value
1168 // numbers. Since all commutative instructions have two operands it is more
1169 // efficient to sort by hand rather than using, say, std::sort.
1170 assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
1171 if (shouldSwapOperands(E->getOperand(N: 0), E->getOperand(N: 1)))
1172 E->swapOperands(First: 0, Second: 1);
1173 }
1174 // Perform simplification.
1175 if (auto *CI = dyn_cast<CmpInst>(Val: I)) {
1176 // Sort the operand value numbers so x<y and y>x get the same value
1177 // number.
1178 CmpInst::Predicate Predicate = CI->getPredicate();
1179 if (shouldSwapOperands(E->getOperand(N: 0), E->getOperand(N: 1))) {
1180 E->swapOperands(First: 0, Second: 1);
1181 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
1182 }
1183 E->setOpcode((CI->getOpcode() << 8) | Predicate);
1184 // TODO: 25% of our time is spent in simplifyCmpInst with pointer operands
1185 assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
1186 "Wrong types on cmp instruction");
1187 assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
1188 E->getOperand(1)->getType() == I->getOperand(1)->getType()));
1189 Value *V =
1190 simplifyCmpInst(Predicate, LHS: E->getOperand(N: 0), RHS: E->getOperand(N: 1), Q);
1191 if (auto Simplified = checkExprResults(E, I, V))
1192 return Simplified;
1193 } else if (isa<SelectInst>(Val: I)) {
1194 if (isa<Constant>(Val: E->getOperand(N: 0)) ||
1195 E->getOperand(N: 1) == E->getOperand(N: 2)) {
1196 assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
1197 E->getOperand(2)->getType() == I->getOperand(2)->getType());
1198 Value *V = simplifySelectInst(Cond: E->getOperand(N: 0), TrueVal: E->getOperand(N: 1),
1199 FalseVal: E->getOperand(N: 2), FMF: FastMathFlags(), Q);
1200 if (auto Simplified = checkExprResults(E, I, V))
1201 return Simplified;
1202 }
1203 } else if (I->isBinaryOp()) {
1204 Value *V =
1205 simplifyBinOp(Opcode: E->getOpcode(), LHS: E->getOperand(N: 0), RHS: E->getOperand(N: 1), Q);
1206 if (auto Simplified = checkExprResults(E, I, V))
1207 return Simplified;
1208 } else if (auto *CI = dyn_cast<CastInst>(Val: I)) {
1209 Value *V =
1210 simplifyCastInst(CastOpc: CI->getOpcode(), Op: E->getOperand(N: 0), Ty: CI->getType(), Q);
1211 if (auto Simplified = checkExprResults(E, I, V))
1212 return Simplified;
1213 } else if (auto *GEPI = dyn_cast<GetElementPtrInst>(Val: I)) {
1214 Value *V = simplifyGEPInst(SrcTy: GEPI->getSourceElementType(), Ptr: *E->op_begin(),
1215 Indices: ArrayRef(std::next(x: E->op_begin()), E->op_end()),
1216 NW: GEPI->getNoWrapFlags(), Q);
1217 if (auto Simplified = checkExprResults(E, I, V))
1218 return Simplified;
1219 } else if (AllConstant) {
1220 // We don't bother trying to simplify unless all of the operands
1221 // were constant.
1222 // TODO: There are a lot of Simplify*'s we could call here, if we
1223 // wanted to. The original motivating case for this code was a
1224 // zext i1 false to i8, which we don't have an interface to
1225 // simplify (IE there is no SimplifyZExt).
1226
1227 SmallVector<Constant *, 8> C;
1228 for (Value *Arg : E->operands())
1229 C.emplace_back(Args: cast<Constant>(Val: Arg));
1230
1231 if (Value *V = ConstantFoldInstOperands(I, Ops: C, DL, TLI))
1232 if (auto Simplified = checkExprResults(E, I, V))
1233 return Simplified;
1234 }
1235 return ExprResult::some(Expr: E);
1236}
1237
1238const AggregateValueExpression *
1239NewGVN::createAggregateValueExpression(Instruction *I) const {
1240 if (auto *II = dyn_cast<InsertValueInst>(Val: I)) {
1241 auto *E = new (ExpressionAllocator)
1242 AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
1243 setBasicExpressionInfo(I, E);
1244 E->allocateIntOperands(Allocator&: ExpressionAllocator);
1245 llvm::copy(Range: II->indices(), Out: int_op_inserter(E));
1246 return E;
1247 } else if (auto *EI = dyn_cast<ExtractValueInst>(Val: I)) {
1248 auto *E = new (ExpressionAllocator)
1249 AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
1250 setBasicExpressionInfo(I: EI, E);
1251 E->allocateIntOperands(Allocator&: ExpressionAllocator);
1252 llvm::copy(Range: EI->indices(), Out: int_op_inserter(E));
1253 return E;
1254 }
1255 llvm_unreachable("Unhandled type of aggregate value operation");
1256}
1257
1258const DeadExpression *NewGVN::createDeadExpression() const {
1259 // DeadExpression has no arguments and all DeadExpression's are the same,
1260 // so we only need one of them.
1261 return SingletonDeadExpression;
1262}
1263
1264const VariableExpression *NewGVN::createVariableExpression(Value *V) const {
1265 auto *E = new (ExpressionAllocator) VariableExpression(V);
1266 E->setOpcode(V->getValueID());
1267 return E;
1268}
1269
1270const Expression *NewGVN::createVariableOrConstant(Value *V) const {
1271 if (auto *C = dyn_cast<Constant>(Val: V))
1272 return createConstantExpression(C);
1273 return createVariableExpression(V);
1274}
1275
1276const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const {
1277 auto *E = new (ExpressionAllocator) ConstantExpression(C);
1278 E->setOpcode(C->getValueID());
1279 return E;
1280}
1281
1282const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const {
1283 auto *E = new (ExpressionAllocator) UnknownExpression(I);
1284 E->setOpcode(I->getOpcode());
1285 return E;
1286}
1287
1288const CallExpression *
1289NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const {
1290 // FIXME: Add operand bundles for calls.
1291 auto *E =
1292 new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);
1293 setBasicExpressionInfo(I: CI, E);
1294 if (CI->isCommutative()) {
1295 // Ensure that commutative intrinsics that only differ by a permutation
1296 // of their operands get the same value number by sorting the operand value
1297 // numbers.
1298 assert(CI->getNumOperands() >= 2 && "Unsupported commutative intrinsic!");
1299 if (shouldSwapOperands(E->getOperand(N: 0), E->getOperand(N: 1)))
1300 E->swapOperands(First: 0, Second: 1);
1301 }
1302 return E;
1303}
1304
1305// Return true if some equivalent of instruction Inst dominates instruction U.
1306bool NewGVN::someEquivalentDominates(const Instruction *Inst,
1307 const Instruction *U) const {
1308 auto *CC = ValueToClass.lookup(Val: Inst);
1309 // This must be an instruction because we are only called from phi nodes
1310 // in the case that the value it needs to check against is an instruction.
1311
1312 // The most likely candidates for dominance are the leader and the next leader.
1313 // The leader or nextleader will dominate in all cases where there is an
1314 // equivalent that is higher up in the dom tree.
1315 // We can't *only* check them, however, because the
1316 // dominator tree could have an infinite number of non-dominating siblings
1317 // with instructions that are in the right congruence class.
1318 // A
1319 // B C D E F G
1320 // |
1321 // H
1322 // Instruction U could be in H, with equivalents in every other sibling.
1323 // Depending on the rpo order picked, the leader could be the equivalent in
1324 // any of these siblings.
1325 if (!CC)
1326 return false;
1327 if (alwaysAvailable(V: CC->getLeader()))
1328 return true;
1329 if (DT->dominates(Def: cast<Instruction>(Val: CC->getLeader()), User: U))
1330 return true;
1331 if (CC->getNextLeader().first &&
1332 DT->dominates(Def: cast<Instruction>(Val: CC->getNextLeader().first), User: U))
1333 return true;
1334 return llvm::any_of(Range&: *CC, P: [&](const Value *Member) {
1335 return Member != CC->getLeader() &&
1336 DT->dominates(Def: cast<Instruction>(Val: Member), User: U);
1337 });
1338}
1339
1340// See if we have a congruence class and leader for this operand, and if so,
1341// return it. Otherwise, return the operand itself.
1342Value *NewGVN::lookupOperandLeader(Value *V) const {
1343 CongruenceClass *CC = ValueToClass.lookup(Val: V);
1344 if (CC) {
1345 // Everything in TOP is represented by poison, as it can be any value.
1346 // We do have to make sure we get the type right though, so we can't set the
1347 // RepLeader to poison.
1348 if (CC == TOPClass)
1349 return PoisonValue::get(T: V->getType());
1350 return CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
1351 }
1352
1353 return V;
1354}
1355
1356const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {
1357 auto *CC = getMemoryClass(MA);
1358 assert(CC->getMemoryLeader() &&
1359 "Every MemoryAccess should be mapped to a congruence class with a "
1360 "representative memory access");
1361 return CC->getMemoryLeader();
1362}
1363
1364// Return true if the MemoryAccess is really equivalent to everything. This is
1365// equivalent to the lattice value "TOP" in most lattices. This is the initial
1366// state of all MemoryAccesses.
1367bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const {
1368 return getMemoryClass(MA) == TOPClass;
1369}
1370
1371LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
1372 LoadInst *LI,
1373 const MemoryAccess *MA) const {
1374 auto *E =
1375 new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));
1376 E->allocateOperands(Recycler&: ArgRecycler, Allocator&: ExpressionAllocator);
1377 E->setType(LoadType);
1378
1379 // Give store and loads same opcode so they value number together.
1380 E->setOpcode(0);
1381 E->op_push_back(Arg: PointerOp);
1382
1383 // TODO: Value number heap versions. We may be able to discover
1384 // things alias analysis can't on it's own (IE that a store and a
1385 // load have the same value, and thus, it isn't clobbering the load).
1386 return E;
1387}
1388
1389const StoreExpression *
1390NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const {
1391 auto *StoredValueLeader = lookupOperandLeader(V: SI->getValueOperand());
1392 auto *E = new (ExpressionAllocator)
1393 StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);
1394 E->allocateOperands(Recycler&: ArgRecycler, Allocator&: ExpressionAllocator);
1395 E->setType(SI->getValueOperand()->getType());
1396
1397 // Give store and loads same opcode so they value number together.
1398 E->setOpcode(0);
1399 E->op_push_back(Arg: lookupOperandLeader(V: SI->getPointerOperand()));
1400
1401 // TODO: Value number heap versions. We may be able to discover
1402 // things alias analysis can't on it's own (IE that a store and a
1403 // load have the same value, and thus, it isn't clobbering the load).
1404 return E;
1405}
1406
1407const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {
1408 // Unlike loads, we never try to eliminate stores, so we do not check if they
1409 // are simple and avoid value numbering them.
1410 auto *SI = cast<StoreInst>(Val: I);
1411 auto *StoreAccess = getMemoryAccess(I: SI);
1412 // Get the expression, if any, for the RHS of the MemoryDef.
1413 const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();
1414 if (EnableStoreRefinement)
1415 StoreRHS = MSSAWalker->getClobberingMemoryAccess(MA: StoreAccess);
1416 // If we bypassed the use-def chains, make sure we add a use.
1417 StoreRHS = lookupMemoryLeader(MA: StoreRHS);
1418 if (StoreRHS != StoreAccess->getDefiningAccess())
1419 addMemoryUsers(To: StoreRHS, U: StoreAccess);
1420 // If we are defined by ourselves, use the live on entry def.
1421 if (StoreRHS == StoreAccess)
1422 StoreRHS = MSSA->getLiveOnEntryDef();
1423
1424 if (SI->isSimple()) {
1425 // See if we are defined by a previous store expression, it already has a
1426 // value, and it's the same value as our current store. FIXME: Right now, we
1427 // only do this for simple stores, we should expand to cover memcpys, etc.
1428 const auto *LastStore = createStoreExpression(SI, MA: StoreRHS);
1429 const auto *LastCC = ExpressionToClass.lookup(Val: LastStore);
1430 // We really want to check whether the expression we matched was a store. No
1431 // easy way to do that. However, we can check that the class we found has a
1432 // store, which, assuming the value numbering state is not corrupt, is
1433 // sufficient, because we must also be equivalent to that store's expression
1434 // for it to be in the same class as the load.
1435 if (LastCC && LastCC->getStoredValue() == LastStore->getStoredValue())
1436 return LastStore;
1437 // Also check if our value operand is defined by a load of the same memory
1438 // location, and the memory state is the same as it was then (otherwise, it
1439 // could have been overwritten later. See test32 in
1440 // transforms/DeadStoreElimination/simple.ll).
1441 if (auto *LI = dyn_cast<LoadInst>(Val: LastStore->getStoredValue()))
1442 if ((lookupOperandLeader(V: LI->getPointerOperand()) ==
1443 LastStore->getOperand(N: 0)) &&
1444 (lookupMemoryLeader(MA: getMemoryAccess(I: LI)->getDefiningAccess()) ==
1445 StoreRHS))
1446 return LastStore;
1447 deleteExpression(E: LastStore);
1448 }
1449
1450 // If the store is not equivalent to anything, value number it as a store that
1451 // produces a unique memory state (instead of using it's MemoryUse, we use
1452 // it's MemoryDef).
1453 return createStoreExpression(SI, MA: StoreAccess);
1454}
1455
1456// See if we can extract the value of a loaded pointer from a load, a store, or
1457// a memory instruction.
1458const Expression *
1459NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
1460 LoadInst *LI, Instruction *DepInst,
1461 MemoryAccess *DefiningAccess) const {
1462 assert((!LI || LI->isSimple()) && "Not a simple load");
1463 if (auto *DepSI = dyn_cast<StoreInst>(Val: DepInst)) {
1464 // Can't forward from non-atomic to atomic without violating memory model.
1465 // Also don't need to coerce if they are the same type, we will just
1466 // propagate.
1467 if (LI->isAtomic() > DepSI->isAtomic() ||
1468 LoadType == DepSI->getValueOperand()->getType())
1469 return nullptr;
1470 int Offset = analyzeLoadFromClobberingStore(LoadTy: LoadType, LoadPtr, DepSI, DL);
1471 if (Offset >= 0) {
1472 if (auto *C = dyn_cast<Constant>(
1473 Val: lookupOperandLeader(V: DepSI->getValueOperand()))) {
1474 if (Constant *Res = getConstantValueForLoad(SrcVal: C, Offset, LoadTy: LoadType, DL)) {
1475 LLVM_DEBUG(dbgs() << "Coercing load from store " << *DepSI
1476 << " to constant " << *Res << "\n");
1477 return createConstantExpression(C: Res);
1478 }
1479 }
1480 }
1481 } else if (auto *DepLI = dyn_cast<LoadInst>(Val: DepInst)) {
1482 // Can't forward from non-atomic to atomic without violating memory model.
1483 if (LI->isAtomic() > DepLI->isAtomic())
1484 return nullptr;
1485 int Offset = analyzeLoadFromClobberingLoad(LoadTy: LoadType, LoadPtr, DepLI, DL);
1486 if (Offset >= 0) {
1487 // We can coerce a constant load into a load.
1488 if (auto *C = dyn_cast<Constant>(Val: lookupOperandLeader(V: DepLI)))
1489 if (auto *PossibleConstant =
1490 getConstantValueForLoad(SrcVal: C, Offset, LoadTy: LoadType, DL)) {
1491 LLVM_DEBUG(dbgs() << "Coercing load from load " << *LI
1492 << " to constant " << *PossibleConstant << "\n");
1493 return createConstantExpression(C: PossibleConstant);
1494 }
1495 }
1496 } else if (auto *DepMI = dyn_cast<MemIntrinsic>(Val: DepInst)) {
1497 int Offset = analyzeLoadFromClobberingMemInst(LoadTy: LoadType, LoadPtr, DepMI, DL);
1498 if (Offset >= 0) {
1499 if (auto *PossibleConstant =
1500 getConstantMemInstValueForLoad(SrcInst: DepMI, Offset, LoadTy: LoadType, DL)) {
1501 LLVM_DEBUG(dbgs() << "Coercing load from meminst " << *DepMI
1502 << " to constant " << *PossibleConstant << "\n");
1503 return createConstantExpression(C: PossibleConstant);
1504 }
1505 }
1506 }
1507
1508 if (auto *II = dyn_cast<IntrinsicInst>(Val: DepInst)) {
1509 if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
1510 auto *LifetimePtr = II->getOperand(i_nocapture: 0);
1511 if (LoadPtr == lookupOperandLeader(V: LifetimePtr) ||
1512 AA->isMustAlias(V1: LoadPtr, V2: LifetimePtr))
1513 return createConstantExpression(C: UndefValue::get(T: LoadType));
1514 }
1515 }
1516
1517 // All of the below are only true if the loaded pointer is produced
1518 // by the dependent instruction.
1519 if (!DepInst->getType()->isPointerTy() ||
1520 (LoadPtr != lookupOperandLeader(V: DepInst) &&
1521 !AA->isMustAlias(V1: LoadPtr, V2: DepInst)))
1522 return nullptr;
1523 // If this load really doesn't depend on anything, then we must be loading an
1524 // undef value. This can happen when loading for a fresh allocation with no
1525 // intervening stores, for example. Note that this is only true in the case
1526 // that the result of the allocation is pointer equal to the load ptr.
1527 if (isa<AllocaInst>(Val: DepInst)) {
1528 return createConstantExpression(C: UndefValue::get(T: LoadType));
1529 } else if (auto *InitVal =
1530 getInitialValueOfAllocation(V: DepInst, TLI, Ty: LoadType))
1531 return createConstantExpression(C: InitVal);
1532
1533 return nullptr;
1534}
1535
1536const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
1537 auto *LI = cast<LoadInst>(Val: I);
1538
1539 // We can eliminate in favor of non-simple loads, but we won't be able to
1540 // eliminate the loads themselves.
1541 if (!LI->isSimple())
1542 return nullptr;
1543
1544 Value *LoadAddressLeader = lookupOperandLeader(V: LI->getPointerOperand());
1545 // Load of undef is UB.
1546 if (isa<UndefValue>(Val: LoadAddressLeader))
1547 return createConstantExpression(C: PoisonValue::get(T: LI->getType()));
1548 MemoryAccess *OriginalAccess = getMemoryAccess(I);
1549 MemoryAccess *DefiningAccess =
1550 MSSAWalker->getClobberingMemoryAccess(MA: OriginalAccess);
1551
1552 if (!MSSA->isLiveOnEntryDef(MA: DefiningAccess)) {
1553 if (auto *MD = dyn_cast<MemoryDef>(Val: DefiningAccess)) {
1554 Instruction *DefiningInst = MD->getMemoryInst();
1555 // If the defining instruction is not reachable, replace with poison.
1556 if (!ReachableBlocks.count(Ptr: DefiningInst->getParent()))
1557 return createConstantExpression(C: PoisonValue::get(T: LI->getType()));
1558 // This will handle stores and memory insts. We only do if it the
1559 // defining access has a different type, or it is a pointer produced by
1560 // certain memory operations that cause the memory to have a fixed value
1561 // (IE things like calloc).
1562 if (const auto *CoercionResult =
1563 performSymbolicLoadCoercion(LoadType: LI->getType(), LoadPtr: LoadAddressLeader, LI,
1564 DepInst: DefiningInst, DefiningAccess))
1565 return CoercionResult;
1566 }
1567 }
1568
1569 const auto *LE = createLoadExpression(LoadType: LI->getType(), PointerOp: LoadAddressLeader, LI,
1570 MA: DefiningAccess);
1571 // If our MemoryLeader is not our defining access, add a use to the
1572 // MemoryLeader, so that we get reprocessed when it changes.
1573 if (LE->getMemoryLeader() != DefiningAccess)
1574 addMemoryUsers(To: LE->getMemoryLeader(), U: OriginalAccess);
1575 return LE;
1576}
1577
1578NewGVN::ExprResult
1579NewGVN::performSymbolicPredicateInfoEvaluation(BitCastInst *I) const {
1580 auto *PI = PredInfo->getPredicateInfoFor(V: I);
1581 if (!PI)
1582 return ExprResult::none();
1583
1584 LLVM_DEBUG(dbgs() << "Found predicate info from instruction !\n");
1585
1586 const std::optional<PredicateConstraint> &Constraint = PI->getConstraint();
1587 if (!Constraint)
1588 return ExprResult::none();
1589
1590 CmpInst::Predicate Predicate = Constraint->Predicate;
1591 Value *CmpOp0 = I->getOperand(i_nocapture: 0);
1592 Value *CmpOp1 = Constraint->OtherOp;
1593
1594 Value *FirstOp = lookupOperandLeader(V: CmpOp0);
1595 Value *SecondOp = lookupOperandLeader(V: CmpOp1);
1596 Value *AdditionallyUsedValue = CmpOp0;
1597
1598 // Sort the ops.
1599 if (shouldSwapOperandsForPredicate(FirstOp, SecondOp, I)) {
1600 std::swap(a&: FirstOp, b&: SecondOp);
1601 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
1602 AdditionallyUsedValue = CmpOp1;
1603 }
1604
1605 if (Predicate == CmpInst::ICMP_EQ)
1606 return ExprResult::some(Expr: createVariableOrConstant(V: FirstOp),
1607 ExtraDep: AdditionallyUsedValue, PredDep: PI);
1608
1609 // Handle the special case of floating point.
1610 if (Predicate == CmpInst::FCMP_OEQ && isa<ConstantFP>(Val: FirstOp) &&
1611 !cast<ConstantFP>(Val: FirstOp)->isZero())
1612 return ExprResult::some(Expr: createConstantExpression(C: cast<Constant>(Val: FirstOp)),
1613 ExtraDep: AdditionallyUsedValue, PredDep: PI);
1614
1615 return ExprResult::none();
1616}
1617
1618// Evaluate read only and pure calls, and create an expression result.
1619NewGVN::ExprResult NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
1620 auto *CI = cast<CallInst>(Val: I);
1621
1622 // FIXME: Currently the calls which may access the thread id may
1623 // be considered as not accessing the memory. But this is
1624 // problematic for coroutines, since coroutines may resume in a
1625 // different thread. So we disable the optimization here for the
1626 // correctness. However, it may block many other correct
1627 // optimizations. Revert this one when we detect the memory
1628 // accessing kind more precisely.
1629 if (CI->getFunction()->isPresplitCoroutine())
1630 return ExprResult::none();
1631
1632 // Do not combine convergent calls since they implicitly depend on the set of
1633 // threads that is currently executing, and they might be in different basic
1634 // blocks.
1635 if (CI->isConvergent())
1636 return ExprResult::none();
1637
1638 if (AA->doesNotAccessMemory(Call: CI)) {
1639 return ExprResult::some(
1640 Expr: createCallExpression(CI, MA: TOPClass->getMemoryLeader()));
1641 } else if (AA->onlyReadsMemory(Call: CI)) {
1642 if (auto *MA = MSSA->getMemoryAccess(I: CI)) {
1643 auto *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(MA);
1644 return ExprResult::some(Expr: createCallExpression(CI, MA: DefiningAccess));
1645 } else // MSSA determined that CI does not access memory.
1646 return ExprResult::some(
1647 Expr: createCallExpression(CI, MA: TOPClass->getMemoryLeader()));
1648 }
1649 return ExprResult::none();
1650}
1651
1652// Retrieve the memory class for a given MemoryAccess.
1653CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {
1654 auto *Result = MemoryAccessToClass.lookup(Val: MA);
1655 assert(Result && "Should have found memory class");
1656 return Result;
1657}
1658
1659// Update the MemoryAccess equivalence table to say that From is equal to To,
1660// and return true if this is different from what already existed in the table.
1661bool NewGVN::setMemoryClass(const MemoryAccess *From,
1662 CongruenceClass *NewClass) {
1663 assert(NewClass &&
1664 "Every MemoryAccess should be getting mapped to a non-null class");
1665 LLVM_DEBUG(dbgs() << "Setting " << *From);
1666 LLVM_DEBUG(dbgs() << " equivalent to congruence class ");
1667 LLVM_DEBUG(dbgs() << NewClass->getID()
1668 << " with current MemoryAccess leader ");
1669 LLVM_DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n");
1670
1671 auto LookupResult = MemoryAccessToClass.find(Val: From);
1672 bool Changed = false;
1673 // If it's already in the table, see if the value changed.
1674 if (LookupResult != MemoryAccessToClass.end()) {
1675 auto *OldClass = LookupResult->second;
1676 if (OldClass != NewClass) {
1677 // If this is a phi, we have to handle memory member updates.
1678 if (auto *MP = dyn_cast<MemoryPhi>(Val: From)) {
1679 OldClass->memory_erase(M: MP);
1680 NewClass->memory_insert(M: MP);
1681 // This may have killed the class if it had no non-memory members
1682 if (OldClass->getMemoryLeader() == From) {
1683 if (OldClass->definesNoMemory()) {
1684 OldClass->setMemoryLeader(nullptr);
1685 } else {
1686 OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1687 LLVM_DEBUG(dbgs() << "Memory class leader change for class "
1688 << OldClass->getID() << " to "
1689 << *OldClass->getMemoryLeader()
1690 << " due to removal of a memory member " << *From
1691 << "\n");
1692 markMemoryLeaderChangeTouched(CC: OldClass);
1693 }
1694 }
1695 }
1696 // It wasn't equivalent before, and now it is.
1697 LookupResult->second = NewClass;
1698 Changed = true;
1699 }
1700 }
1701
1702 return Changed;
1703}
1704
1705// Determine if a instruction is cycle-free. That means the values in the
1706// instruction don't depend on any expressions that can change value as a result
1707// of the instruction. For example, a non-cycle free instruction would be v =
1708// phi(0, v+1).
1709bool NewGVN::isCycleFree(const Instruction *I) const {
1710 // In order to compute cycle-freeness, we do SCC finding on the instruction,
1711 // and see what kind of SCC it ends up in. If it is a singleton, it is
1712 // cycle-free. If it is not in a singleton, it is only cycle free if the
1713 // other members are all phi nodes (as they do not compute anything, they are
1714 // copies).
1715 auto ICS = InstCycleState.lookup(Val: I);
1716 if (ICS == ICS_Unknown) {
1717 SCCFinder.Start(Start: I);
1718 auto &SCC = SCCFinder.getComponentFor(V: I);
1719 // It's cycle free if it's size 1 or the SCC is *only* phi nodes.
1720 if (SCC.size() == 1)
1721 InstCycleState.insert(KV: {I, ICS_CycleFree});
1722 else {
1723 bool AllPhis = llvm::all_of(Range: SCC, P: [](const Value *V) {
1724 return isa<PHINode>(Val: V) || isCopyOfAPHI(V);
1725 });
1726 ICS = AllPhis ? ICS_CycleFree : ICS_Cycle;
1727 for (const auto *Member : SCC)
1728 if (auto *MemberPhi = dyn_cast<PHINode>(Val: Member))
1729 InstCycleState.insert(KV: {MemberPhi, ICS});
1730 }
1731 }
1732 if (ICS == ICS_Cycle)
1733 return false;
1734 return true;
1735}
1736
1737// Evaluate PHI nodes symbolically and create an expression result.
1738const Expression *
1739NewGVN::performSymbolicPHIEvaluation(ArrayRef<ValPair> PHIOps,
1740 Instruction *I,
1741 BasicBlock *PHIBlock) const {
1742 // True if one of the incoming phi edges is a backedge.
1743 bool HasBackedge = false;
1744 // All constant tracks the state of whether all the *original* phi operands
1745 // This is really shorthand for "this phi cannot cycle due to forward
1746 // change in value of the phi is guaranteed not to later change the value of
1747 // the phi. IE it can't be v = phi(undef, v+1)
1748 bool OriginalOpsConstant = true;
1749 auto *E = cast<PHIExpression>(Val: createPHIExpression(
1750 PHIOperands: PHIOps, I, PHIBlock, HasBackedge, OriginalOpsConstant));
1751 // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
1752 // See if all arguments are the same.
1753 // We track if any were undef because they need special handling.
1754 bool HasUndef = false, HasPoison = false;
1755 auto Filtered = make_filter_range(Range: E->operands(), Pred: [&](Value *Arg) {
1756 if (isa<PoisonValue>(Val: Arg)) {
1757 HasPoison = true;
1758 return false;
1759 }
1760 if (isa<UndefValue>(Val: Arg)) {
1761 HasUndef = true;
1762 return false;
1763 }
1764 return true;
1765 });
1766 // If we are left with no operands, it's dead.
1767 if (Filtered.empty()) {
1768 // If it has undef or poison at this point, it means there are no-non-undef
1769 // arguments, and thus, the value of the phi node must be undef.
1770 if (HasUndef) {
1771 LLVM_DEBUG(
1772 dbgs() << "PHI Node " << *I
1773 << " has no non-undef arguments, valuing it as undef\n");
1774 return createConstantExpression(C: UndefValue::get(T: I->getType()));
1775 }
1776 if (HasPoison) {
1777 LLVM_DEBUG(
1778 dbgs() << "PHI Node " << *I
1779 << " has no non-poison arguments, valuing it as poison\n");
1780 return createConstantExpression(C: PoisonValue::get(T: I->getType()));
1781 }
1782
1783 LLVM_DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n");
1784 deleteExpression(E);
1785 return createDeadExpression();
1786 }
1787 Value *AllSameValue = *(Filtered.begin());
1788 ++Filtered.begin();
1789 // Can't use std::equal here, sadly, because filter.begin moves.
1790 if (llvm::all_of(Range&: Filtered, P: equal_to(Arg&: AllSameValue))) {
1791 // Can't fold phi(undef, X) -> X unless X can't be poison (thus X is undef
1792 // in the worst case).
1793 if (HasUndef && !isGuaranteedNotToBePoison(V: AllSameValue, AC, CtxI: nullptr, DT))
1794 return E;
1795
1796 // In LLVM's non-standard representation of phi nodes, it's possible to have
1797 // phi nodes with cycles (IE dependent on other phis that are .... dependent
1798 // on the original phi node), especially in weird CFG's where some arguments
1799 // are unreachable, or uninitialized along certain paths. This can cause
1800 // infinite loops during evaluation. We work around this by not trying to
1801 // really evaluate them independently, but instead using a variable
1802 // expression to say if one is equivalent to the other.
1803 // We also special case undef/poison, so that if we have an undef, we can't
1804 // use the common value unless it dominates the phi block.
1805 if (HasPoison || HasUndef) {
1806 // If we have undef and at least one other value, this is really a
1807 // multivalued phi, and we need to know if it's cycle free in order to
1808 // evaluate whether we can ignore the undef. The other parts of this are
1809 // just shortcuts. If there is no backedge, or all operands are
1810 // constants, it also must be cycle free.
1811 if (HasBackedge && !OriginalOpsConstant &&
1812 !isa<UndefValue>(Val: AllSameValue) && !isCycleFree(I))
1813 return E;
1814
1815 // Only have to check for instructions
1816 if (auto *AllSameInst = dyn_cast<Instruction>(Val: AllSameValue))
1817 if (!someEquivalentDominates(Inst: AllSameInst, U: I))
1818 return E;
1819 }
1820 // Can't simplify to something that comes later in the iteration.
1821 // Otherwise, when and if it changes congruence class, we will never catch
1822 // up. We will always be a class behind it.
1823 if (isa<Instruction>(Val: AllSameValue) &&
1824 InstrToDFSNum(V: AllSameValue) > InstrToDFSNum(V: I))
1825 return E;
1826 NumGVNPhisAllSame++;
1827 LLVM_DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
1828 << "\n");
1829 deleteExpression(E);
1830 return createVariableOrConstant(V: AllSameValue);
1831 }
1832 return E;
1833}
1834
1835const Expression *
1836NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {
1837 if (auto *EI = dyn_cast<ExtractValueInst>(Val: I)) {
1838 auto *WO = dyn_cast<WithOverflowInst>(Val: EI->getAggregateOperand());
1839 if (WO && EI->getNumIndices() == 1 && *EI->idx_begin() == 0)
1840 // EI is an extract from one of our with.overflow intrinsics. Synthesize
1841 // a semantically equivalent expression instead of an extract value
1842 // expression.
1843 return createBinaryExpression(Opcode: WO->getBinaryOp(), T: EI->getType(),
1844 Arg1: WO->getLHS(), Arg2: WO->getRHS(), I);
1845 }
1846
1847 return createAggregateValueExpression(I);
1848}
1849
1850NewGVN::ExprResult NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
1851 assert(isa<CmpInst>(I) && "Expected a cmp instruction.");
1852
1853 auto *CI = cast<CmpInst>(Val: I);
1854 // See if our operands are equal to those of a previous predicate, and if so,
1855 // if it implies true or false.
1856 auto Op0 = lookupOperandLeader(V: CI->getOperand(i_nocapture: 0));
1857 auto Op1 = lookupOperandLeader(V: CI->getOperand(i_nocapture: 1));
1858 auto OurPredicate = CI->getPredicate();
1859 if (shouldSwapOperands(Op0, Op1)) {
1860 std::swap(a&: Op0, b&: Op1);
1861 OurPredicate = CI->getSwappedPredicate();
1862 }
1863
1864 // Avoid processing the same info twice.
1865 const PredicateBase *LastPredInfo = nullptr;
1866 // See if we know something about the comparison itself, like it is the target
1867 // of an assume.
1868 auto *CmpPI = PredInfo->getPredicateInfoFor(V: I);
1869 if (isa_and_nonnull<PredicateAssume>(Val: CmpPI))
1870 return ExprResult::some(
1871 Expr: createConstantExpression(C: ConstantInt::getTrue(Ty: CI->getType())));
1872
1873 if (Op0 == Op1) {
1874 // This condition does not depend on predicates, no need to add users
1875 if (CI->isTrueWhenEqual())
1876 return ExprResult::some(
1877 Expr: createConstantExpression(C: ConstantInt::getTrue(Ty: CI->getType())));
1878 else if (CI->isFalseWhenEqual())
1879 return ExprResult::some(
1880 Expr: createConstantExpression(C: ConstantInt::getFalse(Ty: CI->getType())));
1881 }
1882
1883 // NOTE: Because we are comparing both operands here and below, and using
1884 // previous comparisons, we rely on fact that predicateinfo knows to mark
1885 // comparisons that use renamed operands as users of the earlier comparisons.
1886 // It is *not* enough to just mark predicateinfo renamed operands as users of
1887 // the earlier comparisons, because the *other* operand may have changed in a
1888 // previous iteration.
1889 // Example:
1890 // icmp slt %a, %b
1891 // %b.0 = ssa.copy(%b)
1892 // false branch:
1893 // icmp slt %c, %b.0
1894
1895 // %c and %a may start out equal, and thus, the code below will say the second
1896 // %icmp is false. c may become equal to something else, and in that case the
1897 // %second icmp *must* be reexamined, but would not if only the renamed
1898 // %operands are considered users of the icmp.
1899
1900 // *Currently* we only check one level of comparisons back, and only mark one
1901 // level back as touched when changes happen. If you modify this code to look
1902 // back farther through comparisons, you *must* mark the appropriate
1903 // comparisons as users in PredicateInfo.cpp, or you will cause bugs. See if
1904 // we know something just from the operands themselves
1905
1906 // See if our operands have predicate info, so that we may be able to derive
1907 // something from a previous comparison.
1908 for (const auto &Op : CI->operands()) {
1909 auto *PI = PredInfo->getPredicateInfoFor(V: Op);
1910 if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(Val: PI)) {
1911 if (PI == LastPredInfo)
1912 continue;
1913 LastPredInfo = PI;
1914 // In phi of ops cases, we may have predicate info that we are evaluating
1915 // in a different context.
1916 if (!DT->dominates(A: PBranch->To, B: I->getParent()))
1917 continue;
1918 // TODO: Along the false edge, we may know more things too, like
1919 // icmp of
1920 // same operands is false.
1921 // TODO: We only handle actual comparison conditions below, not
1922 // and/or.
1923 auto *BranchCond = dyn_cast<CmpInst>(Val: PBranch->Condition);
1924 if (!BranchCond)
1925 continue;
1926 auto *BranchOp0 = lookupOperandLeader(V: BranchCond->getOperand(i_nocapture: 0));
1927 auto *BranchOp1 = lookupOperandLeader(V: BranchCond->getOperand(i_nocapture: 1));
1928 auto BranchPredicate = BranchCond->getPredicate();
1929 if (shouldSwapOperands(BranchOp0, BranchOp1)) {
1930 std::swap(a&: BranchOp0, b&: BranchOp1);
1931 BranchPredicate = BranchCond->getSwappedPredicate();
1932 }
1933 if (BranchOp0 == Op0 && BranchOp1 == Op1) {
1934 if (PBranch->TrueEdge) {
1935 // If we know the previous predicate is true and we are in the true
1936 // edge then we may be implied true or false.
1937 if (auto R = ICmpInst::isImpliedByMatchingCmp(Pred1: BranchPredicate,
1938 Pred2: OurPredicate)) {
1939 auto *C = ConstantInt::getBool(Ty: CI->getType(), V: *R);
1940 return ExprResult::some(Expr: createConstantExpression(C), PredDep: PI);
1941 }
1942 } else {
1943 // Just handle the ne and eq cases, where if we have the same
1944 // operands, we may know something.
1945 if (BranchPredicate == OurPredicate) {
1946 // Same predicate, same ops,we know it was false, so this is false.
1947 return ExprResult::some(
1948 Expr: createConstantExpression(C: ConstantInt::getFalse(Ty: CI->getType())),
1949 PredDep: PI);
1950 } else if (BranchPredicate ==
1951 CmpInst::getInversePredicate(pred: OurPredicate)) {
1952 // Inverse predicate, we know the other was false, so this is true.
1953 return ExprResult::some(
1954 Expr: createConstantExpression(C: ConstantInt::getTrue(Ty: CI->getType())),
1955 PredDep: PI);
1956 }
1957 }
1958 }
1959 }
1960 }
1961 // Create expression will take care of simplifyCmpInst
1962 return createExpression(I);
1963}
1964
1965// Substitute and symbolize the instruction before value numbering.
1966NewGVN::ExprResult
1967NewGVN::performSymbolicEvaluation(Instruction *I,
1968 SmallPtrSetImpl<Value *> &Visited) const {
1969
1970 const Expression *E = nullptr;
1971 // TODO: memory intrinsics.
1972 // TODO: Some day, we should do the forward propagation and reassociation
1973 // parts of the algorithm.
1974 switch (I->getOpcode()) {
1975 case Instruction::ExtractValue:
1976 case Instruction::InsertValue:
1977 E = performSymbolicAggrValueEvaluation(I);
1978 break;
1979 case Instruction::PHI: {
1980 SmallVector<ValPair, 3> Ops;
1981 auto *PN = cast<PHINode>(Val: I);
1982 for (unsigned i = 0; i < PN->getNumOperands(); ++i)
1983 Ops.push_back(Elt: {PN->getIncomingValue(i), PN->getIncomingBlock(i)});
1984 // Sort to ensure the invariant createPHIExpression requires is met.
1985 sortPHIOps(Ops);
1986 E = performSymbolicPHIEvaluation(PHIOps: Ops, I, PHIBlock: getBlockForValue(V: I));
1987 } break;
1988 case Instruction::Call:
1989 return performSymbolicCallEvaluation(I);
1990 break;
1991 case Instruction::Store:
1992 E = performSymbolicStoreEvaluation(I);
1993 break;
1994 case Instruction::Load:
1995 E = performSymbolicLoadEvaluation(I);
1996 break;
1997 case Instruction::BitCast:
1998 // Intrinsics with the returned attribute are copies of arguments.
1999 if (I->getType() == I->getOperand(i: 0)->getType())
2000 if (auto Res =
2001 performSymbolicPredicateInfoEvaluation(I: cast<BitCastInst>(Val: I)))
2002 return Res;
2003 [[fallthrough]];
2004 case Instruction::AddrSpaceCast:
2005 case Instruction::Freeze:
2006 return createExpression(I);
2007 break;
2008 case Instruction::ICmp:
2009 case Instruction::FCmp:
2010 return performSymbolicCmpEvaluation(I);
2011 break;
2012 case Instruction::FNeg:
2013 case Instruction::Add:
2014 case Instruction::FAdd:
2015 case Instruction::Sub:
2016 case Instruction::FSub:
2017 case Instruction::Mul:
2018 case Instruction::FMul:
2019 case Instruction::UDiv:
2020 case Instruction::SDiv:
2021 case Instruction::FDiv:
2022 case Instruction::URem:
2023 case Instruction::SRem:
2024 case Instruction::FRem:
2025 case Instruction::Shl:
2026 case Instruction::LShr:
2027 case Instruction::AShr:
2028 case Instruction::And:
2029 case Instruction::Or:
2030 case Instruction::Xor:
2031 case Instruction::Trunc:
2032 case Instruction::ZExt:
2033 case Instruction::SExt:
2034 case Instruction::FPToUI:
2035 case Instruction::FPToSI:
2036 case Instruction::UIToFP:
2037 case Instruction::SIToFP:
2038 case Instruction::FPTrunc:
2039 case Instruction::FPExt:
2040 case Instruction::PtrToInt:
2041 case Instruction::PtrToAddr:
2042 case Instruction::IntToPtr:
2043 case Instruction::Select:
2044 case Instruction::ExtractElement:
2045 case Instruction::InsertElement:
2046 case Instruction::GetElementPtr:
2047 return createExpression(I);
2048 break;
2049 case Instruction::ShuffleVector:
2050 // FIXME: Add support for shufflevector to createExpression.
2051 return ExprResult::none();
2052 default:
2053 return ExprResult::none();
2054 }
2055 return ExprResult::some(Expr: E);
2056}
2057
2058// Look up a container of values/instructions in a map, and touch all the
2059// instructions in the container. Then erase value from the map.
2060template <typename Map, typename KeyType>
2061void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2062 const auto Result = M.find_as(Key);
2063 if (Result != M.end()) {
2064 for (const typename Map::mapped_type::value_type Mapped : Result->second)
2065 TouchedInstructions.set(InstrToDFSNum(Mapped));
2066 M.erase(Result);
2067 }
2068}
2069
2070void NewGVN::addAdditionalUsers(Value *To, Value *User) const {
2071 assert(User && To != User);
2072 if (isa<Instruction>(Val: To))
2073 AdditionalUsers[To].insert(Ptr: User);
2074}
2075
2076void NewGVN::addAdditionalUsers(ExprResult &Res, Instruction *User) const {
2077 if (Res.ExtraDep && Res.ExtraDep != User)
2078 addAdditionalUsers(To: Res.ExtraDep, User);
2079 Res.ExtraDep = nullptr;
2080
2081 if (Res.PredDep) {
2082 if (const auto *PBranch = dyn_cast<PredicateBranch>(Val: Res.PredDep))
2083 PredicateToUsers[PBranch->Condition].insert(Ptr: User);
2084 else if (const auto *PAssume =
2085 dyn_cast<PredicateConditionAssume>(Val: Res.PredDep))
2086 PredicateToUsers[PAssume->Condition].insert(Ptr: User);
2087 }
2088 Res.PredDep = nullptr;
2089}
2090
2091void NewGVN::markUsersTouched(Value *V) {
2092 // Now mark the users as touched.
2093 for (auto *User : V->users()) {
2094 assert(isa<Instruction>(User) && "Use of value not within an instruction?");
2095 TouchedInstructions.set(InstrToDFSNum(V: User));
2096 }
2097 touchAndErase(M&: AdditionalUsers, Key: V);
2098}
2099
2100void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const {
2101 LLVM_DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");
2102 MemoryToUsers[To].insert(Ptr: U);
2103}
2104
2105void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {
2106 TouchedInstructions.set(MemoryToDFSNum(MA));
2107}
2108
2109void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
2110 if (isa<MemoryUse>(Val: MA))
2111 return;
2112 for (const auto *U : MA->users())
2113 TouchedInstructions.set(MemoryToDFSNum(MA: U));
2114 touchAndErase(M&: MemoryToUsers, Key: MA);
2115}
2116
2117// Touch all the predicates that depend on this instruction.
2118void NewGVN::markPredicateUsersTouched(Instruction *I) {
2119 touchAndErase(M&: PredicateToUsers, Key: I);
2120}
2121
2122// Mark users affected by a memory leader change.
2123void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {
2124 for (const auto *M : CC->memory())
2125 markMemoryDefTouched(MA: M);
2126}
2127
2128// Touch the instructions that need to be updated after a congruence class has a
2129// leader change, and mark changed values.
2130void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {
2131 for (auto *M : *CC) {
2132 if (auto *I = dyn_cast<Instruction>(Val: M))
2133 TouchedInstructions.set(InstrToDFSNum(V: I));
2134 LeaderChanges.insert(Ptr: M);
2135 }
2136}
2137
2138// Give a range of things that have instruction DFS numbers, this will return
2139// the member of the range with the smallest dfs number.
2140template <class T, class Range>
2141T *NewGVN::getMinDFSOfRange(const Range &R) const {
2142 std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2143 for (const auto X : R) {
2144 auto DFSNum = InstrToDFSNum(X);
2145 if (DFSNum < MinDFS.second)
2146 MinDFS = {X, DFSNum};
2147 }
2148 return MinDFS.first;
2149}
2150
2151// This function returns the MemoryAccess that should be the next leader of
2152// congruence class CC, under the assumption that the current leader is going to
2153// disappear.
2154const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {
2155 // TODO: If this ends up to slow, we can maintain a next memory leader like we
2156 // do for regular leaders.
2157 // Make sure there will be a leader to find.
2158 assert(!CC->definesNoMemory() && "Can't get next leader if there is none");
2159 if (CC->getStoreCount() > 0) {
2160 if (auto *NL = dyn_cast_or_null<StoreInst>(Val: CC->getNextLeader().first))
2161 return getMemoryAccess(I: NL);
2162 // Find the store with the minimum DFS number.
2163 auto *V = getMinDFSOfRange<Value>(R: make_filter_range(
2164 Range&: *CC, Pred: [&](const Value *V) { return isa<StoreInst>(Val: V); }));
2165 return getMemoryAccess(I: cast<StoreInst>(Val: V));
2166 }
2167 assert(CC->getStoreCount() == 0);
2168
2169 // Given our assertion, hitting this part must mean
2170 // !OldClass->memory_empty()
2171 if (CC->memory_size() == 1)
2172 return *CC->memory_begin();
2173 return getMinDFSOfRange<const MemoryPhi>(R: CC->memory());
2174}
2175
2176// This function returns the next value leader of a congruence class, under the
2177// assumption that the current leader is going away. This should end up being
2178// the next most dominating member.
2179Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {
2180 // We don't need to sort members if there is only 1, and we don't care about
2181 // sorting the TOP class because everything either gets out of it or is
2182 // unreachable.
2183
2184 if (CC->size() == 1 || CC == TOPClass) {
2185 return *(CC->begin());
2186 } else if (CC->getNextLeader().first) {
2187 ++NumGVNAvoidedSortedLeaderChanges;
2188 return CC->getNextLeader().first;
2189 } else {
2190 ++NumGVNSortedLeaderChanges;
2191 // NOTE: If this ends up to slow, we can maintain a dual structure for
2192 // member testing/insertion, or keep things mostly sorted, and sort only
2193 // here, or use SparseBitVector or ....
2194 return getMinDFSOfRange<Value>(R: *CC);
2195 }
2196}
2197
2198// Move a MemoryAccess, currently in OldClass, to NewClass, including updates to
2199// the memory members, etc for the move.
2200//
2201// The invariants of this function are:
2202//
2203// - I must be moving to NewClass from OldClass
2204// - The StoreCount of OldClass and NewClass is expected to have been updated
2205// for I already if it is a store.
2206// - The OldClass memory leader has not been updated yet if I was the leader.
2207void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,
2208 MemoryAccess *InstMA,
2209 CongruenceClass *OldClass,
2210 CongruenceClass *NewClass) {
2211 // If the leader is I, and we had a representative MemoryAccess, it should
2212 // be the MemoryAccess of OldClass.
2213 assert((!InstMA || !OldClass->getMemoryLeader() ||
2214 OldClass->getLeader() != I ||
2215 MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) ==
2216 MemoryAccessToClass.lookup(InstMA)) &&
2217 "Representative MemoryAccess mismatch");
2218 // First, see what happens to the new class
2219 if (!NewClass->getMemoryLeader()) {
2220 // Should be a new class, or a store becoming a leader of a new class.
2221 assert(NewClass->size() == 1 ||
2222 (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));
2223 NewClass->setMemoryLeader(InstMA);
2224 // Mark it touched if we didn't just create a singleton
2225 LLVM_DEBUG(dbgs() << "Memory class leader change for class "
2226 << NewClass->getID()
2227 << " due to new memory instruction becoming leader\n");
2228 markMemoryLeaderChangeTouched(CC: NewClass);
2229 }
2230 setMemoryClass(From: InstMA, NewClass);
2231 // Now, fixup the old class if necessary
2232 if (OldClass->getMemoryLeader() == InstMA) {
2233 if (!OldClass->definesNoMemory()) {
2234 OldClass->setMemoryLeader(getNextMemoryLeader(CC: OldClass));
2235 LLVM_DEBUG(dbgs() << "Memory class leader change for class "
2236 << OldClass->getID() << " to "
2237 << *OldClass->getMemoryLeader()
2238 << " due to removal of old leader " << *InstMA << "\n");
2239 markMemoryLeaderChangeTouched(CC: OldClass);
2240 } else
2241 OldClass->setMemoryLeader(nullptr);
2242 }
2243}
2244
2245// Move a value, currently in OldClass, to be part of NewClass
2246// Update OldClass and NewClass for the move (including changing leaders, etc).
2247void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,
2248 CongruenceClass *OldClass,
2249 CongruenceClass *NewClass) {
2250 if (I == OldClass->getNextLeader().first)
2251 OldClass->resetNextLeader();
2252
2253 OldClass->erase(M: I);
2254 NewClass->insert(M: I);
2255
2256 // Ensure that the leader has the lowest RPO. If the leader changed notify all
2257 // members of the class.
2258 if (NewClass->getLeader() != I &&
2259 NewClass->addPossibleLeader(LeaderPair: {I, InstrToDFSNum(V: I)})) {
2260 markValueLeaderChangeTouched(CC: NewClass);
2261 }
2262
2263 // Handle our special casing of stores.
2264 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
2265 OldClass->decStoreCount();
2266 // Okay, so when do we want to make a store a leader of a class?
2267 // If we have a store defined by an earlier load, we want the earlier load
2268 // to lead the class.
2269 // If we have a store defined by something else, we want the store to lead
2270 // the class so everything else gets the "something else" as a value.
2271 // If we have a store as the single member of the class, we want the store
2272 // as the leader
2273 if (NewClass->getStoreCount() == 0 && !NewClass->getStoredValue()) {
2274 // If it's a store expression we are using, it means we are not equivalent
2275 // to something earlier.
2276 if (auto *SE = dyn_cast<StoreExpression>(Val: E)) {
2277 NewClass->setStoredValue(SE->getStoredValue());
2278 markValueLeaderChangeTouched(CC: NewClass);
2279 // Shift the new class leader to be the store
2280 LLVM_DEBUG(dbgs() << "Changing leader of congruence class "
2281 << NewClass->getID() << " from "
2282 << *NewClass->getLeader() << " to " << *SI
2283 << " because store joined class\n");
2284 // If we changed the leader, we have to mark it changed because we don't
2285 // know what it will do to symbolic evaluation.
2286 NewClass->setLeader({SI, InstrToDFSNum(V: SI)});
2287 }
2288 // We rely on the code below handling the MemoryAccess change.
2289 }
2290 NewClass->incStoreCount();
2291 }
2292 // True if there is no memory instructions left in a class that had memory
2293 // instructions before.
2294
2295 // If it's not a memory use, set the MemoryAccess equivalence
2296 auto *InstMA = dyn_cast_or_null<MemoryDef>(Val: getMemoryAccess(I));
2297 if (InstMA)
2298 moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);
2299 ValueToClass[I] = NewClass;
2300 // See if we destroyed the class or need to swap leaders.
2301 if (OldClass->empty() && OldClass != TOPClass) {
2302 if (OldClass->getDefiningExpr()) {
2303 LLVM_DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()
2304 << " from table\n");
2305 // We erase it as an exact expression to make sure we don't just erase an
2306 // equivalent one.
2307 auto Iter = ExpressionToClass.find_as(
2308 Val: ExactEqualsExpression(*OldClass->getDefiningExpr()));
2309 if (Iter != ExpressionToClass.end())
2310 ExpressionToClass.erase(I: Iter);
2311#ifdef EXPENSIVE_CHECKS
2312 assert(
2313 (*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) &&
2314 "We erased the expression we just inserted, which should not happen");
2315#endif
2316 }
2317 } else if (OldClass->getLeader() == I) {
2318 // When the leader changes, the value numbering of
2319 // everything may change due to symbolization changes, so we need to
2320 // reprocess.
2321 LLVM_DEBUG(dbgs() << "Value class leader change for class "
2322 << OldClass->getID() << "\n");
2323 ++NumGVNLeaderChanges;
2324 // Destroy the stored value if there are no more stores to represent it.
2325 // Note that this is basically clean up for the expression removal that
2326 // happens below. If we remove stores from a class, we may leave it as a
2327 // class of equivalent memory phis.
2328 if (OldClass->getStoreCount() == 0) {
2329 if (OldClass->getStoredValue())
2330 OldClass->setStoredValue(nullptr);
2331 }
2332 OldClass->setLeader({getNextValueLeader(CC: OldClass),
2333 InstrToDFSNum(V: getNextValueLeader(CC: OldClass))});
2334 OldClass->resetNextLeader();
2335 markValueLeaderChangeTouched(CC: OldClass);
2336 }
2337}
2338
2339// For a given expression, mark the phi of ops instructions that could have
2340// changed as a result.
2341void NewGVN::markPhiOfOpsChanged(const Expression *E) {
2342 touchAndErase(M&: ExpressionToPhiOfOps, Key: E);
2343}
2344
2345// Perform congruence finding on a given value numbering expression.
2346void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
2347 // This is guaranteed to return something, since it will at least find
2348 // TOP.
2349
2350 CongruenceClass *IClass = ValueToClass.lookup(Val: I);
2351 assert(IClass && "Should have found a IClass");
2352 // Dead classes should have been eliminated from the mapping.
2353 assert(!IClass->isDead() && "Found a dead class");
2354
2355 CongruenceClass *EClass = nullptr;
2356 if (const auto *VE = dyn_cast<VariableExpression>(Val: E)) {
2357 EClass = ValueToClass.lookup(Val: VE->getVariableValue());
2358 } else if (isa<DeadExpression>(Val: E)) {
2359 EClass = TOPClass;
2360 }
2361 if (!EClass) {
2362 auto lookupResult = ExpressionToClass.try_emplace(Key: E);
2363
2364 // If it's not in the value table, create a new congruence class.
2365 if (lookupResult.second) {
2366 CongruenceClass *NewClass = createCongruenceClass(Leader: nullptr, E);
2367 auto place = lookupResult.first;
2368 place->second = NewClass;
2369
2370 // Constants and variables should always be made the leader.
2371 if (const auto *CE = dyn_cast<ConstantExpression>(Val: E)) {
2372 NewClass->setLeader({CE->getConstantValue(), 0});
2373 } else if (const auto *SE = dyn_cast<StoreExpression>(Val: E)) {
2374 StoreInst *SI = SE->getStoreInst();
2375 NewClass->setLeader({SI, InstrToDFSNum(V: SI)});
2376 NewClass->setStoredValue(SE->getStoredValue());
2377 // The RepMemoryAccess field will be filled in properly by the
2378 // moveValueToNewCongruenceClass call.
2379 } else {
2380 NewClass->setLeader({I, InstrToDFSNum(V: I)});
2381 }
2382 assert(!isa<VariableExpression>(E) &&
2383 "VariableExpression should have been handled already");
2384
2385 EClass = NewClass;
2386 LLVM_DEBUG(dbgs() << "Created new congruence class for " << *I
2387 << " using expression " << *E << " at "
2388 << NewClass->getID() << " and leader "
2389 << *(NewClass->getLeader()));
2390 if (NewClass->getStoredValue())
2391 LLVM_DEBUG(dbgs() << " and stored value "
2392 << *(NewClass->getStoredValue()));
2393 LLVM_DEBUG(dbgs() << "\n");
2394 } else {
2395 EClass = lookupResult.first->second;
2396 if (isa<ConstantExpression>(Val: E))
2397 assert((isa<Constant>(EClass->getLeader()) ||
2398 (EClass->getStoredValue() &&
2399 isa<Constant>(EClass->getStoredValue()))) &&
2400 "Any class with a constant expression should have a "
2401 "constant leader");
2402
2403 assert(EClass && "Somehow don't have an eclass");
2404
2405 assert(!EClass->isDead() && "We accidentally looked up a dead class");
2406 }
2407 }
2408 bool ClassChanged = IClass != EClass;
2409 bool LeaderChanged = LeaderChanges.erase(Ptr: I);
2410 if (ClassChanged || LeaderChanged) {
2411 LLVM_DEBUG(dbgs() << "New class " << EClass->getID() << " for expression "
2412 << *E << "\n");
2413 if (ClassChanged) {
2414 moveValueToNewCongruenceClass(I, E, OldClass: IClass, NewClass: EClass);
2415 markPhiOfOpsChanged(E);
2416 }
2417
2418 markUsersTouched(V: I);
2419 if (MemoryAccess *MA = getMemoryAccess(I))
2420 markMemoryUsersTouched(MA);
2421 if (auto *CI = dyn_cast<CmpInst>(Val: I))
2422 markPredicateUsersTouched(I: CI);
2423 }
2424 // If we changed the class of the store, we want to ensure nothing finds the
2425 // old store expression. In particular, loads do not compare against stored
2426 // value, so they will find old store expressions (and associated class
2427 // mappings) if we leave them in the table.
2428 if (ClassChanged && isa<StoreInst>(Val: I)) {
2429 auto *OldE = ValueToExpression.lookup(Val: I);
2430 // It could just be that the old class died. We don't want to erase it if we
2431 // just moved classes.
2432 if (OldE && isa<StoreExpression>(Val: OldE) && *E != *OldE) {
2433 // Erase this as an exact expression to ensure we don't erase expressions
2434 // equivalent to it.
2435 auto Iter = ExpressionToClass.find_as(Val: ExactEqualsExpression(*OldE));
2436 if (Iter != ExpressionToClass.end())
2437 ExpressionToClass.erase(I: Iter);
2438 }
2439 }
2440 ValueToExpression[I] = E;
2441}
2442
2443// Process the fact that Edge (from, to) is reachable, including marking
2444// any newly reachable blocks and instructions for processing.
2445void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
2446 // Check if the Edge was reachable before.
2447 if (ReachableEdges.insert(V: {From, To}).second) {
2448 // If this block wasn't reachable before, all instructions are touched.
2449 if (ReachableBlocks.insert(Ptr: To).second) {
2450 LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)
2451 << " marked reachable\n");
2452 const auto &InstRange = BlockInstRange.lookup(Val: To);
2453 TouchedInstructions.set(I: InstRange.first, E: InstRange.second);
2454 } else {
2455 LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)
2456 << " was reachable, but new edge {"
2457 << getBlockName(From) << "," << getBlockName(To)
2458 << "} to it found\n");
2459
2460 // We've made an edge reachable to an existing block, which may
2461 // impact predicates. Otherwise, only mark the phi nodes as touched, as
2462 // they are the only thing that depend on new edges. Anything using their
2463 // values will get propagated to if necessary.
2464 if (MemoryAccess *MemPhi = getMemoryAccess(BB: To))
2465 TouchedInstructions.set(InstrToDFSNum(MA: MemPhi));
2466
2467 // FIXME: We should just add a union op on a Bitvector and
2468 // SparseBitVector. We can do it word by word faster than we are doing it
2469 // here.
2470 for (auto InstNum : RevisitOnReachabilityChange[To])
2471 TouchedInstructions.set(InstNum);
2472 }
2473 }
2474}
2475
2476// Given a predicate condition (from a switch, cmp, or whatever) and a block,
2477// see if we know some constant value for it already.
2478Value *NewGVN::findConditionEquivalence(Value *Cond) const {
2479 auto Result = lookupOperandLeader(V: Cond);
2480 return isa<Constant>(Val: Result) ? Result : nullptr;
2481}
2482
2483// Process the outgoing edges of a block for reachability.
2484void NewGVN::processOutgoingEdges(Instruction *TI, BasicBlock *B) {
2485 // Evaluate reachability of terminator instruction.
2486 Value *Cond;
2487 BasicBlock *TrueSucc, *FalseSucc;
2488 if (match(V: TI, P: m_Br(C: m_Value(V&: Cond), T&: TrueSucc, F&: FalseSucc))) {
2489 Value *CondEvaluated = findConditionEquivalence(Cond);
2490 if (!CondEvaluated) {
2491 if (auto *I = dyn_cast<Instruction>(Val: Cond)) {
2492 SmallPtrSet<Value *, 4> Visited;
2493 auto Res = performSymbolicEvaluation(I, Visited);
2494 if (const auto *CE = dyn_cast_or_null<ConstantExpression>(Val: Res.Expr)) {
2495 CondEvaluated = CE->getConstantValue();
2496 addAdditionalUsers(Res, User: I);
2497 } else {
2498 // Did not use simplification result, no need to add the extra
2499 // dependency.
2500 Res.ExtraDep = nullptr;
2501 }
2502 } else if (isa<ConstantInt>(Val: Cond)) {
2503 CondEvaluated = Cond;
2504 }
2505 }
2506 ConstantInt *CI;
2507 if (CondEvaluated && (CI = dyn_cast<ConstantInt>(Val: CondEvaluated))) {
2508 if (CI->isOne()) {
2509 LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI
2510 << " evaluated to true\n");
2511 updateReachableEdge(From: B, To: TrueSucc);
2512 } else if (CI->isZero()) {
2513 LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI
2514 << " evaluated to false\n");
2515 updateReachableEdge(From: B, To: FalseSucc);
2516 }
2517 } else {
2518 updateReachableEdge(From: B, To: TrueSucc);
2519 updateReachableEdge(From: B, To: FalseSucc);
2520 }
2521 } else if (auto *SI = dyn_cast<SwitchInst>(Val: TI)) {
2522 // For switches, propagate the case values into the case
2523 // destinations.
2524
2525 Value *SwitchCond = SI->getCondition();
2526 Value *CondEvaluated = findConditionEquivalence(Cond: SwitchCond);
2527 // See if we were able to turn this switch statement into a constant.
2528 if (CondEvaluated && isa<ConstantInt>(Val: CondEvaluated)) {
2529 auto *CondVal = cast<ConstantInt>(Val: CondEvaluated);
2530 // We should be able to get case value for this.
2531 auto Case = *SI->findCaseValue(C: CondVal);
2532 if (Case.getCaseSuccessor() == SI->getDefaultDest()) {
2533 // We proved the value is outside of the range of the case.
2534 // We can't do anything other than mark the default dest as reachable,
2535 // and go home.
2536 updateReachableEdge(From: B, To: SI->getDefaultDest());
2537 return;
2538 }
2539 // Now get where it goes and mark it reachable.
2540 BasicBlock *TargetBlock = Case.getCaseSuccessor();
2541 updateReachableEdge(From: B, To: TargetBlock);
2542 } else {
2543 for (BasicBlock *TargetBlock : successors(BB: SI->getParent()))
2544 updateReachableEdge(From: B, To: TargetBlock);
2545 }
2546 } else {
2547 // Otherwise this is either unconditional, or a type we have no
2548 // idea about. Just mark successors as reachable.
2549 for (BasicBlock *TargetBlock : successors(BB: TI->getParent()))
2550 updateReachableEdge(From: B, To: TargetBlock);
2551
2552 // This also may be a memory defining terminator, in which case, set it
2553 // equivalent only to itself.
2554 //
2555 auto *MA = getMemoryAccess(I: TI);
2556 if (MA && !isa<MemoryUse>(Val: MA)) {
2557 auto *CC = ensureLeaderOfMemoryClass(MA);
2558 if (setMemoryClass(From: MA, NewClass: CC))
2559 markMemoryUsersTouched(MA);
2560 }
2561 }
2562}
2563
2564// Remove the PHI of Ops PHI for I
2565void NewGVN::removePhiOfOps(Instruction *I, PHINode *PHITemp) {
2566 InstrDFS.erase(Val: PHITemp);
2567 // It's still a temp instruction. We keep it in the array so it gets erased.
2568 // However, it's no longer used by I, or in the block
2569 TempToBlock.erase(Val: PHITemp);
2570 RealToTemp.erase(Val: I);
2571 // We don't remove the users from the phi node uses. This wastes a little
2572 // time, but such is life. We could use two sets to track which were there
2573 // are the start of NewGVN, and which were added, but right nowt he cost of
2574 // tracking is more than the cost of checking for more phi of ops.
2575}
2576
2577// Add PHI Op in BB as a PHI of operations version of ExistingValue.
2578void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB,
2579 Instruction *ExistingValue) {
2580 InstrDFS[Op] = InstrToDFSNum(V: ExistingValue);
2581 AllTempInstructions.insert(V: Op);
2582 TempToBlock[Op] = BB;
2583 RealToTemp[ExistingValue] = Op;
2584 // Add all users to phi node use, as they are now uses of the phi of ops phis
2585 // and may themselves be phi of ops.
2586 for (auto *U : ExistingValue->users())
2587 if (auto *UI = dyn_cast<Instruction>(Val: U))
2588 PHINodeUses.insert(Ptr: UI);
2589}
2590
2591static bool okayForPHIOfOps(const Instruction *I) {
2592 if (!EnablePhiOfOps)
2593 return false;
2594 return isa<BinaryOperator>(Val: I) || isa<SelectInst>(Val: I) || isa<CmpInst>(Val: I) ||
2595 isa<LoadInst>(Val: I);
2596}
2597
2598// Return true if this operand will be safe to use for phi of ops.
2599//
2600// The reason some operands are unsafe is that we are not trying to recursively
2601// translate everything back through phi nodes. We actually expect some lookups
2602// of expressions to fail. In particular, a lookup where the expression cannot
2603// exist in the predecessor. This is true even if the expression, as shown, can
2604// be determined to be constant.
2605bool NewGVN::OpIsSafeForPHIOfOps(Value *V, const BasicBlock *PHIBlock,
2606 SmallPtrSetImpl<const Value *> &Visited) {
2607 SmallVector<Value *, 4> Worklist;
2608 Worklist.push_back(Elt: V);
2609 while (!Worklist.empty()) {
2610 auto *I = Worklist.pop_back_val();
2611 if (!isa<Instruction>(Val: I))
2612 continue;
2613
2614 auto OISIt = OpSafeForPHIOfOps.find(Val: {I, CacheIdx});
2615 if (OISIt != OpSafeForPHIOfOps.end())
2616 return OISIt->second;
2617
2618 // Keep walking until we either dominate the phi block, or hit a phi, or run
2619 // out of things to check.
2620 if (DT->properlyDominates(A: getBlockForValue(V: I), B: PHIBlock)) {
2621 OpSafeForPHIOfOps.insert(KV: {{I, CacheIdx}, true});
2622 continue;
2623 }
2624 // PHI in the same block.
2625 if (isa<PHINode>(Val: I) && getBlockForValue(V: I) == PHIBlock) {
2626 OpSafeForPHIOfOps.insert(KV: {{I, CacheIdx}, false});
2627 return false;
2628 }
2629
2630 auto *OrigI = cast<Instruction>(Val: I);
2631 // When we hit an instruction that reads memory (load, call, etc), we must
2632 // consider any store that may happen in the loop. For now, we assume the
2633 // worst: there is a store in the loop that alias with this read.
2634 // The case where the load is outside the loop is already covered by the
2635 // dominator check above.
2636 // TODO: relax this condition
2637 if (OrigI->mayReadFromMemory())
2638 return false;
2639
2640 // Check the operands of the current instruction.
2641 for (auto *Op : OrigI->operand_values()) {
2642 if (!isa<Instruction>(Val: Op))
2643 continue;
2644 // Stop now if we find an unsafe operand.
2645 auto OISIt = OpSafeForPHIOfOps.find(Val: {OrigI, CacheIdx});
2646 if (OISIt != OpSafeForPHIOfOps.end()) {
2647 if (!OISIt->second) {
2648 OpSafeForPHIOfOps.insert(KV: {{I, CacheIdx}, false});
2649 return false;
2650 }
2651 continue;
2652 }
2653 if (!Visited.insert(Ptr: Op).second)
2654 continue;
2655 Worklist.push_back(Elt: cast<Instruction>(Val: Op));
2656 }
2657 }
2658 OpSafeForPHIOfOps.insert(KV: {{V, CacheIdx}, true});
2659 return true;
2660}
2661
2662// Try to find a leader for instruction TransInst, which is a phi translated
2663// version of something in our original program. Visited is used to ensure we
2664// don't infinite loop during translations of cycles. OrigInst is the
2665// instruction in the original program, and PredBB is the predecessor we
2666// translated it through.
2667Value *NewGVN::findLeaderForInst(Instruction *TransInst,
2668 SmallPtrSetImpl<Value *> &Visited,
2669 MemoryAccess *MemAccess, Instruction *OrigInst,
2670 BasicBlock *PredBB) {
2671 unsigned IDFSNum = InstrToDFSNum(V: OrigInst);
2672 // Make sure it's marked as a temporary instruction.
2673 AllTempInstructions.insert(V: TransInst);
2674 // and make sure anything that tries to add it's DFS number is
2675 // redirected to the instruction we are making a phi of ops
2676 // for.
2677 TempToBlock.insert(KV: {TransInst, PredBB});
2678 InstrDFS.insert(KV: {TransInst, IDFSNum});
2679
2680 auto Res = performSymbolicEvaluation(I: TransInst, Visited);
2681 const Expression *E = Res.Expr;
2682 addAdditionalUsers(Res, User: OrigInst);
2683 InstrDFS.erase(Val: TransInst);
2684 AllTempInstructions.erase(V: TransInst);
2685 TempToBlock.erase(Val: TransInst);
2686 if (MemAccess)
2687 TempToMemory.erase(Val: TransInst);
2688 if (!E)
2689 return nullptr;
2690 auto *FoundVal = findPHIOfOpsLeader(E, OrigInst, PredBB);
2691 if (!FoundVal) {
2692 ExpressionToPhiOfOps[E].insert(Ptr: OrigInst);
2693 LLVM_DEBUG(dbgs() << "Cannot find phi of ops operand for " << *TransInst
2694 << " in block " << getBlockName(PredBB) << "\n");
2695 return nullptr;
2696 }
2697 if (auto *SI = dyn_cast<StoreInst>(Val: FoundVal))
2698 FoundVal = SI->getValueOperand();
2699 return FoundVal;
2700}
2701
2702// When we see an instruction that is an op of phis, generate the equivalent phi
2703// of ops form.
2704const Expression *
2705NewGVN::makePossiblePHIOfOps(Instruction *I,
2706 SmallPtrSetImpl<Value *> &Visited) {
2707 if (!okayForPHIOfOps(I))
2708 return nullptr;
2709
2710 if (!Visited.insert(Ptr: I).second)
2711 return nullptr;
2712 // For now, we require the instruction be cycle free because we don't
2713 // *always* create a phi of ops for instructions that could be done as phi
2714 // of ops, we only do it if we think it is useful. If we did do it all the
2715 // time, we could remove the cycle free check.
2716 if (!isCycleFree(I))
2717 return nullptr;
2718
2719 // TODO: We don't do phi translation on memory accesses because it's
2720 // complicated. For a load, we'd need to be able to simulate a new memoryuse,
2721 // which we don't have a good way of doing ATM.
2722 auto *MemAccess = getMemoryAccess(I);
2723 // If the memory operation is defined by a memory operation this block that
2724 // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi
2725 // can't help, as it would still be killed by that memory operation.
2726 if (MemAccess && !isa<MemoryPhi>(Val: MemAccess->getDefiningAccess()) &&
2727 MemAccess->getDefiningAccess()->getBlock() == I->getParent())
2728 return nullptr;
2729
2730 // Convert op of phis to phi of ops
2731 SmallPtrSet<const Value *, 10> VisitedOps;
2732 SmallVector<Value *, 4> Ops(I->operand_values());
2733 BasicBlock *SamePHIBlock = nullptr;
2734 PHINode *OpPHI = nullptr;
2735 if (!DebugCounter::shouldExecute(Counter&: PHIOfOpsCounter))
2736 return nullptr;
2737 for (auto *Op : Ops) {
2738 if (!isa<PHINode>(Val: Op)) {
2739 auto *ValuePHI = RealToTemp.lookup(Val: Op);
2740 if (!ValuePHI)
2741 continue;
2742 LLVM_DEBUG(dbgs() << "Found possible dependent phi of ops\n");
2743 Op = ValuePHI;
2744 }
2745 OpPHI = cast<PHINode>(Val: Op);
2746 if (!SamePHIBlock) {
2747 SamePHIBlock = getBlockForValue(V: OpPHI);
2748 } else if (SamePHIBlock != getBlockForValue(V: OpPHI)) {
2749 LLVM_DEBUG(
2750 dbgs()
2751 << "PHIs for operands are not all in the same block, aborting\n");
2752 return nullptr;
2753 }
2754 // No point in doing this for one-operand phis.
2755 // Since all PHIs for operands must be in the same block, then they must
2756 // have the same number of operands so we can just abort.
2757 if (OpPHI->getNumOperands() == 1)
2758 return nullptr;
2759 }
2760
2761 if (!OpPHI)
2762 return nullptr;
2763
2764 SmallVector<ValPair, 4> PHIOps;
2765 SmallPtrSet<Value *, 4> Deps;
2766 auto *PHIBlock = getBlockForValue(V: OpPHI);
2767 RevisitOnReachabilityChange[PHIBlock].reset(Idx: InstrToDFSNum(V: I));
2768 for (unsigned PredNum = 0; PredNum < OpPHI->getNumOperands(); ++PredNum) {
2769 auto *PredBB = OpPHI->getIncomingBlock(i: PredNum);
2770 Value *FoundVal = nullptr;
2771 SmallPtrSet<Value *, 4> CurrentDeps;
2772 // We could just skip unreachable edges entirely but it's tricky to do
2773 // with rewriting existing phi nodes.
2774 if (ReachableEdges.count(V: {PredBB, PHIBlock})) {
2775 // Clone the instruction, create an expression from it that is
2776 // translated back into the predecessor, and see if we have a leader.
2777 Instruction *ValueOp = I->clone();
2778 // Emit the temporal instruction in the predecessor basic block where the
2779 // corresponding value is defined.
2780 ValueOp->insertBefore(InsertPos: PredBB->getTerminator()->getIterator());
2781 if (MemAccess)
2782 TempToMemory.insert(KV: {ValueOp, MemAccess});
2783 bool SafeForPHIOfOps = true;
2784 VisitedOps.clear();
2785 for (auto &Op : ValueOp->operands()) {
2786 auto *OrigOp = &*Op;
2787 // When these operand changes, it could change whether there is a
2788 // leader for us or not, so we have to add additional users.
2789 if (isa<PHINode>(Val: Op)) {
2790 Op = Op->DoPHITranslation(CurBB: PHIBlock, PredBB);
2791 if (Op != OrigOp && Op != I)
2792 CurrentDeps.insert(Ptr: Op);
2793 } else if (auto *ValuePHI = RealToTemp.lookup(Val: Op)) {
2794 if (getBlockForValue(V: ValuePHI) == PHIBlock)
2795 Op = ValuePHI->getIncomingValueForBlock(BB: PredBB);
2796 }
2797 // If we phi-translated the op, it must be safe.
2798 SafeForPHIOfOps =
2799 SafeForPHIOfOps &&
2800 (Op != OrigOp || OpIsSafeForPHIOfOps(V: Op, PHIBlock, Visited&: VisitedOps));
2801 }
2802 // FIXME: For those things that are not safe we could generate
2803 // expressions all the way down, and see if this comes out to a
2804 // constant. For anything where that is true, and unsafe, we should
2805 // have made a phi-of-ops (or value numbered it equivalent to something)
2806 // for the pieces already.
2807 FoundVal = !SafeForPHIOfOps ? nullptr
2808 : findLeaderForInst(TransInst: ValueOp, Visited,
2809 MemAccess, OrigInst: I, PredBB);
2810 ValueOp->eraseFromParent();
2811 if (!FoundVal) {
2812 // We failed to find a leader for the current ValueOp, but this might
2813 // change in case of the translated operands change.
2814 if (SafeForPHIOfOps)
2815 for (auto *Dep : CurrentDeps)
2816 addAdditionalUsers(To: Dep, User: I);
2817
2818 return nullptr;
2819 }
2820 Deps.insert_range(R&: CurrentDeps);
2821 } else {
2822 LLVM_DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "
2823 << getBlockName(PredBB)
2824 << " because the block is unreachable\n");
2825 FoundVal = PoisonValue::get(T: I->getType());
2826 RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(V: I));
2827 }
2828
2829 PHIOps.push_back(Elt: {FoundVal, PredBB});
2830 LLVM_DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "
2831 << getBlockName(PredBB) << "\n");
2832 }
2833 for (auto *Dep : Deps)
2834 addAdditionalUsers(To: Dep, User: I);
2835 sortPHIOps(Ops: PHIOps);
2836 auto *E = performSymbolicPHIEvaluation(PHIOps, I, PHIBlock);
2837 if (isa<ConstantExpression>(Val: E) || isa<VariableExpression>(Val: E)) {
2838 LLVM_DEBUG(
2839 dbgs()
2840 << "Not creating real PHI of ops because it simplified to existing "
2841 "value or constant\n");
2842 // We have leaders for all operands, but do not create a real PHI node with
2843 // those leaders as operands, so the link between the operands and the
2844 // PHI-of-ops is not materialized in the IR. If any of those leaders
2845 // changes, the PHI-of-op may change also, so we need to add the operands as
2846 // additional users.
2847 for (auto &O : PHIOps)
2848 addAdditionalUsers(To: O.first, User: I);
2849
2850 return E;
2851 }
2852 auto *ValuePHI = RealToTemp.lookup(Val: I);
2853 bool NewPHI = false;
2854 if (!ValuePHI) {
2855 ValuePHI =
2856 PHINode::Create(Ty: I->getType(), NumReservedValues: OpPHI->getNumOperands(), NameStr: "phiofops");
2857 addPhiOfOps(Op: ValuePHI, BB: PHIBlock, ExistingValue: I);
2858 NewPHI = true;
2859 NumGVNPHIOfOpsCreated++;
2860 }
2861 if (NewPHI) {
2862 for (auto PHIOp : PHIOps)
2863 ValuePHI->addIncoming(V: PHIOp.first, BB: PHIOp.second);
2864 } else {
2865 TempToBlock[ValuePHI] = PHIBlock;
2866 unsigned int i = 0;
2867 for (auto PHIOp : PHIOps) {
2868 ValuePHI->setIncomingValue(i, V: PHIOp.first);
2869 ValuePHI->setIncomingBlock(i, BB: PHIOp.second);
2870 ++i;
2871 }
2872 }
2873 RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(V: I));
2874 LLVM_DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *I
2875 << "\n");
2876
2877 return E;
2878}
2879
2880// The algorithm initially places the values of the routine in the TOP
2881// congruence class. The leader of TOP is the undetermined value `poison`.
2882// When the algorithm has finished, values still in TOP are unreachable.
2883void NewGVN::initializeCongruenceClasses(Function &F) {
2884 NextCongruenceNum = 0;
2885
2886 // Note that even though we use the live on entry def as a representative
2887 // MemoryAccess, it is *not* the same as the actual live on entry def. We
2888 // have no real equivalent to poison for MemoryAccesses, and so we really
2889 // should be checking whether the MemoryAccess is top if we want to know if it
2890 // is equivalent to everything. Otherwise, what this really signifies is that
2891 // the access "it reaches all the way back to the beginning of the function"
2892
2893 // Initialize all other instructions to be in TOP class.
2894 TOPClass = createCongruenceClass(Leader: nullptr, E: nullptr);
2895 TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());
2896 // The live on entry def gets put into it's own class
2897 MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =
2898 createMemoryClass(MA: MSSA->getLiveOnEntryDef());
2899
2900 for (auto *DTN : nodes(G: DT)) {
2901 BasicBlock *BB = DTN->getBlock();
2902 // All MemoryAccesses are equivalent to live on entry to start. They must
2903 // be initialized to something so that initial changes are noticed. For
2904 // the maximal answer, we initialize them all to be the same as
2905 // liveOnEntry.
2906 auto *MemoryBlockDefs = MSSA->getBlockDefs(BB);
2907 if (MemoryBlockDefs)
2908 for (const auto &Def : *MemoryBlockDefs) {
2909 MemoryAccessToClass[&Def] = TOPClass;
2910 auto *MD = dyn_cast<MemoryDef>(Val: &Def);
2911 // Insert the memory phis into the member list.
2912 if (!MD) {
2913 const MemoryPhi *MP = cast<MemoryPhi>(Val: &Def);
2914 TOPClass->memory_insert(M: MP);
2915 MemoryPhiState.insert(KV: {MP, MPS_TOP});
2916 }
2917
2918 if (MD && isa<StoreInst>(Val: MD->getMemoryInst()))
2919 TOPClass->incStoreCount();
2920 }
2921
2922 // FIXME: This is trying to discover which instructions are uses of phi
2923 // nodes. We should move this into one of the myriad of places that walk
2924 // all the operands already.
2925 for (auto &I : *BB) {
2926 if (isa<PHINode>(Val: &I))
2927 for (auto *U : I.users())
2928 if (auto *UInst = dyn_cast<Instruction>(Val: U))
2929 if (InstrToDFSNum(V: UInst) != 0 && okayForPHIOfOps(I: UInst))
2930 PHINodeUses.insert(Ptr: UInst);
2931 // Don't insert void terminators into the class. We don't value number
2932 // them, and they just end up sitting in TOP.
2933 if (I.isTerminator() && I.getType()->isVoidTy())
2934 continue;
2935 TOPClass->insert(M: &I);
2936 ValueToClass[&I] = TOPClass;
2937 }
2938 }
2939
2940 // Initialize arguments to be in their own unique congruence classes
2941 for (auto &FA : F.args())
2942 createSingletonCongruenceClass(Member: &FA);
2943}
2944
2945void NewGVN::cleanupTables() {
2946 for (CongruenceClass *&CC : CongruenceClasses) {
2947 LLVM_DEBUG(dbgs() << "Congruence class " << CC->getID() << " has "
2948 << CC->size() << " members\n");
2949 // Make sure we delete the congruence class (probably worth switching to
2950 // a unique_ptr at some point.
2951 delete CC;
2952 CC = nullptr;
2953 }
2954
2955 // Destroy the value expressions
2956 SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(),
2957 AllTempInstructions.end());
2958 AllTempInstructions.clear();
2959
2960 // We have to drop all references for everything first, so there are no uses
2961 // left as we delete them.
2962 for (auto *I : TempInst) {
2963 I->dropAllReferences();
2964 }
2965
2966 while (!TempInst.empty()) {
2967 auto *I = TempInst.pop_back_val();
2968 I->deleteValue();
2969 }
2970
2971 ValueToClass.clear();
2972 ArgRecycler.clear(ExpressionAllocator);
2973 ExpressionAllocator.Reset();
2974 CongruenceClasses.clear();
2975 ExpressionToClass.clear();
2976 ValueToExpression.clear();
2977 RealToTemp.clear();
2978 AdditionalUsers.clear();
2979 ExpressionToPhiOfOps.clear();
2980 TempToBlock.clear();
2981 TempToMemory.clear();
2982 PHINodeUses.clear();
2983 OpSafeForPHIOfOps.clear();
2984 ReachableBlocks.clear();
2985 ReachableEdges.clear();
2986#ifndef NDEBUG
2987 ProcessedCount.clear();
2988#endif
2989 InstrDFS.clear();
2990 InstructionsToErase.clear();
2991 DFSToInstr.clear();
2992 BlockInstRange.clear();
2993 TouchedInstructions.clear();
2994 MemoryAccessToClass.clear();
2995 PredicateToUsers.clear();
2996 MemoryToUsers.clear();
2997 RevisitOnReachabilityChange.clear();
2998 PredicateSwapChoice.clear();
2999}
3000
3001// Assign local DFS number mapping to instructions, and leave space for Value
3002// PHI's.
3003std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
3004 unsigned Start) {
3005 unsigned End = Start;
3006 if (MemoryAccess *MemPhi = getMemoryAccess(BB: B)) {
3007 InstrDFS[MemPhi] = End++;
3008 DFSToInstr.emplace_back(Args&: MemPhi);
3009 }
3010
3011 // Then the real block goes next.
3012 for (auto &I : *B) {
3013 // There's no need to call isInstructionTriviallyDead more than once on
3014 // an instruction. Therefore, once we know that an instruction is dead
3015 // we change its DFS number so that it doesn't get value numbered.
3016 if (isInstructionTriviallyDead(I: &I, TLI)) {
3017 InstrDFS[&I] = 0;
3018 LLVM_DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");
3019 salvageDebugInfo(I);
3020 markInstructionForDeletion(&I);
3021 continue;
3022 }
3023 if (isa<PHINode>(Val: &I))
3024 RevisitOnReachabilityChange[B].set(End);
3025 InstrDFS[&I] = End++;
3026 DFSToInstr.emplace_back(Args: &I);
3027 }
3028
3029 // All of the range functions taken half-open ranges (open on the end side).
3030 // So we do not subtract one from count, because at this point it is one
3031 // greater than the last instruction.
3032 return std::make_pair(x&: Start, y&: End);
3033}
3034
3035void NewGVN::updateProcessedCount(const Value *V) {
3036#ifndef NDEBUG
3037 assert(++ProcessedCount[V] < 100 &&
3038 "Seem to have processed the same Value a lot");
3039#endif
3040}
3041
3042// Evaluate MemoryPhi nodes symbolically, just like PHI nodes
3043void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
3044 // If all the arguments are the same, the MemoryPhi has the same value as the
3045 // argument. Filter out unreachable blocks and self phis from our operands.
3046 // TODO: We could do cycle-checking on the memory phis to allow valueizing for
3047 // self-phi checking.
3048 const BasicBlock *PHIBlock = MP->getBlock();
3049 auto Filtered = make_filter_range(Range: MP->operands(), Pred: [&](const Use &U) {
3050 return cast<MemoryAccess>(Val: U) != MP &&
3051 !isMemoryAccessTOP(MA: cast<MemoryAccess>(Val: U)) &&
3052 ReachableEdges.count(V: {MP->getIncomingBlock(U), PHIBlock});
3053 });
3054 // If all that is left is nothing, our memoryphi is poison. We keep it as
3055 // InitialClass. Note: The only case this should happen is if we have at
3056 // least one self-argument.
3057 if (Filtered.begin() == Filtered.end()) {
3058 if (setMemoryClass(From: MP, NewClass: TOPClass))
3059 markMemoryUsersTouched(MA: MP);
3060 return;
3061 }
3062
3063 // Transform the remaining operands into operand leaders.
3064 // FIXME: mapped_iterator should have a range version.
3065 auto LookupFunc = [&](const Use &U) {
3066 return lookupMemoryLeader(MA: cast<MemoryAccess>(Val: U));
3067 };
3068 auto MappedBegin = map_iterator(I: Filtered.begin(), F: LookupFunc);
3069 auto MappedEnd = map_iterator(I: Filtered.end(), F: LookupFunc);
3070
3071 // and now check if all the elements are equal.
3072 // Sadly, we can't use std::equals since these are random access iterators.
3073 const auto *AllSameValue = *MappedBegin;
3074 ++MappedBegin;
3075 bool AllEqual = std::all_of(
3076 first: MappedBegin, last: MappedEnd,
3077 pred: [&AllSameValue](const MemoryAccess *V) { return V == AllSameValue; });
3078
3079 if (AllEqual)
3080 LLVM_DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue
3081 << "\n");
3082 else
3083 LLVM_DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
3084 // If it's equal to something, it's in that class. Otherwise, it has to be in
3085 // a class where it is the leader (other things may be equivalent to it, but
3086 // it needs to start off in its own class, which means it must have been the
3087 // leader, and it can't have stopped being the leader because it was never
3088 // removed).
3089 CongruenceClass *CC =
3090 AllEqual ? getMemoryClass(MA: AllSameValue) : ensureLeaderOfMemoryClass(MA: MP);
3091 auto OldState = MemoryPhiState.lookup(Val: MP);
3092 assert(OldState != MPS_Invalid && "Invalid memory phi state");
3093 auto NewState = AllEqual ? MPS_Equivalent : MPS_Unique;
3094 MemoryPhiState[MP] = NewState;
3095 if (setMemoryClass(From: MP, NewClass: CC) || OldState != NewState)
3096 markMemoryUsersTouched(MA: MP);
3097}
3098
3099// Value number a single instruction, symbolically evaluating, performing
3100// congruence finding, and updating mappings.
3101void NewGVN::valueNumberInstruction(Instruction *I) {
3102 LLVM_DEBUG(dbgs() << "Processing instruction " << *I << "\n");
3103 if (!I->isTerminator()) {
3104 const Expression *Symbolized = nullptr;
3105 SmallPtrSet<Value *, 2> Visited;
3106 if (DebugCounter::shouldExecute(Counter&: VNCounter)) {
3107 auto Res = performSymbolicEvaluation(I, Visited);
3108 Symbolized = Res.Expr;
3109 addAdditionalUsers(Res, User: I);
3110
3111 // Make a phi of ops if necessary
3112 if (Symbolized && !isa<ConstantExpression>(Val: Symbolized) &&
3113 !isa<VariableExpression>(Val: Symbolized) && PHINodeUses.count(Ptr: I)) {
3114 auto *PHIE = makePossiblePHIOfOps(I, Visited);
3115 // If we created a phi of ops, use it.
3116 // If we couldn't create one, make sure we don't leave one lying around
3117 if (PHIE) {
3118 Symbolized = PHIE;
3119 } else if (auto *Op = RealToTemp.lookup(Val: I)) {
3120 removePhiOfOps(I, PHITemp: Op);
3121 }
3122 }
3123 } else {
3124 // Mark the instruction as unused so we don't value number it again.
3125 InstrDFS[I] = 0;
3126 }
3127 // If we couldn't come up with a symbolic expression, use the unknown
3128 // expression
3129 if (Symbolized == nullptr)
3130 Symbolized = createUnknownExpression(I);
3131 performCongruenceFinding(I, E: Symbolized);
3132 } else {
3133 // Handle terminators that return values. All of them produce values we
3134 // don't currently understand. We don't place non-value producing
3135 // terminators in a class.
3136 if (!I->getType()->isVoidTy()) {
3137 auto *Symbolized = createUnknownExpression(I);
3138 performCongruenceFinding(I, E: Symbolized);
3139 }
3140 processOutgoingEdges(TI: I, B: I->getParent());
3141 }
3142}
3143
3144// Check if there is a path, using single or equal argument phi nodes, from
3145// First to Second.
3146bool NewGVN::singleReachablePHIPath(
3147 SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First,
3148 const MemoryAccess *Second) const {
3149 if (First == Second)
3150 return true;
3151 if (MSSA->isLiveOnEntryDef(MA: First))
3152 return false;
3153
3154 // This is not perfect, but as we're just verifying here, we can live with
3155 // the loss of precision. The real solution would be that of doing strongly
3156 // connected component finding in this routine, and it's probably not worth
3157 // the complexity for the time being. So, we just keep a set of visited
3158 // MemoryAccess and return true when we hit a cycle.
3159 if (!Visited.insert(Ptr: First).second)
3160 return true;
3161
3162 const auto *EndDef = First;
3163 for (const auto *ChainDef : optimized_def_chain(MA: First)) {
3164 if (ChainDef == Second)
3165 return true;
3166 if (MSSA->isLiveOnEntryDef(MA: ChainDef))
3167 return false;
3168 EndDef = ChainDef;
3169 }
3170 auto *MP = cast<MemoryPhi>(Val: EndDef);
3171 auto ReachableOperandPred = [&](const Use &U) {
3172 return ReachableEdges.count(V: {MP->getIncomingBlock(U), MP->getBlock()});
3173 };
3174 auto FilteredPhiArgs =
3175 make_filter_range(Range: MP->operands(), Pred: ReachableOperandPred);
3176 SmallVector<const Value *, 32> OperandList(FilteredPhiArgs);
3177 bool Okay = all_equal(Range&: OperandList);
3178 if (Okay)
3179 return singleReachablePHIPath(Visited, First: cast<MemoryAccess>(Val: OperandList[0]),
3180 Second);
3181 return false;
3182}
3183
3184// Verify the that the memory equivalence table makes sense relative to the
3185// congruence classes. Note that this checking is not perfect, and is currently
3186// subject to very rare false negatives. It is only useful for
3187// testing/debugging.
3188void NewGVN::verifyMemoryCongruency() const {
3189#ifndef NDEBUG
3190 // Verify that the memory table equivalence and memory member set match
3191 for (const auto *CC : CongruenceClasses) {
3192 if (CC == TOPClass || CC->isDead())
3193 continue;
3194 if (CC->getStoreCount() != 0) {
3195 assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&
3196 "Any class with a store as a leader should have a "
3197 "representative stored value");
3198 assert(CC->getMemoryLeader() &&
3199 "Any congruence class with a store should have a "
3200 "representative access");
3201 }
3202
3203 if (CC->getMemoryLeader())
3204 assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&
3205 "Representative MemoryAccess does not appear to be reverse "
3206 "mapped properly");
3207 for (const auto *M : CC->memory())
3208 assert(MemoryAccessToClass.lookup(M) == CC &&
3209 "Memory member does not appear to be reverse mapped properly");
3210 }
3211
3212 // Anything equivalent in the MemoryAccess table should be in the same
3213 // congruence class.
3214
3215 // Filter out the unreachable and trivially dead entries, because they may
3216 // never have been updated if the instructions were not processed.
3217 auto ReachableAccessPred =
3218 [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {
3219 bool Result = ReachableBlocks.count(Pair.first->getBlock());
3220 if (!Result || MSSA->isLiveOnEntryDef(Pair.first) ||
3221 MemoryToDFSNum(Pair.first) == 0)
3222 return false;
3223 if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
3224 return !isInstructionTriviallyDead(MemDef->getMemoryInst());
3225
3226 // We could have phi nodes which operands are all trivially dead,
3227 // so we don't process them.
3228 if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) {
3229 for (const auto &U : MemPHI->incoming_values()) {
3230 if (auto *I = dyn_cast<Instruction>(&*U)) {
3231 if (!isInstructionTriviallyDead(I))
3232 return true;
3233 }
3234 }
3235 return false;
3236 }
3237
3238 return true;
3239 };
3240
3241 auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);
3242 for (auto KV : Filtered) {
3243 if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
3244 auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());
3245 if (FirstMUD && SecondMUD) {
3246 SmallPtrSet<const MemoryAccess *, 8> VisitedMAS;
3247 assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||
3248 ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
3249 ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
3250 "The instructions for these memory operations should have "
3251 "been in the same congruence class or reachable through"
3252 "a single argument phi");
3253 }
3254 } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
3255 // We can only sanely verify that MemoryDefs in the operand list all have
3256 // the same class.
3257 auto ReachableOperandPred = [&](const Use &U) {
3258 return ReachableEdges.count(
3259 {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&
3260 isa<MemoryDef>(U);
3261 };
3262 // All arguments should in the same class, ignoring unreachable arguments
3263 auto FilteredPhiArgs =
3264 make_filter_range(FirstMP->operands(), ReachableOperandPred);
3265 SmallVector<const CongruenceClass *, 16> PhiOpClasses;
3266 std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
3267 std::back_inserter(PhiOpClasses), [&](const Use &U) {
3268 const MemoryDef *MD = cast<MemoryDef>(U);
3269 return ValueToClass.lookup(MD->getMemoryInst());
3270 });
3271 assert(all_equal(PhiOpClasses) &&
3272 "All MemoryPhi arguments should be in the same class");
3273 }
3274 }
3275#endif
3276}
3277
3278// Verify that the sparse propagation we did actually found the maximal fixpoint
3279// We do this by storing the value to class mapping, touching all instructions,
3280// and redoing the iteration to see if anything changed.
3281void NewGVN::verifyIterationSettled(Function &F) {
3282#ifndef NDEBUG
3283 LLVM_DEBUG(dbgs() << "Beginning iteration verification\n");
3284 if (DebugCounter::isCounterSet(VNCounter))
3285 DebugCounter::setCounterState(VNCounter, StartingVNCounter);
3286
3287 // Note that we have to store the actual classes, as we may change existing
3288 // classes during iteration. This is because our memory iteration propagation
3289 // is not perfect, and so may waste a little work. But it should generate
3290 // exactly the same congruence classes we have now, with different IDs.
3291 std::map<const Value *, CongruenceClass> BeforeIteration;
3292
3293 for (auto &KV : ValueToClass) {
3294 if (auto *I = dyn_cast<Instruction>(KV.first))
3295 // Skip unused/dead instructions.
3296 if (InstrToDFSNum(I) == 0)
3297 continue;
3298 BeforeIteration.insert({KV.first, *KV.second});
3299 }
3300
3301 TouchedInstructions.set();
3302 TouchedInstructions.reset(0);
3303 OpSafeForPHIOfOps.clear();
3304 CacheIdx = 0;
3305 iterateTouchedInstructions();
3306 DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>>
3307 EqualClasses;
3308 for (const auto &KV : ValueToClass) {
3309 if (auto *I = dyn_cast<Instruction>(KV.first))
3310 // Skip unused/dead instructions.
3311 if (InstrToDFSNum(I) == 0)
3312 continue;
3313 // We could sink these uses, but i think this adds a bit of clarity here as
3314 // to what we are comparing.
3315 auto *BeforeCC = &BeforeIteration.find(KV.first)->second;
3316 auto *AfterCC = KV.second;
3317 // Note that the classes can't change at this point, so we memoize the set
3318 // that are equal.
3319 if (!EqualClasses.count({BeforeCC, AfterCC})) {
3320 assert(BeforeCC->isEquivalentTo(AfterCC) &&
3321 "Value number changed after main loop completed!");
3322 EqualClasses.insert({BeforeCC, AfterCC});
3323 }
3324 }
3325#endif
3326}
3327
3328// Verify that for each store expression in the expression to class mapping,
3329// only the latest appears, and multiple ones do not appear.
3330// Because loads do not use the stored value when doing equality with stores,
3331// if we don't erase the old store expressions from the table, a load can find
3332// a no-longer valid StoreExpression.
3333void NewGVN::verifyStoreExpressions() const {
3334#ifndef NDEBUG
3335 // This is the only use of this, and it's not worth defining a complicated
3336 // densemapinfo hash/equality function for it.
3337 std::set<
3338 std::pair<const Value *,
3339 std::tuple<const Value *, const CongruenceClass *, Value *>>>
3340 StoreExpressionSet;
3341 for (const auto &KV : ExpressionToClass) {
3342 if (auto *SE = dyn_cast<StoreExpression>(KV.first)) {
3343 // Make sure a version that will conflict with loads is not already there
3344 auto Res = StoreExpressionSet.insert(
3345 {SE->getOperand(0), std::make_tuple(SE->getMemoryLeader(), KV.second,
3346 SE->getStoredValue())});
3347 bool Okay = Res.second;
3348 // It's okay to have the same expression already in there if it is
3349 // identical in nature.
3350 // This can happen when the leader of the stored value changes over time.
3351 if (!Okay)
3352 Okay = (std::get<1>(Res.first->second) == KV.second) &&
3353 (lookupOperandLeader(std::get<2>(Res.first->second)) ==
3354 lookupOperandLeader(SE->getStoredValue()));
3355 assert(Okay && "Stored expression conflict exists in expression table");
3356 auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst());
3357 assert(ValueExpr && ValueExpr->equals(*SE) &&
3358 "StoreExpression in ExpressionToClass is not latest "
3359 "StoreExpression for value");
3360 }
3361 }
3362#endif
3363}
3364
3365// This is the main value numbering loop, it iterates over the initial touched
3366// instruction set, propagating value numbers, marking things touched, etc,
3367// until the set of touched instructions is completely empty.
3368void NewGVN::iterateTouchedInstructions() {
3369 uint64_t Iterations = 0;
3370 // Figure out where touchedinstructions starts
3371 int FirstInstr = TouchedInstructions.find_first();
3372 // Nothing set, nothing to iterate, just return.
3373 if (FirstInstr == -1)
3374 return;
3375 const BasicBlock *LastBlock = getBlockForValue(V: InstrFromDFSNum(DFSNum: FirstInstr));
3376 while (TouchedInstructions.any()) {
3377 ++Iterations;
3378 // Walk through all the instructions in all the blocks in RPO.
3379 // TODO: As we hit a new block, we should push and pop equalities into a
3380 // table lookupOperandLeader can use, to catch things PredicateInfo
3381 // might miss, like edge-only equivalences.
3382 for (unsigned InstrNum : TouchedInstructions.set_bits()) {
3383
3384 // This instruction was found to be dead. We don't bother looking
3385 // at it again.
3386 if (InstrNum == 0) {
3387 TouchedInstructions.reset(Idx: InstrNum);
3388 continue;
3389 }
3390
3391 Value *V = InstrFromDFSNum(DFSNum: InstrNum);
3392 const BasicBlock *CurrBlock = getBlockForValue(V);
3393
3394 // If we hit a new block, do reachability processing.
3395 if (CurrBlock != LastBlock) {
3396 LastBlock = CurrBlock;
3397 bool BlockReachable = ReachableBlocks.count(Ptr: CurrBlock);
3398 const auto &CurrInstRange = BlockInstRange.lookup(Val: CurrBlock);
3399
3400 // If it's not reachable, erase any touched instructions and move on.
3401 if (!BlockReachable) {
3402 TouchedInstructions.reset(I: CurrInstRange.first, E: CurrInstRange.second);
3403 LLVM_DEBUG(dbgs() << "Skipping instructions in block "
3404 << getBlockName(CurrBlock)
3405 << " because it is unreachable\n");
3406 continue;
3407 }
3408 // Use the appropriate cache for "OpIsSafeForPHIOfOps".
3409 CacheIdx = RPOOrdering.lookup(Val: DT->getNode(BB: CurrBlock)) - 1;
3410 updateProcessedCount(V: CurrBlock);
3411 }
3412 // Reset after processing (because we may mark ourselves as touched when
3413 // we propagate equalities).
3414 TouchedInstructions.reset(Idx: InstrNum);
3415
3416 if (auto *MP = dyn_cast<MemoryPhi>(Val: V)) {
3417 LLVM_DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
3418 valueNumberMemoryPhi(MP);
3419 } else if (auto *I = dyn_cast<Instruction>(Val: V)) {
3420 valueNumberInstruction(I);
3421 } else {
3422 llvm_unreachable("Should have been a MemoryPhi or Instruction");
3423 }
3424 updateProcessedCount(V);
3425 }
3426 }
3427 NumGVNMaxIterations = std::max(a: NumGVNMaxIterations.getValue(), b: Iterations);
3428}
3429
3430// This is the main transformation entry point.
3431bool NewGVN::runGVN() {
3432 if (DebugCounter::isCounterSet(Info&: VNCounter))
3433 StartingVNCounter = DebugCounter::getCounterState(Info&: VNCounter);
3434 bool Changed = false;
3435 NumFuncArgs = F.arg_size();
3436 MSSAWalker = MSSA->getWalker();
3437 SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();
3438
3439 // Count number of instructions for sizing of hash tables, and come
3440 // up with a global dfs numbering for instructions.
3441 unsigned ICount = 1;
3442 // Add an empty instruction to account for the fact that we start at 1
3443 DFSToInstr.emplace_back(Args: nullptr);
3444 // Note: We want ideal RPO traversal of the blocks, which is not quite the
3445 // same as dominator tree order, particularly with regard whether backedges
3446 // get visited first or second, given a block with multiple successors.
3447 // If we visit in the wrong order, we will end up performing N times as many
3448 // iterations.
3449 // The dominator tree does guarantee that, for a given dom tree node, it's
3450 // parent must occur before it in the RPO ordering. Thus, we only need to sort
3451 // the siblings.
3452 ReversePostOrderTraversal<Function *> RPOT(&F);
3453 unsigned Counter = 0;
3454 for (auto &B : RPOT) {
3455 auto *Node = DT->getNode(BB: B);
3456 assert(Node && "RPO and Dominator tree should have same reachability");
3457 RPOOrdering[Node] = ++Counter;
3458 }
3459 // Sort dominator tree children arrays into RPO.
3460 // TODO: this code shouldn't rely on domtree internals. It also most probably
3461 // shouldn't rely on the order of nodes in the tree...
3462 for (auto &B : RPOT) {
3463 auto *Node = DT->getNode(BB: B);
3464 if (Node->isLeaf())
3465 continue;
3466 SmallVector<DomTreeNode *> Children;
3467 while (!Node->isLeaf()) {
3468 Children.push_back(Elt: *Node->begin());
3469 Node->removeChild(C: *Node->begin());
3470 }
3471 llvm::sort(C&: Children, Comp: [&](const DomTreeNode *A, const DomTreeNode *B) {
3472 return RPOOrdering[A] < RPOOrdering[B];
3473 });
3474 for (DomTreeNode *Child : Children)
3475 Node->addChild(C: Child);
3476 }
3477
3478 // Now a standard depth first ordering of the domtree is equivalent to RPO.
3479 for (auto *DTN : depth_first(G: DT->getRootNode())) {
3480 BasicBlock *B = DTN->getBlock();
3481 const auto &BlockRange = assignDFSNumbers(B, Start: ICount);
3482 BlockInstRange.insert(KV: {B, BlockRange});
3483 ICount += BlockRange.second - BlockRange.first;
3484 }
3485 initializeCongruenceClasses(F);
3486
3487 TouchedInstructions.resize(N: ICount);
3488 // Ensure we don't end up resizing the expressionToClass map, as
3489 // that can be quite expensive. At most, we have one expression per
3490 // instruction.
3491 ExpressionToClass.reserve(NumEntries: ICount);
3492
3493 // Initialize the touched instructions to include the entry block.
3494 const auto &InstRange = BlockInstRange.lookup(Val: &F.getEntryBlock());
3495 TouchedInstructions.set(I: InstRange.first, E: InstRange.second);
3496 LLVM_DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())
3497 << " marked reachable\n");
3498 ReachableBlocks.insert(Ptr: &F.getEntryBlock());
3499 // Use index corresponding to entry block.
3500 CacheIdx = 0;
3501
3502 iterateTouchedInstructions();
3503 verifyMemoryCongruency();
3504 verifyIterationSettled(F);
3505 verifyStoreExpressions();
3506
3507 Changed |= eliminateInstructions(F);
3508
3509 // Delete all instructions marked for deletion.
3510 for (Instruction *ToErase : InstructionsToErase) {
3511 if (!ToErase->use_empty())
3512 ToErase->replaceAllUsesWith(V: PoisonValue::get(T: ToErase->getType()));
3513
3514 assert(ToErase->getParent() &&
3515 "BB containing ToErase deleted unexpectedly!");
3516 ToErase->eraseFromParent();
3517 }
3518 Changed |= !InstructionsToErase.empty();
3519
3520 // Delete all unreachable blocks.
3521 auto UnreachableBlockPred = [&](const BasicBlock &BB) {
3522 return !ReachableBlocks.count(Ptr: &BB);
3523 };
3524
3525 for (auto &BB : make_filter_range(Range&: F, Pred: UnreachableBlockPred)) {
3526 LLVM_DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
3527 << " is unreachable\n");
3528 deleteInstructionsInBlock(&BB);
3529 Changed = true;
3530 }
3531
3532 cleanupTables();
3533 return Changed;
3534}
3535
3536struct NewGVN::ValueDFS {
3537 int DFSIn = 0;
3538 int DFSOut = 0;
3539 int LocalNum = 0;
3540
3541 // Only one of Def and U will be set.
3542 // The bool in the Def tells us whether the Def is the stored value of a
3543 // store.
3544 PointerIntPair<Value *, 1, bool> Def;
3545 Use *U = nullptr;
3546
3547 bool operator<(const ValueDFS &Other) const {
3548 // It's not enough that any given field be less than - we have sets
3549 // of fields that need to be evaluated together to give a proper ordering.
3550 // For example, if you have;
3551 // DFS (1, 3)
3552 // Val 0
3553 // DFS (1, 2)
3554 // Val 50
3555 // We want the second to be less than the first, but if we just go field
3556 // by field, we will get to Val 0 < Val 50 and say the first is less than
3557 // the second. We only want it to be less than if the DFS orders are equal.
3558 //
3559 // Each LLVM instruction only produces one value, and thus the lowest-level
3560 // differentiator that really matters for the stack (and what we use as a
3561 // replacement) is the local dfs number.
3562 // Everything else in the structure is instruction level, and only affects
3563 // the order in which we will replace operands of a given instruction.
3564 //
3565 // For a given instruction (IE things with equal dfsin, dfsout, localnum),
3566 // the order of replacement of uses does not matter.
3567 // IE given,
3568 // a = 5
3569 // b = a + a
3570 // When you hit b, you will have two valuedfs with the same dfsin, out, and
3571 // localnum.
3572 // The .val will be the same as well.
3573 // The .u's will be different.
3574 // You will replace both, and it does not matter what order you replace them
3575 // in (IE whether you replace operand 2, then operand 1, or operand 1, then
3576 // operand 2).
3577 // Similarly for the case of same dfsin, dfsout, localnum, but different
3578 // .val's
3579 // a = 5
3580 // b = 6
3581 // c = a + b
3582 // in c, we will a valuedfs for a, and one for b,with everything the same
3583 // but .val and .u.
3584 // It does not matter what order we replace these operands in.
3585 // You will always end up with the same IR, and this is guaranteed.
3586 return std::tie(args: DFSIn, args: DFSOut, args: LocalNum, args: Def, args: U) <
3587 std::tie(args: Other.DFSIn, args: Other.DFSOut, args: Other.LocalNum, args: Other.Def,
3588 args: Other.U);
3589 }
3590};
3591
3592// This function converts the set of members for a congruence class from values,
3593// to sets of defs and uses with associated DFS info. The total number of
3594// reachable uses for each value is stored in UseCount, and instructions that
3595// seem
3596// dead (have no non-dead uses) are stored in ProbablyDead.
3597void NewGVN::convertClassToDFSOrdered(
3598 const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,
3599 DenseMap<const Value *, unsigned int> &UseCounts,
3600 SmallPtrSetImpl<Instruction *> &ProbablyDead) const {
3601 for (auto *D : Dense) {
3602 // First add the value.
3603 BasicBlock *BB = getBlockForValue(V: D);
3604 // Constants are handled prior to ever calling this function, so
3605 // we should only be left with instructions as members.
3606 assert(BB && "Should have figured out a basic block for value");
3607 ValueDFS VDDef;
3608 DomTreeNode *DomNode = DT->getNode(BB);
3609 VDDef.DFSIn = DomNode->getDFSNumIn();
3610 VDDef.DFSOut = DomNode->getDFSNumOut();
3611 // If it's a store, use the leader of the value operand, if it's always
3612 // available, or the value operand. TODO: We could do dominance checks to
3613 // find a dominating leader, but not worth it ATM.
3614 if (auto *SI = dyn_cast<StoreInst>(Val: D)) {
3615 auto Leader = lookupOperandLeader(V: SI->getValueOperand());
3616 if (alwaysAvailable(V: Leader)) {
3617 VDDef.Def.setPointer(Leader);
3618 } else {
3619 VDDef.Def.setPointer(SI->getValueOperand());
3620 VDDef.Def.setInt(true);
3621 }
3622 } else {
3623 VDDef.Def.setPointer(D);
3624 }
3625 assert(isa<Instruction>(D) &&
3626 "The dense set member should always be an instruction");
3627 Instruction *Def = cast<Instruction>(Val: D);
3628 VDDef.LocalNum = InstrToDFSNum(V: D);
3629 DFSOrderedSet.push_back(Elt: VDDef);
3630 // If there is a phi node equivalent, add it
3631 if (auto *PN = RealToTemp.lookup(Val: Def)) {
3632 auto *PHIE =
3633 dyn_cast_or_null<PHIExpression>(Val: ValueToExpression.lookup(Val: Def));
3634 if (PHIE) {
3635 VDDef.Def.setInt(false);
3636 VDDef.Def.setPointer(PN);
3637 VDDef.LocalNum = 0;
3638 DFSOrderedSet.push_back(Elt: VDDef);
3639 }
3640 }
3641
3642 unsigned int UseCount = 0;
3643 // Now add the uses.
3644 for (auto &U : Def->uses()) {
3645 if (auto *I = dyn_cast<Instruction>(Val: U.getUser())) {
3646 // Don't try to replace into dead uses
3647 if (InstructionsToErase.count(Ptr: I))
3648 continue;
3649 ValueDFS VDUse;
3650 // Put the phi node uses in the incoming block.
3651 BasicBlock *IBlock;
3652 if (auto *P = dyn_cast<PHINode>(Val: I)) {
3653 IBlock = P->getIncomingBlock(U);
3654 // Make phi node users appear last in the incoming block
3655 // they are from.
3656 VDUse.LocalNum = InstrDFS.size() + 1;
3657 } else {
3658 IBlock = getBlockForValue(V: I);
3659 VDUse.LocalNum = InstrToDFSNum(V: I);
3660 }
3661
3662 // Skip uses in unreachable blocks, as we're going
3663 // to delete them.
3664 if (!ReachableBlocks.contains(Ptr: IBlock))
3665 continue;
3666
3667 DomTreeNode *DomNode = DT->getNode(BB: IBlock);
3668 VDUse.DFSIn = DomNode->getDFSNumIn();
3669 VDUse.DFSOut = DomNode->getDFSNumOut();
3670 VDUse.U = &U;
3671 ++UseCount;
3672 DFSOrderedSet.emplace_back(Args&: VDUse);
3673 }
3674 }
3675
3676 // If there are no uses, it's probably dead (but it may have side-effects,
3677 // so not definitely dead. Otherwise, store the number of uses so we can
3678 // track if it becomes dead later).
3679 if (UseCount == 0)
3680 ProbablyDead.insert(Ptr: Def);
3681 else
3682 UseCounts[Def] = UseCount;
3683 }
3684}
3685
3686// This function converts the set of members for a congruence class from values,
3687// to the set of defs for loads and stores, with associated DFS info.
3688void NewGVN::convertClassToLoadsAndStores(
3689 const CongruenceClass &Dense,
3690 SmallVectorImpl<ValueDFS> &LoadsAndStores) const {
3691 for (auto *D : Dense) {
3692 if (!isa<LoadInst>(Val: D) && !isa<StoreInst>(Val: D))
3693 continue;
3694
3695 BasicBlock *BB = getBlockForValue(V: D);
3696 ValueDFS VD;
3697 DomTreeNode *DomNode = DT->getNode(BB);
3698 VD.DFSIn = DomNode->getDFSNumIn();
3699 VD.DFSOut = DomNode->getDFSNumOut();
3700 VD.Def.setPointer(D);
3701
3702 // If it's an instruction, use the real local dfs number.
3703 if (auto *I = dyn_cast<Instruction>(Val: D))
3704 VD.LocalNum = InstrToDFSNum(V: I);
3705 else
3706 llvm_unreachable("Should have been an instruction");
3707
3708 LoadsAndStores.emplace_back(Args&: VD);
3709 }
3710}
3711
3712static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
3713 patchReplacementInstruction(I, Repl);
3714 I->replaceAllUsesWith(V: Repl);
3715}
3716
3717void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
3718 LLVM_DEBUG(dbgs() << " BasicBlock Dead:" << *BB);
3719 ++NumGVNBlocksDeleted;
3720
3721 // Delete the instructions backwards, as it has a reduced likelihood of having
3722 // to update as many def-use and use-def chains. Start after the terminator.
3723 auto StartPoint = BB->rbegin();
3724 ++StartPoint;
3725 // Note that we explicitly recalculate BB->rend() on each iteration,
3726 // as it may change when we remove the first instruction.
3727 for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
3728 Instruction &Inst = *I++;
3729 if (!Inst.use_empty())
3730 Inst.replaceAllUsesWith(V: PoisonValue::get(T: Inst.getType()));
3731 if (isa<LandingPadInst>(Val: Inst))
3732 continue;
3733 salvageKnowledge(I: &Inst, AC);
3734
3735 Inst.eraseFromParent();
3736 ++NumGVNInstrDeleted;
3737 }
3738 // Now insert something that simplifycfg will turn into an unreachable.
3739 Type *Int8Ty = Type::getInt8Ty(C&: BB->getContext());
3740 new StoreInst(
3741 PoisonValue::get(T: Int8Ty),
3742 Constant::getNullValue(Ty: PointerType::getUnqual(C&: BB->getContext())),
3743 BB->getTerminator()->getIterator());
3744}
3745
3746void NewGVN::markInstructionForDeletion(Instruction *I) {
3747 LLVM_DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
3748 InstructionsToErase.insert(Ptr: I);
3749}
3750
3751void NewGVN::replaceInstruction(Instruction *I, Value *V) {
3752 LLVM_DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
3753 patchAndReplaceAllUsesWith(I, Repl: V);
3754 // We save the actual erasing to avoid invalidating memory
3755 // dependencies until we are done with everything.
3756 markInstructionForDeletion(I);
3757}
3758
3759namespace {
3760
3761// This is a stack that contains both the value and dfs info of where
3762// that value is valid.
3763class ValueDFSStack {
3764public:
3765 Value *back() const { return ValueStack.back(); }
3766 std::pair<int, int> dfs_back() const { return DFSStack.back(); }
3767
3768 void push_back(Value *V, int DFSIn, int DFSOut) {
3769 ValueStack.emplace_back(Args&: V);
3770 DFSStack.emplace_back(Args&: DFSIn, Args&: DFSOut);
3771 }
3772
3773 bool empty() const { return DFSStack.empty(); }
3774
3775 bool isInScope(int DFSIn, int DFSOut) const {
3776 if (empty())
3777 return false;
3778 return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
3779 }
3780
3781 void popUntilDFSScope(int DFSIn, int DFSOut) {
3782
3783 // These two should always be in sync at this point.
3784 assert(ValueStack.size() == DFSStack.size() &&
3785 "Mismatch between ValueStack and DFSStack");
3786 while (
3787 !DFSStack.empty() &&
3788 !(DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second)) {
3789 DFSStack.pop_back();
3790 ValueStack.pop_back();
3791 }
3792 }
3793
3794private:
3795 SmallVector<Value *, 8> ValueStack;
3796 SmallVector<std::pair<int, int>, 8> DFSStack;
3797};
3798
3799} // end anonymous namespace
3800
3801// Given an expression, get the congruence class for it.
3802CongruenceClass *NewGVN::getClassForExpression(const Expression *E) const {
3803 if (auto *VE = dyn_cast<VariableExpression>(Val: E))
3804 return ValueToClass.lookup(Val: VE->getVariableValue());
3805 else if (isa<DeadExpression>(Val: E))
3806 return TOPClass;
3807 return ExpressionToClass.lookup(Val: E);
3808}
3809
3810// Given a value and a basic block we are trying to see if it is available in,
3811// see if the value has a leader available in that block.
3812Value *NewGVN::findPHIOfOpsLeader(const Expression *E,
3813 const Instruction *OrigInst,
3814 const BasicBlock *BB) const {
3815 // It would already be constant if we could make it constant
3816 if (auto *CE = dyn_cast<ConstantExpression>(Val: E))
3817 return CE->getConstantValue();
3818 if (auto *VE = dyn_cast<VariableExpression>(Val: E)) {
3819 auto *V = VE->getVariableValue();
3820 if (alwaysAvailable(V) || DT->dominates(A: getBlockForValue(V), B: BB))
3821 return VE->getVariableValue();
3822 }
3823
3824 auto *CC = getClassForExpression(E);
3825 if (!CC)
3826 return nullptr;
3827 if (alwaysAvailable(V: CC->getLeader()))
3828 return CC->getLeader();
3829
3830 for (auto *Member : *CC) {
3831 auto *MemberInst = dyn_cast<Instruction>(Val: Member);
3832 if (MemberInst == OrigInst)
3833 continue;
3834 // Anything that isn't an instruction is always available.
3835 if (!MemberInst)
3836 return Member;
3837 if (DT->dominates(A: getBlockForValue(V: MemberInst), B: BB))
3838 return Member;
3839 }
3840 return nullptr;
3841}
3842
3843bool NewGVN::eliminateInstructions(Function &F) {
3844 // This is a non-standard eliminator. The normal way to eliminate is
3845 // to walk the dominator tree in order, keeping track of available
3846 // values, and eliminating them. However, this is mildly
3847 // pointless. It requires doing lookups on every instruction,
3848 // regardless of whether we will ever eliminate it. For
3849 // instructions part of most singleton congruence classes, we know we
3850 // will never eliminate them.
3851
3852 // Instead, this eliminator looks at the congruence classes directly, sorts
3853 // them into a DFS ordering of the dominator tree, and then we just
3854 // perform elimination straight on the sets by walking the congruence
3855 // class member uses in order, and eliminate the ones dominated by the
3856 // last member. This is worst case O(E log E) where E = number of
3857 // instructions in a single congruence class. In theory, this is all
3858 // instructions. In practice, it is much faster, as most instructions are
3859 // either in singleton congruence classes or can't possibly be eliminated
3860 // anyway (if there are no overlapping DFS ranges in class).
3861 // When we find something not dominated, it becomes the new leader
3862 // for elimination purposes.
3863 // TODO: If we wanted to be faster, We could remove any members with no
3864 // overlapping ranges while sorting, as we will never eliminate anything
3865 // with those members, as they don't dominate anything else in our set.
3866
3867 bool AnythingReplaced = false;
3868
3869 // Since we are going to walk the domtree anyway, and we can't guarantee the
3870 // DFS numbers are updated, we compute some ourselves.
3871 DT->updateDFSNumbers();
3872
3873 // Go through all of our phi nodes, and kill the arguments associated with
3874 // unreachable edges.
3875 auto ReplaceUnreachablePHIArgs = [&](PHINode *PHI, BasicBlock *BB) {
3876 for (auto &Operand : PHI->incoming_values())
3877 if (!ReachableEdges.count(V: {PHI->getIncomingBlock(U: Operand), BB})) {
3878 LLVM_DEBUG(dbgs() << "Replacing incoming value of " << PHI
3879 << " for block "
3880 << getBlockName(PHI->getIncomingBlock(Operand))
3881 << " with poison due to it being unreachable\n");
3882 Operand.set(PoisonValue::get(T: PHI->getType()));
3883 }
3884 };
3885 // Replace unreachable phi arguments.
3886 // At this point, RevisitOnReachabilityChange only contains:
3887 //
3888 // 1. PHIs
3889 // 2. Temporaries that will convert to PHIs
3890 // 3. Operations that are affected by an unreachable edge but do not fit into
3891 // 1 or 2 (rare).
3892 // So it is a slight overshoot of what we want. We could make it exact by
3893 // using two SparseBitVectors per block.
3894 DenseMap<const BasicBlock *, unsigned> ReachablePredCount;
3895 for (auto &KV : ReachableEdges)
3896 ReachablePredCount[KV.getEnd()]++;
3897 for (auto &BBPair : RevisitOnReachabilityChange) {
3898 for (auto InstNum : BBPair.second) {
3899 auto *Inst = InstrFromDFSNum(DFSNum: InstNum);
3900 auto *PHI = dyn_cast<PHINode>(Val: Inst);
3901 PHI = PHI ? PHI : dyn_cast_or_null<PHINode>(Val: RealToTemp.lookup(Val: Inst));
3902 if (!PHI)
3903 continue;
3904 auto *BB = BBPair.first;
3905 if (ReachablePredCount.lookup(Val: BB) != PHI->getNumIncomingValues())
3906 ReplaceUnreachablePHIArgs(PHI, BB);
3907 }
3908 }
3909
3910 // Map to store the use counts
3911 DenseMap<const Value *, unsigned int> UseCounts;
3912 for (auto *CC : reverse(C&: CongruenceClasses)) {
3913 LLVM_DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID()
3914 << "\n");
3915 // Track the equivalent store info so we can decide whether to try
3916 // dead store elimination.
3917 SmallVector<ValueDFS, 8> PossibleDeadStores;
3918 SmallPtrSet<Instruction *, 8> ProbablyDead;
3919 if (CC->isDead() || CC->empty())
3920 continue;
3921 // Everything still in the TOP class is unreachable or dead.
3922 if (CC == TOPClass) {
3923 for (auto *M : *CC) {
3924 auto *VTE = ValueToExpression.lookup(Val: M);
3925 if (VTE && isa<DeadExpression>(Val: VTE))
3926 markInstructionForDeletion(I: cast<Instruction>(Val: M));
3927 assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||
3928 InstructionsToErase.count(cast<Instruction>(M))) &&
3929 "Everything in TOP should be unreachable or dead at this "
3930 "point");
3931 }
3932 continue;
3933 }
3934
3935 assert(CC->getLeader() && "We should have had a leader");
3936 // If this is a leader that is always available, and it's a
3937 // constant or has no equivalences, just replace everything with
3938 // it. We then update the congruence class with whatever members
3939 // are left.
3940 Value *Leader =
3941 CC->getStoredValue() ? CC->getStoredValue() : CC->getLeader();
3942 if (alwaysAvailable(V: Leader)) {
3943 CongruenceClass::MemberSet MembersLeft;
3944 for (auto *M : *CC) {
3945 Value *Member = M;
3946 // Void things have no uses we can replace.
3947 if (Member == Leader || !isa<Instruction>(Val: Member) ||
3948 Member->getType()->isVoidTy()) {
3949 MembersLeft.insert(Ptr: Member);
3950 continue;
3951 }
3952
3953 LLVM_DEBUG(dbgs() << "Found replacement " << *(Leader) << " for "
3954 << *Member << "\n");
3955 auto *I = cast<Instruction>(Val: Member);
3956 assert(Leader != I && "About to accidentally remove our leader");
3957 replaceInstruction(I, V: Leader);
3958 AnythingReplaced = true;
3959 }
3960 CC->swap(Other&: MembersLeft);
3961 } else {
3962 // If this is a singleton, we can skip it.
3963 if (CC->size() != 1 || RealToTemp.count(Val: Leader)) {
3964 // This is a stack because equality replacement/etc may place
3965 // constants in the middle of the member list, and we want to use
3966 // those constant values in preference to the current leader, over
3967 // the scope of those constants.
3968 ValueDFSStack EliminationStack;
3969
3970 // Convert the members to DFS ordered sets and then merge them.
3971 SmallVector<ValueDFS, 8> DFSOrderedSet;
3972 convertClassToDFSOrdered(Dense: *CC, DFSOrderedSet, UseCounts, ProbablyDead);
3973
3974 // Sort the whole thing.
3975 llvm::sort(C&: DFSOrderedSet);
3976 for (auto &VD : DFSOrderedSet) {
3977 int MemberDFSIn = VD.DFSIn;
3978 int MemberDFSOut = VD.DFSOut;
3979 Value *Def = VD.Def.getPointer();
3980 bool FromStore = VD.Def.getInt();
3981 Use *U = VD.U;
3982 // We ignore void things because we can't get a value from them.
3983 if (Def && Def->getType()->isVoidTy())
3984 continue;
3985 auto *DefInst = dyn_cast_or_null<Instruction>(Val: Def);
3986 if (DefInst && AllTempInstructions.count(V: DefInst)) {
3987 auto *PN = cast<PHINode>(Val: DefInst);
3988
3989 // If this is a value phi and that's the expression we used, insert
3990 // it into the program
3991 // remove from temp instruction list.
3992 AllTempInstructions.erase(V: PN);
3993 auto *DefBlock = getBlockForValue(V: Def);
3994 LLVM_DEBUG(dbgs() << "Inserting fully real phi of ops" << *Def
3995 << " into block "
3996 << getBlockName(getBlockForValue(Def)) << "\n");
3997 PN->insertBefore(InsertPos: DefBlock->begin());
3998 Def = PN;
3999 NumGVNPHIOfOpsEliminations++;
4000 }
4001
4002 if (EliminationStack.empty()) {
4003 LLVM_DEBUG(dbgs() << "Elimination Stack is empty\n");
4004 } else {
4005 LLVM_DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
4006 << EliminationStack.dfs_back().first << ","
4007 << EliminationStack.dfs_back().second << ")\n");
4008 }
4009
4010 LLVM_DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
4011 << MemberDFSOut << ")\n");
4012 // First, we see if we are out of scope or empty. If so,
4013 // and there equivalences, we try to replace the top of
4014 // stack with equivalences (if it's on the stack, it must
4015 // not have been eliminated yet).
4016 // Then we synchronize to our current scope, by
4017 // popping until we are back within a DFS scope that
4018 // dominates the current member.
4019 // Then, what happens depends on a few factors
4020 // If the stack is now empty, we need to push
4021 // If we have a constant or a local equivalence we want to
4022 // start using, we also push.
4023 // Otherwise, we walk along, processing members who are
4024 // dominated by this scope, and eliminate them.
4025 bool ShouldPush = Def && EliminationStack.empty();
4026 bool OutOfScope =
4027 !EliminationStack.isInScope(DFSIn: MemberDFSIn, DFSOut: MemberDFSOut);
4028
4029 if (OutOfScope || ShouldPush) {
4030 // Sync to our current scope.
4031 EliminationStack.popUntilDFSScope(DFSIn: MemberDFSIn, DFSOut: MemberDFSOut);
4032 bool ShouldPush = Def && EliminationStack.empty();
4033 if (ShouldPush) {
4034 EliminationStack.push_back(V: Def, DFSIn: MemberDFSIn, DFSOut: MemberDFSOut);
4035 }
4036 }
4037
4038 // Skip the Def's, we only want to eliminate on their uses. But mark
4039 // dominated defs as dead.
4040 if (Def) {
4041 // For anything in this case, what and how we value number
4042 // guarantees that any side-effects that would have occurred (ie
4043 // throwing, etc) can be proven to either still occur (because it's
4044 // dominated by something that has the same side-effects), or never
4045 // occur. Otherwise, we would not have been able to prove it value
4046 // equivalent to something else. For these things, we can just mark
4047 // it all dead. Note that this is different from the "ProbablyDead"
4048 // set, which may not be dominated by anything, and thus, are only
4049 // easy to prove dead if they are also side-effect free. Note that
4050 // because stores are put in terms of the stored value, we skip
4051 // stored values here. If the stored value is really dead, it will
4052 // still be marked for deletion when we process it in its own class.
4053 auto *DefI = dyn_cast<Instruction>(Val: Def);
4054 if (!EliminationStack.empty() && DefI && !FromStore) {
4055 Value *DominatingLeader = EliminationStack.back();
4056 if (DominatingLeader != Def) {
4057 // Even if the instruction is removed, we still need to update
4058 // flags/metadata due to downstreams users of the leader.
4059 patchReplacementInstruction(I: DefI, Repl: DominatingLeader);
4060
4061 SmallVector<DbgVariableRecord *> DVRUsers;
4062 findDbgUsers(V: DefI, DbgVariableRecords&: DVRUsers);
4063
4064 for (auto *DVR : DVRUsers)
4065 DVR->replaceVariableLocationOp(OldValue: DefI, NewValue: DominatingLeader);
4066
4067 markInstructionForDeletion(I: DefI);
4068 }
4069 }
4070 continue;
4071 }
4072 // At this point, we know it is a Use we are trying to possibly
4073 // replace.
4074
4075 assert(isa<Instruction>(U->get()) &&
4076 "Current def should have been an instruction");
4077 assert(isa<Instruction>(U->getUser()) &&
4078 "Current user should have been an instruction");
4079
4080 // If the thing we are replacing into is already marked to be dead,
4081 // this use is dead. Note that this is true regardless of whether
4082 // we have anything dominating the use or not. We do this here
4083 // because we are already walking all the uses anyway.
4084 Instruction *InstUse = cast<Instruction>(Val: U->getUser());
4085 if (InstructionsToErase.count(Ptr: InstUse)) {
4086 auto &UseCount = UseCounts[U->get()];
4087 if (--UseCount == 0) {
4088 ProbablyDead.insert(Ptr: cast<Instruction>(Val: U->get()));
4089 }
4090 }
4091
4092 // If we get to this point, and the stack is empty we must have a use
4093 // with nothing we can use to eliminate this use, so just skip it.
4094 if (EliminationStack.empty())
4095 continue;
4096
4097 Value *DominatingLeader = EliminationStack.back();
4098
4099 Instruction *SSACopy = nullptr;
4100 if (auto *BC = dyn_cast<BitCastInst>(Val: DominatingLeader)) {
4101 if (BC->getType() == BC->getOperand(i_nocapture: 0)->getType() &&
4102 PredInfo->getPredicateInfoFor(V: DominatingLeader)) {
4103 SSACopy = BC;
4104 DominatingLeader = BC->getOperand(i_nocapture: 0);
4105 }
4106 }
4107
4108 // Don't replace our existing users with ourselves.
4109 if (U->get() == DominatingLeader)
4110 continue;
4111
4112 // If we replaced something in an instruction, handle the patching of
4113 // metadata. Skip this if we are replacing predicateinfo with its
4114 // original operand, as we already know we can just drop it.
4115 auto *ReplacedInst = cast<Instruction>(Val: U->get());
4116 auto *PI = PredInfo->getPredicateInfoFor(V: ReplacedInst);
4117 if (!PI || DominatingLeader != PI->OriginalOp)
4118 patchReplacementInstruction(I: ReplacedInst, Repl: DominatingLeader);
4119
4120 LLVM_DEBUG(dbgs()
4121 << "Found replacement " << *DominatingLeader << " for "
4122 << *U->get() << " in " << *(U->getUser()) << "\n");
4123 U->set(DominatingLeader);
4124 // This is now a use of the dominating leader, which means if the
4125 // dominating leader was dead, it's now live!
4126 auto &LeaderUseCount = UseCounts[DominatingLeader];
4127 // It's about to be alive again.
4128 if (LeaderUseCount == 0 && isa<Instruction>(Val: DominatingLeader))
4129 ProbablyDead.erase(Ptr: cast<Instruction>(Val: DominatingLeader));
4130 // For copy instructions, we use their operand as a leader,
4131 // which means we remove a user of the copy and it may become dead.
4132 if (SSACopy) {
4133 auto It = UseCounts.find(Val: SSACopy);
4134 if (It != UseCounts.end()) {
4135 unsigned &IIUseCount = It->second;
4136 if (--IIUseCount == 0)
4137 ProbablyDead.insert(Ptr: SSACopy);
4138 }
4139 }
4140 ++LeaderUseCount;
4141 AnythingReplaced = true;
4142 }
4143 }
4144 }
4145
4146 // At this point, anything still in the ProbablyDead set is actually dead if
4147 // would be trivially dead.
4148 for (auto *I : ProbablyDead)
4149 if (wouldInstructionBeTriviallyDead(I))
4150 markInstructionForDeletion(I);
4151
4152 // Cleanup the congruence class.
4153 CongruenceClass::MemberSet MembersLeft;
4154 for (auto *Member : *CC)
4155 if (!isa<Instruction>(Val: Member) ||
4156 !InstructionsToErase.count(Ptr: cast<Instruction>(Val: Member)))
4157 MembersLeft.insert(Ptr: Member);
4158 CC->swap(Other&: MembersLeft);
4159
4160 // If we have possible dead stores to look at, try to eliminate them.
4161 if (CC->getStoreCount() > 0) {
4162 convertClassToLoadsAndStores(Dense: *CC, LoadsAndStores&: PossibleDeadStores);
4163 llvm::sort(C&: PossibleDeadStores);
4164 ValueDFSStack EliminationStack;
4165 for (auto &VD : PossibleDeadStores) {
4166 int MemberDFSIn = VD.DFSIn;
4167 int MemberDFSOut = VD.DFSOut;
4168 Instruction *Member = cast<Instruction>(Val: VD.Def.getPointer());
4169 if (EliminationStack.empty() ||
4170 !EliminationStack.isInScope(DFSIn: MemberDFSIn, DFSOut: MemberDFSOut)) {
4171 // Sync to our current scope.
4172 EliminationStack.popUntilDFSScope(DFSIn: MemberDFSIn, DFSOut: MemberDFSOut);
4173 if (EliminationStack.empty()) {
4174 EliminationStack.push_back(V: Member, DFSIn: MemberDFSIn, DFSOut: MemberDFSOut);
4175 continue;
4176 }
4177 }
4178 // We already did load elimination, so nothing to do here.
4179 if (isa<LoadInst>(Val: Member))
4180 continue;
4181 assert(!EliminationStack.empty());
4182 Instruction *Leader = cast<Instruction>(Val: EliminationStack.back());
4183 (void)Leader;
4184 assert(DT->dominates(Leader->getParent(), Member->getParent()));
4185 // Member is dominater by Leader, and thus dead
4186 LLVM_DEBUG(dbgs() << "Marking dead store " << *Member
4187 << " that is dominated by " << *Leader << "\n");
4188 markInstructionForDeletion(I: Member);
4189 CC->erase(M: Member);
4190 ++NumGVNDeadStores;
4191 }
4192 }
4193 }
4194 return AnythingReplaced;
4195}
4196
4197// This function provides global ranking of operations so that we can place them
4198// in a canonical order. Note that rank alone is not necessarily enough for a
4199// complete ordering, as constants all have the same rank. However, generally,
4200// we will simplify an operation with all constants so that it doesn't matter
4201// what order they appear in.
4202unsigned int NewGVN::getRank(const Value *V) const {
4203 // Prefer constants to undef to anything else
4204 // Undef is a constant, have to check it first.
4205 // Prefer poison to undef as it's less defined.
4206 // Prefer smaller constants to constantexprs
4207 // Note that the order here matters because of class inheritance
4208 if (isa<ConstantExpr>(Val: V))
4209 return 3;
4210 if (isa<PoisonValue>(Val: V))
4211 return 1;
4212 if (isa<UndefValue>(Val: V))
4213 return 2;
4214 if (isa<Constant>(Val: V))
4215 return 0;
4216 if (auto *A = dyn_cast<Argument>(Val: V))
4217 return 4 + A->getArgNo();
4218
4219 // Need to shift the instruction DFS by number of arguments + 5 to account for
4220 // the constant and argument ranking above.
4221 unsigned Result = InstrToDFSNum(V);
4222 if (Result > 0)
4223 return 5 + NumFuncArgs + Result;
4224 // Unreachable or something else, just return a really large number.
4225 return ~0;
4226}
4227
4228// This is a function that says whether two commutative operations should
4229// have their order swapped when canonicalizing.
4230bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
4231 // Because we only care about a total ordering, and don't rewrite expressions
4232 // in this order, we order by rank, which will give a strict weak ordering to
4233 // everything but constants, and then we order by pointer address.
4234 return std::make_pair(x: getRank(V: A), y&: A) > std::make_pair(x: getRank(V: B), y&: B);
4235}
4236
4237bool NewGVN::shouldSwapOperandsForPredicate(const Value *A, const Value *B,
4238 const BitCastInst *I) const {
4239 if (shouldSwapOperands(A, B)) {
4240 PredicateSwapChoice[I] = B;
4241 return true;
4242 }
4243
4244 auto LookupResult = PredicateSwapChoice.find(Val: I);
4245 if (LookupResult != PredicateSwapChoice.end()) {
4246 auto *SeenPredicate = LookupResult->second;
4247 if (SeenPredicate) {
4248 // We previously decided to swap B to the left. Keep that choice.
4249 if (SeenPredicate == B)
4250 return true;
4251 else
4252 LookupResult->second = nullptr;
4253 }
4254 }
4255 return false;
4256}
4257
4258PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
4259 // Apparently the order in which we get these results matter for
4260 // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
4261 // the same order here, just in case.
4262 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
4263 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
4264 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
4265 auto &AA = AM.getResult<AAManager>(IR&: F);
4266 auto &MSSA = AM.getResult<MemorySSAAnalysis>(IR&: F).getMSSA();
4267 bool Changed =
4268 NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getDataLayout())
4269 .runGVN();
4270 if (!Changed)
4271 return PreservedAnalyses::all();
4272 PreservedAnalyses PA;
4273 PA.preserve<DominatorTreeAnalysis>();
4274 return PA;
4275}
4276