1//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This transformation implements the well known scalar replacement of
10/// aggregates transformation. It tries to identify promotable elements of an
11/// aggregate alloca, and promote them to registers. It will also try to
12/// convert uses of an element (or set of elements) of an alloca into a vector
13/// or bitfield-style integer scalar if appropriate.
14///
15/// It works to do this with minimal slicing of the alloca so that regions
16/// which are merely transferred in and out of external memory remain unchanged
17/// and are not decomposed to scalar code.
18///
19/// Because this also performs alloca promotion, it can be thought of as also
20/// serving the purpose of SSA formation. The algorithm iterates on the
21/// function until all opportunities for promotion have been realized.
22///
23//===----------------------------------------------------------------------===//
24
25#include "llvm/Transforms/Scalar/SROA.h"
26#include "ScalarOptions.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/ArrayRef.h"
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/MapVector.h"
31#include "llvm/ADT/PointerIntPair.h"
32#include "llvm/ADT/STLExtras.h"
33#include "llvm/ADT/SetVector.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/SparseBitVector.h"
37#include "llvm/ADT/Statistic.h"
38#include "llvm/ADT/StringRef.h"
39#include "llvm/ADT/Twine.h"
40#include "llvm/ADT/iterator.h"
41#include "llvm/ADT/iterator_range.h"
42#include "llvm/Analysis/AssumptionCache.h"
43#include "llvm/Analysis/DomTreeUpdater.h"
44#include "llvm/Analysis/GlobalsModRef.h"
45#include "llvm/Analysis/Loads.h"
46#include "llvm/Analysis/PtrUseVisitor.h"
47#include "llvm/Analysis/ValueTracking.h"
48#include "llvm/Analysis/VectorUtils.h"
49#include "llvm/IR/BasicBlock.h"
50#include "llvm/IR/Constant.h"
51#include "llvm/IR/ConstantFolder.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DIBuilder.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugInfo.h"
56#include "llvm/IR/DebugInfoMetadata.h"
57#include "llvm/IR/DerivedTypes.h"
58#include "llvm/IR/Dominators.h"
59#include "llvm/IR/Function.h"
60#include "llvm/IR/GlobalAlias.h"
61#include "llvm/IR/IRBuilder.h"
62#include "llvm/IR/InstVisitor.h"
63#include "llvm/IR/Instruction.h"
64#include "llvm/IR/Instructions.h"
65#include "llvm/IR/IntrinsicInst.h"
66#include "llvm/IR/LLVMContext.h"
67#include "llvm/IR/Metadata.h"
68#include "llvm/IR/Module.h"
69#include "llvm/IR/Operator.h"
70#include "llvm/IR/PassManager.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/User.h"
74#include "llvm/IR/Value.h"
75#include "llvm/IR/ValueHandle.h"
76#include "llvm/InitializePasses.h"
77#include "llvm/Pass.h"
78#include "llvm/Support/Casting.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/Debug.h"
81#include "llvm/Support/ErrorHandling.h"
82#include "llvm/Support/raw_ostream.h"
83#include "llvm/Transforms/Scalar.h"
84#include "llvm/Transforms/Utils/BasicBlockUtils.h"
85#include "llvm/Transforms/Utils/Local.h"
86#include "llvm/Transforms/Utils/PromoteMemToReg.h"
87#include "llvm/Transforms/Utils/SSAUpdater.h"
88#include <algorithm>
89#include <cassert>
90#include <cstddef>
91#include <cstdint>
92#include <cstring>
93#include <iterator>
94#include <string>
95#include <tuple>
96#include <utility>
97#include <variant>
98#include <vector>
99
100using namespace llvm;
101
102#define DEBUG_TYPE "sroa"
103
104STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112STATISTIC(NumLoadsPredicated,
113 "Number of loads rewritten into predicated loads to allow promotion");
114STATISTIC(
115 NumStoresPredicated,
116 "Number of stores rewritten into predicated stores to allow promotion");
117STATISTIC(NumDeleted, "Number of instructions deleted");
118STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120namespace {
121
122class AllocaSliceRewriter;
123class AllocaSlices;
124class Partition;
125class Slice;
126
127class SelectHandSpeculativity {
128 unsigned char Storage = 0; // None are speculatable by default.
129 using TrueVal = Bitfield::Element<bool, 0, 1>; // Low 0'th bit.
130 using FalseVal = Bitfield::Element<bool, 1, 1>; // Low 1'th bit.
131public:
132 SelectHandSpeculativity() = default;
133 SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
134 bool isSpeculatable(bool isTrueVal) const;
135 bool areAllSpeculatable() const;
136 bool areAnySpeculatable() const;
137 bool areNoneSpeculatable() const;
138 // For interop as int half of PointerIntPair.
139 explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
140 explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
141};
142static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
143
144using PossiblySpeculatableLoad =
145 PointerIntPair<LoadInst *, 2, SelectHandSpeculativity>;
146using UnspeculatableStore = StoreInst *;
147using RewriteableMemOp =
148 std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
149using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
150using InstructionSliceMap =
151 SmallDenseMap<Instruction *, SmallPtrSet<Slice *, 8>, 8>;
152
153/// An optimization pass providing Scalar Replacement of Aggregates.
154///
155/// This pass takes allocations which can be completely analyzed (that is, they
156/// don't escape) and tries to turn them into scalar SSA values. There are
157/// a few steps to this process.
158///
159/// 1) It takes allocations of aggregates and analyzes the ways in which they
160/// are used to try to split them into smaller allocations, ideally of
161/// a single scalar data type. It will split up memcpy and memset accesses
162/// as necessary and try to isolate individual scalar accesses.
163/// 2) It will transform accesses into forms which are suitable for SSA value
164/// promotion. This can be replacing a memset with a scalar store of an
165/// integer value, or it can involve speculating operations on a PHI or
166/// select to be a PHI or select of the results.
167/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
168/// onto insert and extract operations on a vector value, and convert them to
169/// this form. By doing so, it will enable promotion of vector aggregates to
170/// SSA vector values.
171class SROA {
172 LLVMContext *const C;
173 DomTreeUpdater *const DTU;
174 AssumptionCache *const AC;
175 const bool PreserveCFG;
176 const bool AggregateToVector;
177
178 /// Worklist of alloca instructions to simplify.
179 ///
180 /// Each alloca in the function is added to this. Each new alloca formed gets
181 /// added to it as well to recursively simplify unless that alloca can be
182 /// directly promoted. Finally, each time we rewrite a use of an alloca other
183 /// the one being actively rewritten, we add it back onto the list if not
184 /// already present to ensure it is re-visited.
185 SmallSetVector<AllocaInst *, 16> Worklist;
186
187 /// A collection of instructions to delete.
188 /// We try to batch deletions to simplify code and make things a bit more
189 /// efficient. We also make sure there is no dangling pointers.
190 SmallVector<WeakVH, 8> DeadInsts;
191
192 /// Post-promotion worklist.
193 ///
194 /// Sometimes we discover an alloca which has a high probability of becoming
195 /// viable for SROA after a round of promotion takes place. In those cases,
196 /// the alloca is enqueued here for re-processing.
197 ///
198 /// Note that we have to be very careful to clear allocas out of this list in
199 /// the event they are deleted.
200 SmallSetVector<AllocaInst *, 16> PostPromotionWorklist;
201
202 /// A collection of alloca instructions we can directly promote.
203 SetVector<AllocaInst *, SmallVector<AllocaInst *>,
204 SmallPtrSet<AllocaInst *, 16>, 16>
205 PromotableAllocas;
206
207 /// A worklist of PHIs to speculate prior to promoting allocas.
208 ///
209 /// All of these PHIs have been checked for the safety of speculation and by
210 /// being speculated will allow promoting allocas currently in the promotable
211 /// queue.
212 SmallSetVector<PHINode *, 8> SpeculatablePHIs;
213
214 /// A worklist of PHIs whose stores should be predicated onto incoming edges.
215 ///
216 /// All of these PHIs have been checked so that rewriting their stores will
217 /// allow the corresponding allocas to be promoted. Edges which require
218 /// splitting are only present when CFG modification is allowed.
219 SmallSetVector<PHINode *, 8> PHIsWithStoreToRewrite;
220
221 /// A worklist of select instructions to rewrite prior to promoting
222 /// allocas.
223 SmallMapVector<SelectInst *, RewriteableMemOps, 8> SelectsToRewrite;
224
225 /// Check whether the loads and stores of the select can be speculatively
226 /// executed or predicated (if \p PreserveCFG is false).
227 ///
228 /// Speculation converts a load like
229 ///
230 /// %P = select i1 %cond, ptr %Alloca, ptr %Other
231 /// %V = load <type>, ptr %P
232 ///
233 /// to:
234 ///
235 /// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
236 /// %V2 = load <type>, ptr %Other
237 /// %V = select i1 %cond, <type> %V1, <type> %V2
238 ///
239 /// For stores and some kinds of loads that can't be speculated, we need to
240 /// predicate. Predication changes the CFG, so it requires \p PreserveCFG to
241 /// be false. Predication converts a store like:
242 ///
243 /// block:
244 /// %P = select i1 %cond, ptr %Alloca, ptr %Other
245 /// store <type> %Value, ptr %P
246 ///
247 /// to:
248 ///
249 /// block:
250 /// br i1 %cond, label %block.then, label %block.else
251 ///
252 /// block.then:
253 /// store <type> %Value, ptr %Alloca
254 /// br label %block.cont
255 ///
256 /// block.else:
257 /// store <type> %Value, ptr %Other
258 /// br label %block.cont
259 ///
260 /// Returns `std::nullopt` when
261 /// - the select has users other than loads or stores, or
262 /// - at least one of the loads and stores is volatile, or
263 /// - the loads and stores can't be speculated and can't be predicated because
264 /// we're not allowed to modify the CFG. (\p PreserveCFG is true)
265 ///
266 /// Otherwise the stores and loads in the returned vector can either be
267 /// speculated or predicated.
268 static std::optional<RewriteableMemOps>
269 isSafeSelectToSpeculateOrPredicate(SelectInst &SI, bool PreserveCFG);
270
271public:
272 SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
273 SROAOptions Options)
274 : C(C), DTU(DTU), AC(AC),
275 PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
276 AggregateToVector(Options.AggregateToVector) {}
277
278 /// Main run method used by both the SROAPass and by the legacy pass.
279 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
280
281private:
282 friend class AllocaSliceRewriter;
283
284 bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
285 std::pair<AllocaInst *, uint64_t>
286 rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
287 bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
288 bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
289 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runOnAlloca(AllocaInst &AI);
290 void clobberUse(Use &U);
291 bool deleteDeadInstructions(AllocaInst *AI,
292 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
293 bool promoteAllocas();
294};
295
296} // end anonymous namespace
297
298/// Calculate the fragment of a variable to use when slicing a store
299/// based on the slice dimensions, existing fragment, and base storage
300/// fragment.
301/// Results:
302/// UseFrag - Use Target as the new fragment.
303/// UseNoFrag - The new slice already covers the whole variable.
304/// Skip - The new alloca slice doesn't include this variable.
305/// FIXME: Can we use calculateFragmentIntersect instead?
306namespace {
307enum FragCalcResult { UseFrag, UseNoFrag, Skip };
308}
309static FragCalcResult
310calculateFragment(DILocalVariable *Variable,
311 uint64_t NewStorageSliceOffsetInBits,
312 uint64_t NewStorageSliceSizeInBits,
313 std::optional<DIExpression::FragmentInfo> StorageFragment,
314 std::optional<DIExpression::FragmentInfo> CurrentFragment,
315 DIExpression::FragmentInfo &Target) {
316 // If the base storage describes part of the variable apply the offset and
317 // the size constraint.
318 if (StorageFragment) {
319 Target.SizeInBits =
320 std::min(a: NewStorageSliceSizeInBits, b: StorageFragment->SizeInBits);
321 Target.OffsetInBits =
322 NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits;
323 } else {
324 Target.SizeInBits = NewStorageSliceSizeInBits;
325 Target.OffsetInBits = NewStorageSliceOffsetInBits;
326 }
327
328 // If this slice extracts the entirety of an independent variable from a
329 // larger alloca, do not produce a fragment expression, as the variable is
330 // not fragmented.
331 if (!CurrentFragment) {
332 if (auto Size = Variable->getSizeInBits()) {
333 // Treat the current fragment as covering the whole variable.
334 CurrentFragment = DIExpression::FragmentInfo(*Size, 0);
335 if (Target == CurrentFragment)
336 return UseNoFrag;
337 }
338 }
339
340 // No additional work to do if there isn't a fragment already, or there is
341 // but it already exactly describes the new assignment.
342 if (!CurrentFragment || *CurrentFragment == Target)
343 return UseFrag;
344
345 // Reject the target fragment if it doesn't fit wholly within the current
346 // fragment. TODO: We could instead chop up the target to fit in the case of
347 // a partial overlap.
348 if (Target.startInBits() < CurrentFragment->startInBits() ||
349 Target.endInBits() > CurrentFragment->endInBits())
350 return Skip;
351
352 // Target fits within the current fragment, return it.
353 return UseFrag;
354}
355
356static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
357 return DebugVariable(DVR->getVariable(), std::nullopt,
358 DVR->getDebugLoc().getInlinedAt());
359}
360
361/// Find linked dbg.assign and generate a new one with the correct
362/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
363/// value component is copied from the old dbg.assign to the new.
364/// \param OldAlloca Alloca for the variable before splitting.
365/// \param IsSplit True if the store (not necessarily alloca)
366/// is being split.
367/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
368/// \param SliceSizeInBits New number of bits being written to.
369/// \param OldInst Instruction that is being split.
370/// \param Inst New instruction performing this part of the
371/// split store.
372/// \param Dest Store destination.
373/// \param Value Stored value.
374/// \param DL Datalayout.
375static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
376 uint64_t OldAllocaOffsetInBits,
377 uint64_t SliceSizeInBits, Instruction *OldInst,
378 Instruction *Inst, Value *Dest, Value *Value,
379 const DataLayout &DL) {
380 // If we want allocas to be migrated using this helper then we need to ensure
381 // that the BaseFragments map code still works. A simple solution would be
382 // to choose to always clone alloca dbg_assigns (rather than sometimes
383 // "stealing" them).
384 assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
385
386 auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(Inst: OldInst);
387 // Nothing to do if OldInst has no linked dbg.assign intrinsics.
388 if (DVRAssignMarkerRange.empty())
389 return;
390
391 LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
392 LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
393 LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
394 LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
395 << "\n");
396 LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
397 LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
398 LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
399 LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
400 if (Value)
401 LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
402
403 /// Map of aggregate variables to their fragment associated with OldAlloca.
404 DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>>
405 BaseFragments;
406 for (auto *DVR : at::getDVRAssignmentMarkers(Inst: OldAlloca))
407 BaseFragments[getAggregateVariable(DVR)] =
408 DVR->getExpression()->getFragmentInfo();
409
410 // The new inst needs a DIAssignID unique metadata tag (if OldInst has
411 // one). It shouldn't already have one: assert this assumption.
412 assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
413 DIAssignID *NewID = nullptr;
414 auto &Ctx = Inst->getContext();
415 DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
416 assert(OldAlloca->isStaticAlloca());
417
418 auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
419 LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
420 << "\n");
421 auto *Expr = DbgAssign->getExpression();
422 bool SetKillLocation = false;
423
424 if (IsSplit) {
425 std::optional<DIExpression::FragmentInfo> BaseFragment;
426 {
427 auto R = BaseFragments.find(Val: getAggregateVariable(DVR: DbgAssign));
428 if (R == BaseFragments.end())
429 return;
430 BaseFragment = R->second;
431 }
432 std::optional<DIExpression::FragmentInfo> CurrentFragment =
433 Expr->getFragmentInfo();
434 DIExpression::FragmentInfo NewFragment;
435 FragCalcResult Result = calculateFragment(
436 Variable: DbgAssign->getVariable(), NewStorageSliceOffsetInBits: OldAllocaOffsetInBits, NewStorageSliceSizeInBits: SliceSizeInBits,
437 StorageFragment: BaseFragment, CurrentFragment, Target&: NewFragment);
438
439 if (Result == Skip)
440 return;
441 if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
442 if (CurrentFragment) {
443 // Rewrite NewFragment to be relative to the existing one (this is
444 // what createFragmentExpression wants). CalculateFragment has
445 // already resolved the size for us. FIXME: Should it return the
446 // relative fragment too?
447 NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits;
448 }
449 // Add the new fragment info to the existing expression if possible.
450 if (auto E = DIExpression::createFragmentExpression(
451 Expr, OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits)) {
452 Expr = *E;
453 } else {
454 // Otherwise, add the new fragment info to an empty expression and
455 // discard the value component of this dbg.assign as the value cannot
456 // be computed with the new fragment.
457 Expr = *DIExpression::createFragmentExpression(
458 Expr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
459 OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits);
460 SetKillLocation = true;
461 }
462 }
463 }
464
465 // If we haven't created a DIAssignID ID do that now and attach it to Inst.
466 if (!NewID) {
467 NewID = DIAssignID::getDistinct(Context&: Ctx);
468 Inst->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: NewID);
469 }
470
471 DbgVariableRecord *NewAssign;
472 if (IsSplit) {
473 ::Value *NewValue = Value ? Value : DbgAssign->getValue();
474 NewAssign = cast<DbgVariableRecord>(Val: DIB.insertDbgAssign(
475 LinkedInstr: Inst, Val: NewValue, SrcVar: DbgAssign->getVariable(), ValExpr: Expr, Addr: Dest,
476 AddrExpr: DIExpression::get(Context&: Expr->getContext(), Elements: {}), DL: DbgAssign->getDebugLoc()));
477 } else {
478 // The store is not split, simply steal the existing dbg_assign.
479 NewAssign = DbgAssign;
480 NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
481 NewAssign->setAddress(Dest);
482 if (Value)
483 NewAssign->replaceVariableLocationOp(OpIdx: 0u, NewValue: Value);
484 assert(Expr == NewAssign->getExpression());
485 }
486
487 // If we've updated the value but the original dbg.assign has an arglist
488 // then kill it now - we can't use the requested new value.
489 // We can't replace the DIArgList with the new value as it'd leave
490 // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
491 // an arglist). And we can't keep the DIArgList in case the linked store
492 // is being split - in which case the DIArgList + expression may no longer
493 // be computing the correct value.
494 // This should be a very rare situation as it requires the value being
495 // stored to differ from the dbg.assign (i.e., the value has been
496 // represented differently in the debug intrinsic for some reason).
497 SetKillLocation |=
498 Value && (DbgAssign->hasArgList() ||
499 !DbgAssign->getExpression()->isSingleLocationExpression());
500 if (SetKillLocation)
501 NewAssign->setKillLocation();
502
503 // We could use more precision here at the cost of some additional (code)
504 // complexity - if the original dbg.assign was adjacent to its store, we
505 // could position this new dbg.assign adjacent to its store rather than the
506 // old dbg.assgn. That would result in interleaved dbg.assigns rather than
507 // what we get now:
508 // split store !1
509 // split store !2
510 // dbg.assign !1
511 // dbg.assign !2
512 // This (current behaviour) results results in debug assignments being
513 // noted as slightly offset (in code) from the store. In practice this
514 // should have little effect on the debugging experience due to the fact
515 // that all the split stores should get the same line number.
516 if (NewAssign != DbgAssign) {
517 NewAssign->moveBefore(MoveBefore: DbgAssign->getIterator());
518 NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
519 }
520 LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
521 };
522
523 for_each(Range&: DVRAssignMarkerRange, F: MigrateDbgAssign);
524}
525
526namespace {
527
528/// A custom IRBuilder inserter which prefixes all names, but only in
529/// Assert builds.
530class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
531 std::string Prefix;
532
533 Twine getNameWithPrefix(const Twine &Name) const {
534 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
535 }
536
537public:
538 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
539
540 void InsertHelper(Instruction *I, const Twine &Name,
541 BasicBlock::iterator InsertPt) const {
542 IRBuilderDefaultInserter::InsertHelper(I, Name: getNameWithPrefix(Name),
543 InsertPt);
544 }
545};
546
547/// Provide a type for IRBuilder that drops names in release builds.
548using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
549
550/// A used slice of an alloca.
551///
552/// This structure represents a slice of an alloca used by some instruction. It
553/// stores both the begin and end offsets of this use, a pointer to the use
554/// itself, and a flag indicating whether we can classify the use as splittable
555/// or not when forming partitions of the alloca.
556class Slice {
557 /// The beginning offset of the range.
558 uint64_t BeginOffset = 0;
559
560 /// The ending offset, not included in the range.
561 uint64_t EndOffset = 0;
562
563 /// Storage for both the use of this slice and whether it can be
564 /// split.
565 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
566
567public:
568 Slice() = default;
569
570 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
571 : BeginOffset(BeginOffset), EndOffset(EndOffset),
572 UseAndIsSplittable(U, IsSplittable) {}
573
574 uint64_t beginOffset() const { return BeginOffset; }
575 uint64_t endOffset() const { return EndOffset; }
576
577 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
578 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
579
580 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
581
582 bool isDead() const { return getUse() == nullptr; }
583 void kill() { UseAndIsSplittable.setPointer(nullptr); }
584
585 /// Support for ordering ranges.
586 ///
587 /// This provides an ordering over ranges such that start offsets are
588 /// always increasing, and within equal start offsets, the end offsets are
589 /// decreasing. Thus the spanning range comes first in a cluster with the
590 /// same start position.
591 bool operator<(const Slice &RHS) const {
592 if (beginOffset() < RHS.beginOffset())
593 return true;
594 if (beginOffset() > RHS.beginOffset())
595 return false;
596 if (isSplittable() != RHS.isSplittable())
597 return !isSplittable();
598 if (endOffset() > RHS.endOffset())
599 return true;
600 return false;
601 }
602
603 /// Support comparison with a single offset to allow binary searches.
604 [[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
605 return LHS.beginOffset() < RHSOffset;
606 }
607 [[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
608 return LHSOffset < RHS.beginOffset();
609 }
610
611 bool operator==(const Slice &RHS) const {
612 return isSplittable() == RHS.isSplittable() &&
613 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
614 }
615 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
616};
617
618/// Representation of the alloca slices.
619///
620/// This class represents the slices of an alloca which are formed by its
621/// various uses. If a pointer escapes, we can't fully build a representation
622/// for the slices used and we reflect that in this structure. The uses are
623/// stored, sorted by increasing beginning offset and with unsplittable slices
624/// starting at a particular offset before splittable slices.
625class AllocaSlices {
626public:
627 /// Construct the slices of a particular alloca.
628 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
629
630 /// Test whether a pointer to the allocation escapes our analysis.
631 ///
632 /// If this is true, the slices are never fully built and should be
633 /// ignored.
634 bool isEscaped() const { return PointerEscapingInstr; }
635 bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
636
637 /// Support for iterating over the slices.
638 /// @{
639 using iterator = SmallVectorImpl<Slice>::iterator;
640 using range = iterator_range<iterator>;
641
642 iterator begin() { return Slices.begin(); }
643 iterator end() { return Slices.end(); }
644
645 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
646
647 const_iterator begin() const { return Slices.begin(); }
648 const_iterator end() const { return Slices.end(); }
649 /// @}
650
651 /// Erase a range of slices.
652 void erase(iterator Start, iterator Stop) { Slices.erase(CS: Start, CE: Stop); }
653
654 /// Insert new slices for this alloca.
655 ///
656 /// This moves the slices into the alloca's slices collection, and re-sorts
657 /// everything so that the usual ordering properties of the alloca's slices
658 /// hold.
659 void insert(ArrayRef<Slice> NewSlices) {
660 int OldSize = Slices.size();
661 Slices.append(in_start: NewSlices.begin(), in_end: NewSlices.end());
662 auto SliceI = Slices.begin() + OldSize;
663 std::stable_sort(first: SliceI, last: Slices.end());
664 std::inplace_merge(first: Slices.begin(), middle: SliceI, last: Slices.end());
665 }
666
667 // Forward declare the iterator and range accessor for walking the
668 // partitions.
669 class partition_iterator;
670 iterator_range<partition_iterator> partitions();
671
672 /// Access the dead users for this alloca.
673 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
674
675 /// Access Uses that should be dropped if the alloca is promotable.
676 ArrayRef<Use *> getDeadUsesIfPromotable() const {
677 return DeadUseIfPromotable;
678 }
679
680 /// Access the dead operands referring to this alloca.
681 ///
682 /// These are operands which have cannot actually be used to refer to the
683 /// alloca as they are outside its range and the user doesn't correct for
684 /// that. These mostly consist of PHI node inputs and the like which we just
685 /// need to replace with undef.
686 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
687
688#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
689 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
690 void printSlice(raw_ostream &OS, const_iterator I,
691 StringRef Indent = " ") const;
692 void printUse(raw_ostream &OS, const_iterator I,
693 StringRef Indent = " ") const;
694 void print(raw_ostream &OS) const;
695 void dump(const_iterator I) const;
696 void dump() const;
697#endif
698
699private:
700 class SliceBuilder;
701
702 friend class AllocaSlices::SliceBuilder;
703
704#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
705 /// Handle to alloca instruction to simplify method interfaces.
706 AllocaInst &AI;
707#endif
708
709 /// The instruction responsible for this alloca not having a known set
710 /// of slices.
711 ///
712 /// When an instruction (potentially) escapes the pointer to the alloca, we
713 /// store a pointer to that here and abort trying to form slices of the
714 /// alloca. This will be null if the alloca slices are analyzed successfully.
715 Instruction *PointerEscapingInstr;
716 Instruction *PointerEscapingInstrReadOnly;
717
718 /// The slices of the alloca.
719 ///
720 /// We store a vector of the slices formed by uses of the alloca here. This
721 /// vector is sorted by increasing begin offset, and then the unsplittable
722 /// slices before the splittable ones. See the Slice inner class for more
723 /// details.
724 SmallVector<Slice, 8> Slices;
725
726 /// Instructions which will become dead if we rewrite the alloca.
727 ///
728 /// Note that these are not separated by slice. This is because we expect an
729 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
730 /// all these instructions can simply be removed and replaced with poison as
731 /// they come from outside of the allocated space.
732 SmallVector<Instruction *, 8> DeadUsers;
733
734 /// Uses which will become dead if can promote the alloca.
735 SmallVector<Use *, 8> DeadUseIfPromotable;
736
737 /// Operands which will become dead if we rewrite the alloca.
738 ///
739 /// These are operands that in their particular use can be replaced with
740 /// poison when we rewrite the alloca. These show up in out-of-bounds inputs
741 /// to PHI nodes and the like. They aren't entirely dead (there might be
742 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
743 /// want to swap this particular input for poison to simplify the use lists of
744 /// the alloca.
745 SmallVector<Use *, 8> DeadOperands;
746};
747
748/// A partition of the slices.
749///
750/// An ephemeral representation for a range of slices which can be viewed as
751/// a partition of the alloca. This range represents a span of the alloca's
752/// memory which cannot be split, and provides access to all of the slices
753/// overlapping some part of the partition.
754///
755/// Objects of this type are produced by traversing the alloca's slices, but
756/// are only ephemeral and not persistent.
757class Partition {
758private:
759 friend class AllocaSlices;
760 friend class AllocaSlices::partition_iterator;
761
762 using iterator = AllocaSlices::iterator;
763
764 /// The beginning and ending offsets of the alloca for this
765 /// partition.
766 uint64_t BeginOffset = 0, EndOffset = 0;
767
768 /// The start and end iterators of this partition.
769 iterator SI, SJ;
770
771 /// A collection of split slice tails overlapping the partition.
772 SmallVector<Slice *, 4> SplitTails;
773
774 /// Raw constructor builds an empty partition starting and ending at
775 /// the given iterator.
776 Partition(iterator SI) : SI(SI), SJ(SI) {}
777
778public:
779 /// The start offset of this partition.
780 ///
781 /// All of the contained slices start at or after this offset.
782 uint64_t beginOffset() const { return BeginOffset; }
783
784 /// The end offset of this partition.
785 ///
786 /// All of the contained slices end at or before this offset.
787 uint64_t endOffset() const { return EndOffset; }
788
789 /// The size of the partition.
790 ///
791 /// Note that this can never be zero.
792 uint64_t size() const {
793 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
794 return EndOffset - BeginOffset;
795 }
796
797 /// Test whether this partition contains no slices, and merely spans
798 /// a region occupied by split slices.
799 bool empty() const { return SI == SJ; }
800
801 /// \name Iterate slices that start within the partition.
802 /// These may be splittable or unsplittable. They have a begin offset >= the
803 /// partition begin offset.
804 /// @{
805 // FIXME: We should probably define a "concat_iterator" helper and use that
806 // to stitch together pointee_iterators over the split tails and the
807 // contiguous iterators of the partition. That would give a much nicer
808 // interface here. We could then additionally expose filtered iterators for
809 // split, unsplit, and unsplittable splices based on the usage patterns.
810 iterator begin() const { return SI; }
811 iterator end() const { return SJ; }
812 /// @}
813
814 /// Get the sequence of split slice tails.
815 ///
816 /// These tails are of slices which start before this partition but are
817 /// split and overlap into the partition. We accumulate these while forming
818 /// partitions.
819 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
820};
821
822} // end anonymous namespace
823
824/// An iterator over partitions of the alloca's slices.
825///
826/// This iterator implements the core algorithm for partitioning the alloca's
827/// slices. It is a forward iterator as we don't support backtracking for
828/// efficiency reasons, and re-use a single storage area to maintain the
829/// current set of split slices.
830///
831/// It is templated on the slice iterator type to use so that it can operate
832/// with either const or non-const slice iterators.
833class AllocaSlices::partition_iterator
834 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
835 Partition> {
836 friend class AllocaSlices;
837
838 /// Most of the state for walking the partitions is held in a class
839 /// with a nice interface for examining them.
840 Partition P;
841
842 /// We need to keep the end of the slices to know when to stop.
843 AllocaSlices::iterator SE;
844
845 /// We also need to keep track of the maximum split end offset seen.
846 /// FIXME: Do we really?
847 uint64_t MaxSplitSliceEndOffset = 0;
848
849 /// Sets the partition to be empty at given iterator, and sets the
850 /// end iterator.
851 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
852 : P(SI), SE(SE) {
853 // If not already at the end, advance our state to form the initial
854 // partition.
855 if (SI != SE)
856 advance();
857 }
858
859 /// Advance the iterator to the next partition.
860 ///
861 /// Requires that the iterator not be at the end of the slices.
862 void advance() {
863 assert((P.SI != SE || !P.SplitTails.empty()) &&
864 "Cannot advance past the end of the slices!");
865
866 // Clear out any split uses which have ended.
867 if (!P.SplitTails.empty()) {
868 if (P.EndOffset >= MaxSplitSliceEndOffset) {
869 // If we've finished all splits, this is easy.
870 P.SplitTails.clear();
871 MaxSplitSliceEndOffset = 0;
872 } else {
873 // Remove the uses which have ended in the prior partition. This
874 // cannot change the max split slice end because we just checked that
875 // the prior partition ended prior to that max.
876 llvm::erase_if(C&: P.SplitTails,
877 P: [&](Slice *S) { return S->endOffset() <= P.EndOffset; });
878 assert(llvm::any_of(P.SplitTails,
879 [&](Slice *S) {
880 return S->endOffset() == MaxSplitSliceEndOffset;
881 }) &&
882 "Could not find the current max split slice offset!");
883 assert(llvm::all_of(P.SplitTails,
884 [&](Slice *S) {
885 return S->endOffset() <= MaxSplitSliceEndOffset;
886 }) &&
887 "Max split slice end offset is not actually the max!");
888 }
889 }
890
891 // If P.SI is already at the end, then we've cleared the split tail and
892 // now have an end iterator.
893 if (P.SI == SE) {
894 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
895 return;
896 }
897
898 // If we had a non-empty partition previously, set up the state for
899 // subsequent partitions.
900 if (P.SI != P.SJ) {
901 // Accumulate all the splittable slices which started in the old
902 // partition into the split list.
903 for (Slice &S : P)
904 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
905 P.SplitTails.push_back(Elt: &S);
906 MaxSplitSliceEndOffset =
907 std::max(a: S.endOffset(), b: MaxSplitSliceEndOffset);
908 }
909
910 // Start from the end of the previous partition.
911 P.SI = P.SJ;
912
913 // If P.SI is now at the end, we at most have a tail of split slices.
914 if (P.SI == SE) {
915 P.BeginOffset = P.EndOffset;
916 P.EndOffset = MaxSplitSliceEndOffset;
917 return;
918 }
919
920 // If the we have split slices and the next slice is after a gap and is
921 // not splittable immediately form an empty partition for the split
922 // slices up until the next slice begins.
923 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
924 !P.SI->isSplittable()) {
925 P.BeginOffset = P.EndOffset;
926 P.EndOffset = P.SI->beginOffset();
927 return;
928 }
929 }
930
931 // OK, we need to consume new slices. Set the end offset based on the
932 // current slice, and step SJ past it. The beginning offset of the
933 // partition is the beginning offset of the next slice unless we have
934 // pre-existing split slices that are continuing, in which case we begin
935 // at the prior end offset.
936 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
937 P.EndOffset = P.SI->endOffset();
938 ++P.SJ;
939
940 // There are two strategies to form a partition based on whether the
941 // partition starts with an unsplittable slice or a splittable slice.
942 if (!P.SI->isSplittable()) {
943 // When we're forming an unsplittable region, it must always start at
944 // the first slice and will extend through its end.
945 assert(P.BeginOffset == P.SI->beginOffset());
946
947 // Form a partition including all of the overlapping slices with this
948 // unsplittable slice.
949 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
950 if (!P.SJ->isSplittable())
951 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
952 ++P.SJ;
953 }
954
955 // We have a partition across a set of overlapping unsplittable
956 // partitions.
957 return;
958 }
959
960 // If we're starting with a splittable slice, then we need to form
961 // a synthetic partition spanning it and any other overlapping splittable
962 // splices.
963 assert(P.SI->isSplittable() && "Forming a splittable partition!");
964
965 // Collect all of the overlapping splittable slices.
966 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
967 P.SJ->isSplittable()) {
968 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
969 ++P.SJ;
970 }
971
972 // Back upiP.EndOffset if we ended the span early when encountering an
973 // unsplittable slice. This synthesizes the early end offset of
974 // a partition spanning only splittable slices.
975 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
976 assert(!P.SJ->isSplittable());
977 P.EndOffset = P.SJ->beginOffset();
978 }
979 }
980
981public:
982 bool operator==(const partition_iterator &RHS) const {
983 assert(SE == RHS.SE &&
984 "End iterators don't match between compared partition iterators!");
985
986 // The observed positions of partitions is marked by the P.SI iterator and
987 // the emptiness of the split slices. The latter is only relevant when
988 // P.SI == SE, as the end iterator will additionally have an empty split
989 // slices list, but the prior may have the same P.SI and a tail of split
990 // slices.
991 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
992 assert(P.SJ == RHS.P.SJ &&
993 "Same set of slices formed two different sized partitions!");
994 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
995 "Same slice position with differently sized non-empty split "
996 "slice tails!");
997 return true;
998 }
999 return false;
1000 }
1001
1002 partition_iterator &operator++() {
1003 advance();
1004 return *this;
1005 }
1006
1007 Partition &operator*() { return P; }
1008};
1009
1010/// A forward range over the partitions of the alloca's slices.
1011///
1012/// This accesses an iterator range over the partitions of the alloca's
1013/// slices. It computes these partitions on the fly based on the overlapping
1014/// offsets of the slices and the ability to split them. It will visit "empty"
1015/// partitions to cover regions of the alloca only accessed via split
1016/// slices.
1017iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
1018 return make_range(x: partition_iterator(begin(), end()),
1019 y: partition_iterator(end(), end()));
1020}
1021
1022static Value *foldSelectInst(SelectInst &SI) {
1023 // If the condition being selected on is a constant or the same value is
1024 // being selected between, fold the select. Yes this does (rarely) happen
1025 // early on.
1026 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: SI.getCondition()))
1027 return SI.getOperand(i_nocapture: 1 + CI->isZero());
1028 if (SI.getOperand(i_nocapture: 1) == SI.getOperand(i_nocapture: 2))
1029 return SI.getOperand(i_nocapture: 1);
1030
1031 return nullptr;
1032}
1033
1034/// A helper that folds a PHI node or a select.
1035static Value *foldPHINodeOrSelectInst(Instruction &I) {
1036 if (PHINode *PN = dyn_cast<PHINode>(Val: &I)) {
1037 // If PN merges together the same value, return that value.
1038 return PN->hasConstantValue();
1039 }
1040 return foldSelectInst(SI&: cast<SelectInst>(Val&: I));
1041}
1042
1043/// Builder for the alloca slices.
1044///
1045/// This class builds a set of alloca slices by recursively visiting the uses
1046/// of an alloca and making a slice for each load and store at each offset.
1047class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1048 friend class PtrUseVisitor<SliceBuilder>;
1049 friend class InstVisitor<SliceBuilder>;
1050
1051 using Base = PtrUseVisitor<SliceBuilder>;
1052
1053 const uint64_t AllocSize;
1054 AllocaSlices &AS;
1055
1056 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
1057 SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
1058
1059 /// Set to de-duplicate dead instructions found in the use walk.
1060 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
1061
1062public:
1063 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1064 : PtrUseVisitor<SliceBuilder>(DL),
1065 AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1066
1067private:
1068 void markAsDead(Instruction &I) {
1069 if (VisitedDeadInsts.insert(Ptr: &I).second)
1070 AS.DeadUsers.push_back(Elt: &I);
1071 }
1072
1073 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1074 bool IsSplittable = false) {
1075 // Completely skip uses which have a zero size or start either before or
1076 // past the end of the allocation.
1077 if (Size == 0 || Offset.uge(RHS: AllocSize)) {
1078 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1079 << Offset
1080 << " which has zero size or starts outside of the "
1081 << AllocSize << " byte alloca:\n"
1082 << " alloca: " << AS.AI << "\n"
1083 << " use: " << I << "\n");
1084 return markAsDead(I);
1085 }
1086
1087 uint64_t BeginOffset = Offset.getZExtValue();
1088 uint64_t EndOffset = BeginOffset + Size;
1089
1090 // Clamp the end offset to the end of the allocation. Note that this is
1091 // formulated to handle even the case where "BeginOffset + Size" overflows.
1092 // This may appear superficially to be something we could ignore entirely,
1093 // but that is not so! There may be widened loads or PHI-node uses where
1094 // some instructions are dead but not others. We can't completely ignore
1095 // them, and so have to record at least the information here.
1096 assert(AllocSize >= BeginOffset); // Established above.
1097 if (Size > AllocSize - BeginOffset) {
1098 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1099 << Offset << " to remain within the " << AllocSize
1100 << " byte alloca:\n"
1101 << " alloca: " << AS.AI << "\n"
1102 << " use: " << I << "\n");
1103 EndOffset = AllocSize;
1104 }
1105
1106 AS.Slices.push_back(Elt: Slice(BeginOffset, EndOffset, U, IsSplittable));
1107 }
1108
1109 void visitBitCastInst(BitCastInst &BC) {
1110 if (BC.use_empty())
1111 return markAsDead(I&: BC);
1112
1113 return Base::visitBitCastInst(BC);
1114 }
1115
1116 void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1117 if (ASC.use_empty())
1118 return markAsDead(I&: ASC);
1119
1120 return Base::visitAddrSpaceCastInst(ASC);
1121 }
1122
1123 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1124 if (GEPI.use_empty())
1125 return markAsDead(I&: GEPI);
1126
1127 return Base::visitGetElementPtrInst(GEPI);
1128 }
1129
1130 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
1131 uint64_t Size, bool IsVolatile) {
1132 // We allow splitting of non-volatile loads and stores where the type is an
1133 // integer type. These may be used to implement 'memcpy' or other "transfer
1134 // of bits" patterns.
1135 bool IsSplittable =
1136 Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1137
1138 insertUse(I, Offset, Size, IsSplittable);
1139 }
1140
1141 void visitLoadInst(LoadInst &LI) {
1142 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
1143 "All simple FCA loads should have been pre-split");
1144
1145 // If there is a load with an unknown offset, we can still perform store
1146 // to load forwarding for other known-offset loads.
1147 if (!IsOffsetKnown)
1148 return PI.setEscapedReadOnly(&LI);
1149
1150 TypeSize Size = DL.getTypeStoreSize(Ty: LI.getType());
1151 if (Size.isScalable()) {
1152 unsigned VScale = LI.getFunction()->getVScaleValue();
1153 if (!VScale)
1154 return PI.setAborted(&LI);
1155
1156 Size = TypeSize::getFixed(ExactSize: Size.getKnownMinValue() * VScale);
1157 }
1158
1159 return handleLoadOrStore(Ty: LI.getType(), I&: LI, Offset, Size: Size.getFixedValue(),
1160 IsVolatile: LI.isVolatile());
1161 }
1162
1163 void visitStoreInst(StoreInst &SI) {
1164 Value *ValOp = SI.getValueOperand();
1165 if (ValOp == *U)
1166 return PI.setEscapedAndAborted(&SI);
1167 if (!IsOffsetKnown)
1168 return PI.setAborted(&SI);
1169
1170 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValOp->getType());
1171 if (StoreSize.isScalable()) {
1172 unsigned VScale = SI.getFunction()->getVScaleValue();
1173 if (!VScale)
1174 return PI.setAborted(&SI);
1175
1176 StoreSize = TypeSize::getFixed(ExactSize: StoreSize.getKnownMinValue() * VScale);
1177 }
1178
1179 uint64_t Size = StoreSize.getFixedValue();
1180
1181 // If this memory access can be shown to *statically* extend outside the
1182 // bounds of the allocation, it's behavior is undefined, so simply
1183 // ignore it. Note that this is more strict than the generic clamping
1184 // behavior of insertUse. We also try to handle cases which might run the
1185 // risk of overflow.
1186 // FIXME: We should instead consider the pointer to have escaped if this
1187 // function is being instrumented for addressing bugs or race conditions.
1188 if (Size > AllocSize || Offset.ugt(RHS: AllocSize - Size)) {
1189 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1190 << Offset << " which extends past the end of the "
1191 << AllocSize << " byte alloca:\n"
1192 << " alloca: " << AS.AI << "\n"
1193 << " use: " << SI << "\n");
1194 return markAsDead(I&: SI);
1195 }
1196
1197 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1198 "All simple FCA stores should have been pre-split");
1199 handleLoadOrStore(Ty: ValOp->getType(), I&: SI, Offset, Size, IsVolatile: SI.isVolatile());
1200 }
1201
1202 void visitMemSetInst(MemSetInst &II) {
1203 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1204 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1205 if ((Length && Length->getValue() == 0) ||
1206 (IsOffsetKnown && Offset.uge(RHS: AllocSize)))
1207 // Zero-length mem transfer intrinsics can be ignored entirely.
1208 return markAsDead(I&: II);
1209
1210 if (!IsOffsetKnown)
1211 return PI.setAborted(&II);
1212
1213 insertUse(I&: II, Offset,
1214 Size: Length ? Length->getLimitedValue()
1215 : AllocSize - Offset.getLimitedValue(),
1216 IsSplittable: (bool)Length);
1217 }
1218
1219 void visitMemTransferInst(MemTransferInst &II) {
1220 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1221 if (Length && Length->getValue() == 0)
1222 // Zero-length mem transfer intrinsics can be ignored entirely.
1223 return markAsDead(I&: II);
1224
1225 // Because we can visit these intrinsics twice, also check to see if the
1226 // first time marked this instruction as dead. If so, skip it.
1227 if (VisitedDeadInsts.count(Ptr: &II))
1228 return;
1229
1230 if (!IsOffsetKnown)
1231 return PI.setAborted(&II);
1232
1233 // This side of the transfer is completely out-of-bounds, and so we can
1234 // nuke the entire transfer. However, we also need to nuke the other side
1235 // if already added to our partitions.
1236 // FIXME: Yet another place we really should bypass this when
1237 // instrumenting for ASan.
1238 if (Offset.uge(RHS: AllocSize)) {
1239 auto MTPI = MemTransferSliceMap.find(Val: &II);
1240 if (MTPI != MemTransferSliceMap.end())
1241 AS.Slices[MTPI->second].kill();
1242 return markAsDead(I&: II);
1243 }
1244
1245 uint64_t RawOffset = Offset.getLimitedValue();
1246 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1247
1248 // Check for the special case where the same exact value is used for both
1249 // source and dest.
1250 if (*U == II.getRawDest() && *U == II.getRawSource()) {
1251 // For non-volatile transfers this is a no-op.
1252 if (!II.isVolatile())
1253 return markAsDead(I&: II);
1254
1255 return insertUse(I&: II, Offset, Size, /*IsSplittable=*/false);
1256 }
1257
1258 // If we have seen both source and destination for a mem transfer, then
1259 // they both point to the same alloca.
1260 bool Inserted;
1261 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
1262 std::tie(args&: MTPI, args&: Inserted) =
1263 MemTransferSliceMap.insert(KV: std::make_pair(x: &II, y: AS.Slices.size()));
1264 unsigned PrevIdx = MTPI->second;
1265 if (!Inserted) {
1266 Slice &PrevP = AS.Slices[PrevIdx];
1267
1268 // Check if the begin offsets match and this is a non-volatile transfer.
1269 // In that case, we can completely elide the transfer.
1270 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1271 PrevP.kill();
1272 return markAsDead(I&: II);
1273 }
1274
1275 // Otherwise we have an offset transfer within the same alloca. We can't
1276 // split those.
1277 PrevP.makeUnsplittable();
1278 }
1279
1280 // Insert the use now that we've fixed up the splittable nature.
1281 insertUse(I&: II, Offset, Size, /*IsSplittable=*/Inserted && Length);
1282
1283 // Check that we ended up with a valid index in the map.
1284 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1285 "Map index doesn't point back to a slice with this user.");
1286 }
1287
1288 // Disable SRoA for any intrinsics except for lifetime invariants.
1289 // FIXME: What about debug intrinsics? This matches old behavior, but
1290 // doesn't make sense.
1291 void visitIntrinsicInst(IntrinsicInst &II) {
1292 if (II.isDroppable()) {
1293 AS.DeadUseIfPromotable.push_back(Elt: U);
1294 return;
1295 }
1296
1297 if (!IsOffsetKnown)
1298 return PI.setAborted(&II);
1299
1300 if (II.isLifetimeStartOrEnd()) {
1301 insertUse(I&: II, Offset, Size: AllocSize, IsSplittable: true);
1302 return;
1303 }
1304
1305 Base::visitIntrinsicInst(II);
1306 }
1307
1308 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
1309 // We consider any PHI or select that results in a direct load or store of
1310 // the same offset to be a viable use for slicing purposes. These uses
1311 // are considered unsplittable and the size is the maximum loaded or stored
1312 // size.
1313 SmallPtrSet<Instruction *, 4> Visited;
1314 SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses;
1315 Visited.insert(Ptr: Root);
1316 Uses.push_back(Elt: std::make_pair(x: cast<Instruction>(Val&: *U), y&: Root));
1317 const DataLayout &DL = Root->getDataLayout();
1318 // If there are no loads or stores, the access is dead. We mark that as
1319 // a size zero access.
1320 Size = 0;
1321 do {
1322 Instruction *I, *UsedI;
1323 std::tie(args&: UsedI, args&: I) = Uses.pop_back_val();
1324
1325 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1326 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
1327 if (LoadSize.isScalable()) {
1328 PI.setAborted(LI);
1329 return nullptr;
1330 }
1331 Size = std::max(a: Size, b: LoadSize.getFixedValue());
1332 continue;
1333 }
1334 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1335 Value *Op = SI->getOperand(i_nocapture: 0);
1336 if (Op == UsedI)
1337 return SI;
1338 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Op->getType());
1339 if (StoreSize.isScalable()) {
1340 PI.setAborted(SI);
1341 return nullptr;
1342 }
1343 Size = std::max(a: Size, b: StoreSize.getFixedValue());
1344 continue;
1345 }
1346
1347 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1348 if (!GEP->hasAllZeroIndices())
1349 return GEP;
1350 } else if (!isa<BitCastInst>(Val: I) && !isa<PHINode>(Val: I) &&
1351 !isa<SelectInst>(Val: I) && !isa<AddrSpaceCastInst>(Val: I)) {
1352 return I;
1353 }
1354
1355 for (User *U : I->users())
1356 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
1357 Uses.push_back(Elt: std::make_pair(x&: I, y: cast<Instruction>(Val: U)));
1358 } while (!Uses.empty());
1359
1360 return nullptr;
1361 }
1362
1363 void visitPHINodeOrSelectInst(Instruction &I) {
1364 assert(isa<PHINode>(I) || isa<SelectInst>(I));
1365 if (I.use_empty())
1366 return markAsDead(I);
1367
1368 // If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1369 // instructions in this BB, which may be required during rewriting. Bail out
1370 // on these cases.
1371 if (isa<PHINode>(Val: I) && !I.getParent()->hasInsertionPt())
1372 return PI.setAborted(&I);
1373
1374 // TODO: We could use simplifyInstruction here to fold PHINodes and
1375 // SelectInsts. However, doing so requires to change the current
1376 // dead-operand-tracking mechanism. For instance, suppose neither loading
1377 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1378 // trap either. However, if we simply replace %U with undef using the
1379 // current dead-operand-tracking mechanism, "load (select undef, undef,
1380 // %other)" may trap because the select may return the first operand
1381 // "undef".
1382 if (Value *Result = foldPHINodeOrSelectInst(I)) {
1383 if (Result == *U)
1384 // If the result of the constant fold will be the pointer, recurse
1385 // through the PHI/select as if we had RAUW'ed it.
1386 enqueueUsers(I);
1387 else
1388 // Otherwise the operand to the PHI/select is dead, and we can replace
1389 // it with poison.
1390 AS.DeadOperands.push_back(Elt: U);
1391
1392 return;
1393 }
1394
1395 if (!IsOffsetKnown)
1396 return PI.setAborted(&I);
1397
1398 // See if we already have computed info on this node.
1399 uint64_t &Size = PHIOrSelectSizes[&I];
1400 if (!Size) {
1401 // This is a new PHI/Select, check for an unsafe use of it.
1402 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(Root: &I, Size))
1403 return PI.setAborted(UnsafeI);
1404 }
1405
1406 // For PHI and select operands outside the alloca, we can't nuke the entire
1407 // phi or select -- the other side might still be relevant, so we special
1408 // case them here and use a separate structure to track the operands
1409 // themselves which should be replaced with poison.
1410 // FIXME: This should instead be escaped in the event we're instrumenting
1411 // for address sanitization.
1412 if (Offset.uge(RHS: AllocSize)) {
1413 AS.DeadOperands.push_back(Elt: U);
1414 return;
1415 }
1416
1417 insertUse(I, Offset, Size);
1418 }
1419
1420 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(I&: PN); }
1421
1422 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(I&: SI); }
1423
1424 /// Disable SROA entirely if there are unhandled users of the alloca.
1425 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1426
1427 void visitCallBase(CallBase &CB) {
1428 // If the call operand is read-only and only does a read-only or address
1429 // capture, then we mark it as EscapedReadOnly.
1430 if (CB.isDataOperand(U) &&
1431 !capturesFullProvenance(CC: CB.getCaptureInfo(OpNo: U->getOperandNo())) &&
1432 CB.onlyReadsMemory(OpNo: U->getOperandNo())) {
1433 PI.setEscapedReadOnly(&CB);
1434 return;
1435 }
1436
1437 Base::visitCallBase(CB);
1438 }
1439};
1440
1441AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1442 :
1443#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1444 AI(AI),
1445#endif
1446 PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1447 SliceBuilder PB(DL, AI, *this);
1448 SliceBuilder::PtrInfo PtrI = PB.visitPtr(I&: AI);
1449 if (PtrI.isEscaped() || PtrI.isAborted()) {
1450 // FIXME: We should sink the escape vs. abort info into the caller nicely,
1451 // possibly by just storing the PtrInfo in the AllocaSlices.
1452 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1453 : PtrI.getAbortingInst();
1454 assert(PointerEscapingInstr && "Did not track a bad instruction");
1455 return;
1456 }
1457 PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1458
1459 llvm::erase_if(C&: Slices, P: [](const Slice &S) { return S.isDead(); });
1460
1461 // Sort the uses. This arranges for the offsets to be in ascending order,
1462 // and the sizes to be in descending order.
1463 llvm::stable_sort(Range&: Slices);
1464}
1465
1466#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1467
1468void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1469 StringRef Indent) const {
1470 printSlice(OS, I, Indent);
1471 OS << "\n";
1472 printUse(OS, I, Indent);
1473}
1474
1475void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1476 StringRef Indent) const {
1477 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1478 << " slice #" << (I - begin())
1479 << (I->isSplittable() ? " (splittable)" : "");
1480}
1481
1482void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1483 StringRef Indent) const {
1484 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1485}
1486
1487void AllocaSlices::print(raw_ostream &OS) const {
1488 if (PointerEscapingInstr) {
1489 OS << "Can't analyze slices for alloca: " << AI << "\n"
1490 << " A pointer to this alloca escaped by:\n"
1491 << " " << *PointerEscapingInstr << "\n";
1492 return;
1493 }
1494
1495 if (PointerEscapingInstrReadOnly)
1496 OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1497
1498 OS << "Slices of alloca: " << AI << "\n";
1499 for (const_iterator I = begin(), E = end(); I != E; ++I)
1500 print(OS, I);
1501}
1502
1503LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1504 print(dbgs(), I);
1505}
1506LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1507
1508#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1509
1510/// Find a common load/store type used through a pointer PHI or select.
1511///
1512/// Look through a PHI or select to see if all of its users are loads or stores
1513/// of one common type. Whether those accesses can be speculated does not affect
1514/// the type they use and is checked separately when attempting promotion.
1515static Type *findCommonTypeThroughPHIOrSelect(Instruction &I) {
1516 assert((isa<PHINode, SelectInst>(I)) && "expected a PHI or select");
1517 Type *Ty = nullptr;
1518
1519 for (User *U : I.users()) {
1520 Type *UserTy = nullptr;
1521 if (auto *LI = dyn_cast<LoadInst>(Val: U))
1522 UserTy = LI->getType();
1523 else if (auto *Store = dyn_cast<StoreInst>(Val: U))
1524 // Slice building rejects stores of the PHI-or-select-derived pointer, so
1525 // it must be the store's pointer operand here.
1526 UserTy = Store->getValueOperand()->getType();
1527
1528 if (!UserTy || (Ty && Ty != UserTy))
1529 return nullptr;
1530 Ty = UserTy;
1531 }
1532
1533 return Ty;
1534}
1535
1536/// Walk the range of a partitioning looking for a common type to cover this
1537/// sequence of slices.
1538static std::pair<Type *, IntegerType *>
1539findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1540 uint64_t EndOffset) {
1541 Type *Ty = nullptr;
1542 bool TyIsCommon = true;
1543 IntegerType *ITy = nullptr;
1544
1545 // Note that we need to look at *every* alloca slice's Use to ensure we
1546 // always get consistent results regardless of the order of slices.
1547 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1548 Use *U = I->getUse();
1549 if (isa<IntrinsicInst>(Val: *U->getUser()))
1550 continue;
1551 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1552 continue;
1553
1554 Type *UserTy = nullptr;
1555 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
1556 UserTy = LI->getType();
1557 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
1558 UserTy = SI->getValueOperand()->getType();
1559 } else if (isa<PHINode, SelectInst>(Val: U->getUser())) {
1560 UserTy =
1561 findCommonTypeThroughPHIOrSelect(I&: *cast<Instruction>(Val: U->getUser()));
1562 }
1563
1564 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(Val: UserTy)) {
1565 // If the type is larger than the partition, skip it. We only encounter
1566 // this for split integer operations where we want to use the type of the
1567 // entity causing the split. Also skip if the type is not a byte width
1568 // multiple.
1569 if (UserITy->getBitWidth() % 8 != 0 ||
1570 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
1571 continue;
1572
1573 // Track the largest bitwidth integer type used in this way in case there
1574 // is no common type.
1575 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1576 ITy = UserITy;
1577 }
1578
1579 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1580 // depend on types skipped above.
1581 if (!UserTy || (Ty && Ty != UserTy))
1582 TyIsCommon = false; // Give up on anything but an iN type.
1583 else
1584 Ty = UserTy;
1585 }
1586
1587 return {TyIsCommon ? Ty : nullptr, ITy};
1588}
1589
1590/// PHI instructions that use an alloca and are subsequently loaded can be
1591/// rewritten to load both input pointers in the pred blocks and then PHI the
1592/// results, allowing the load of the alloca to be promoted.
1593/// From this:
1594/// %P2 = phi [i32* %Alloca, i32* %Other]
1595/// %V = load i32* %P2
1596/// to:
1597/// %V1 = load i32* %Alloca -> will be mem2reg'd
1598/// ...
1599/// %V2 = load i32* %Other
1600/// ...
1601/// %V = phi [i32 %V1, i32 %V2]
1602///
1603/// We can do this to a PHI if its only uses are loads and if any loads moved
1604/// across other outgoing edges can be executed unconditionally.
1605///
1606/// FIXME: This should be hoisted into a generic utility, likely in
1607/// Transforms/Util/Local.h
1608static bool isSafePHIToSpeculate(PHINode &PN) {
1609 const DataLayout &DL = PN.getDataLayout();
1610
1611 // For now, we can only do this promotion if the load is in the same block
1612 // as the PHI, and if there are no stores between the phi and load.
1613 // TODO: Allow recursive phi users.
1614 BasicBlock *BB = PN.getParent();
1615 Align MaxAlign;
1616 uint64_t APWidth = DL.getIndexTypeSizeInBits(Ty: PN.getType());
1617 Type *LoadType = nullptr;
1618 for (User *U : PN.users()) {
1619 LoadInst *LI = dyn_cast<LoadInst>(Val: U);
1620 if (!LI || !LI->isSimple())
1621 return false;
1622
1623 // For now we only allow loads in the same block as the PHI. This is
1624 // a common case that happens when instcombine merges two loads through
1625 // a PHI.
1626 if (LI->getParent() != BB)
1627 return false;
1628
1629 if (LoadType) {
1630 if (LoadType != LI->getType())
1631 return false;
1632 } else {
1633 LoadType = LI->getType();
1634 }
1635
1636 // Ensure that there are no instructions between the PHI and the load that
1637 // could store.
1638 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1639 if (BBI->mayWriteToMemory())
1640 return false;
1641
1642 MaxAlign = std::max(a: MaxAlign, b: LI->getAlign());
1643 }
1644
1645 if (!LoadType)
1646 return false;
1647
1648 APInt LoadSize =
1649 APInt(APWidth, DL.getTypeStoreSize(Ty: LoadType).getFixedValue());
1650
1651 // We can only transform this if it is safe to push the loads into the
1652 // predecessor blocks. The only thing to watch out for is that we can't put
1653 // a possibly trapping load in the predecessor if it is a critical edge.
1654 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1655 Instruction *TI = PN.getIncomingBlock(i: Idx)->getTerminator();
1656 Value *InVal = PN.getIncomingValue(i: Idx);
1657
1658 // If the value is produced by the terminator of the predecessor (an
1659 // invoke) or it has side-effects, there is no valid place to put a load
1660 // in the predecessor.
1661 if (TI == InVal || TI->mayHaveSideEffects())
1662 return false;
1663
1664 // If the predecessor has a single successor, then the edge isn't
1665 // critical.
1666 if (TI->getNumSuccessors() == 1)
1667 continue;
1668
1669 // If this pointer is always safe to load, or if we can prove that there
1670 // is already a load in the block, then we can move the load to the pred
1671 // block.
1672 if (isSafeToLoadUnconditionally(V: InVal, Alignment: MaxAlign, Size: LoadSize,
1673 SQ: SimplifyQuery(DL, TI)))
1674 continue;
1675
1676 return false;
1677 }
1678
1679 return true;
1680}
1681
1682/// Check whether a single store through PN can be moved onto each incoming
1683/// edge.
1684static StoreInst *getPHIStoreToRewrite(PHINode &PN, bool PreserveCFG,
1685 DominatorTree &DT) {
1686 // TODO: Support multiple stores and mixed load/store users.
1687 // TODO: Look through other instructions, such as other phis or addrspacecasts
1688 if (!PN.hasOneUse())
1689 return nullptr;
1690
1691 auto *SI = dyn_cast<StoreInst>(Val: PN.user_back());
1692 if (!SI || SI->getPointerOperand() != &PN)
1693 return nullptr;
1694
1695 if (SI->isVolatile())
1696 return nullptr;
1697
1698 BasicBlock *BB = PN.getParent();
1699 // TODO: Allow a harmless prefix between the PHIs and the store.
1700 if (&*BB->getFirstNonPHIOrPseudoProbe() != SI)
1701 return nullptr;
1702
1703 Value *StoredValue = SI->getValueOperand();
1704 SmallPtrSet<BasicBlock *, 4> SeenPreds;
1705 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1706 BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1707
1708 // Only one store is needed for duplicate edges from the same predecessor.
1709 if (!SeenPreds.insert(Ptr: Pred).second)
1710 continue;
1711
1712 // TODO: Support other terminators.
1713 Instruction *TI = Pred->getTerminator();
1714 if (!isa<CondBrInst, UncondBrInst>(Val: TI))
1715 return nullptr;
1716
1717 if (Pred == BB)
1718 return nullptr;
1719
1720 // TODO: If StoredValue is another PHI in BB, use its corresponding
1721 // incoming value instead of requiring it to dominate every predecessor.
1722 if (!DT.dominates(Def: StoredValue, User: TI))
1723 return nullptr;
1724
1725 if (TI->getNumSuccessors() == 1)
1726 continue;
1727
1728 // If the predecessor has more than one successor, then we will need to
1729 // split it so that we can insert the store only on the path to this BB.
1730 if (PreserveCFG || !BB->canSplitPredecessors())
1731 return nullptr;
1732 }
1733
1734 return SI;
1735}
1736
1737static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1738 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1739
1740 LoadInst *SomeLoad = cast<LoadInst>(Val: PN.user_back());
1741 Type *LoadTy = SomeLoad->getType();
1742 IRB.SetInsertPoint(&PN);
1743 PHINode *NewPN = IRB.CreatePHI(Ty: LoadTy, NumReservedValues: PN.getNumIncomingValues(),
1744 Name: PN.getName() + ".sroa.speculated");
1745
1746 // Get the AA tags and alignment to use from one of the loads. It does not
1747 // matter which one we get and if any differ.
1748 AAMDNodes AATags = SomeLoad->getAAMetadata();
1749 Align Alignment = SomeLoad->getAlign();
1750
1751 // Rewrite all loads of the PN to use the new PHI.
1752 while (!PN.use_empty()) {
1753 LoadInst *LI = cast<LoadInst>(Val: PN.user_back());
1754 LI->replaceAllUsesWith(V: NewPN);
1755 LI->eraseFromParent();
1756 }
1757
1758 // Inject loads into all of the pred blocks.
1759 DenseMap<BasicBlock *, Value *> InjectedLoads;
1760 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1761 BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1762 Value *InVal = PN.getIncomingValue(i: Idx);
1763
1764 // A PHI node is allowed to have multiple (duplicated) entries for the same
1765 // basic block, as long as the value is the same. So if we already injected
1766 // a load in the predecessor, then we should reuse the same load for all
1767 // duplicated entries.
1768 if (Value *V = InjectedLoads.lookup(Val: Pred)) {
1769 NewPN->addIncoming(V, BB: Pred);
1770 continue;
1771 }
1772
1773 Instruction *TI = Pred->getTerminator();
1774 IRB.SetInsertPoint(TI);
1775
1776 LoadInst *Load = IRB.CreateAlignedLoad(
1777 Ty: LoadTy, Ptr: InVal, Align: Alignment,
1778 Name: (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1779 ++NumLoadsSpeculated;
1780 if (AATags)
1781 Load->setAAMetadata(AATags);
1782 NewPN->addIncoming(V: Load, BB: Pred);
1783 InjectedLoads[Pred] = Load;
1784 }
1785
1786 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1787 PN.eraseFromParent();
1788}
1789
1790/// Move a store through a pointer PHI onto each of the PHI's incoming edges.
1791/// Returns whether this required modifying the CFG.
1792static bool rewritePHINodeStore(PHINode &PN, StoreInst &SI, DomTreeUpdater &DTU,
1793 SmallSetVector<AllocaInst *, 16> &Worklist) {
1794 LLVM_DEBUG(dbgs() << " original: " << PN << "\n"
1795 << " " << SI << "\n");
1796
1797 // Splitting one edge rewrites all PHIs in the destination block. Snapshot
1798 // the original predecessor/value pairs before making any CFG changes.
1799 SmallVector<std::pair<BasicBlock *, Value *>, 4> IncomingValues;
1800 SmallPtrSet<BasicBlock *, 4> SeenPreds;
1801 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1802 BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1803 if (!SeenPreds.insert(Ptr: Pred).second)
1804 continue;
1805 Value *InVal = PN.getIncomingValue(i: Idx);
1806 IncomingValues.emplace_back(Args&: Pred, Args&: InVal);
1807
1808 // Revisit every alloca exposed by removing the pointer PHI.
1809 if (auto *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: InVal)))
1810 Worklist.insert(X: AI);
1811 }
1812
1813 bool CFGChanged = false;
1814 BasicBlock *BB = PN.getParent();
1815 for (auto [Pred, InVal] : IncomingValues) {
1816 BasicBlock *StoreBB = Pred;
1817 if (Pred->getTerminator()->getNumSuccessors() != 1) {
1818 StoreBB = SplitBlockPredecessors(BB, Preds: {Pred}, Suffix: ".sroa.store", DTU: &DTU);
1819 assert(StoreBB && "store edge was not checked for splitting");
1820 CFGChanged = true;
1821 }
1822
1823 auto *NewStore = cast<StoreInst>(Val: SI.clone());
1824 NewStore->setOperand(i_nocapture: StoreInst::getPointerOperandIndex(), Val_nocapture: InVal);
1825 NewStore->insertBefore(InsertPos: StoreBB->getTerminator()->getIterator());
1826 ++NumStoresPredicated;
1827 LLVM_DEBUG(dbgs() << " to: " << *NewStore << "\n");
1828 }
1829
1830 SI.eraseFromParent();
1831 PN.eraseFromParent();
1832 return CFGChanged;
1833}
1834
1835SelectHandSpeculativity &
1836SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1837 if (isTrueVal)
1838 Bitfield::set<SelectHandSpeculativity::TrueVal>(Packed&: Storage, Value: true);
1839 else
1840 Bitfield::set<SelectHandSpeculativity::FalseVal>(Packed&: Storage, Value: true);
1841 return *this;
1842}
1843
1844bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1845 return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Packed: Storage)
1846 : Bitfield::get<SelectHandSpeculativity::FalseVal>(Packed: Storage);
1847}
1848
1849bool SelectHandSpeculativity::areAllSpeculatable() const {
1850 return isSpeculatable(/*isTrueVal=*/true) &&
1851 isSpeculatable(/*isTrueVal=*/false);
1852}
1853
1854bool SelectHandSpeculativity::areAnySpeculatable() const {
1855 return isSpeculatable(/*isTrueVal=*/true) ||
1856 isSpeculatable(/*isTrueVal=*/false);
1857}
1858bool SelectHandSpeculativity::areNoneSpeculatable() const {
1859 return !areAnySpeculatable();
1860}
1861
1862static SelectHandSpeculativity
1863isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1864 assert(LI.isSimple() && "Only for simple loads");
1865 SelectHandSpeculativity Spec;
1866
1867 const DataLayout &DL = SI.getDataLayout();
1868 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1869 if (isSafeToLoadUnconditionally(V: Value, Ty: LI.getType(), Alignment: LI.getAlign(),
1870 SQ: SimplifyQuery(DL, &LI)))
1871 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1872 else if (PreserveCFG)
1873 return Spec;
1874
1875 return Spec;
1876}
1877
1878std::optional<RewriteableMemOps>
1879SROA::isSafeSelectToSpeculateOrPredicate(SelectInst &SI, bool PreserveCFG) {
1880 RewriteableMemOps Ops;
1881
1882 SmallVector<User *> Users;
1883 for (User *U : SI.users()) {
1884 if (isa<AddrSpaceCastInst>(Val: U))
1885 append_range(C&: Users, R: U->users());
1886 else
1887 Users.push_back(Elt: U);
1888 }
1889
1890 for (User *U : Users) {
1891 if (auto *Store = dyn_cast<StoreInst>(Val: U)) {
1892 // We don't care about atomicity of stores as they're always predicated.
1893 if (Store->isVolatile() || PreserveCFG)
1894 return {}; // Give up on this `select`.
1895 Ops.emplace_back(Args&: Store);
1896 continue;
1897 }
1898
1899 auto *LI = dyn_cast<LoadInst>(Val: U);
1900
1901 if (!LI || LI->isVolatile())
1902 return {}; // Give up on this `select`.
1903
1904 PossiblySpeculatableLoad Load(LI);
1905
1906 // Atomic loads need to be predicated: speculation emits plain loads, which
1907 // would drop the atomicity of the load when one of the hands is non-local,
1908 // introducing a race.
1909 // We could allow this when both hands are allocas (atomicity doesn't mean
1910 // anything on allocas), but for now we give up if we're not allowed to
1911 // predicate.
1912 if (!LI->isSimple()) { // or `LI->isAtomic()`, `isVolatile` already checked
1913 if (PreserveCFG)
1914 return {}; // Give up on this `select`.
1915 Ops.emplace_back(Args&: Load);
1916 continue;
1917 }
1918
1919 SelectHandSpeculativity Spec =
1920 isSafeLoadOfSelectToSpeculate(LI&: *LI, SI, PreserveCFG);
1921 if (PreserveCFG && !Spec.areAllSpeculatable())
1922 return {}; // Give up on this `select`.
1923
1924 Load.setInt(Spec);
1925 Ops.emplace_back(Args&: Load);
1926 }
1927
1928 return Ops;
1929}
1930
1931static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1932 IRBuilderTy &IRB) {
1933 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1934
1935 Value *TV = SI.getTrueValue();
1936 Value *FV = SI.getFalseValue();
1937 // Replace the given load of the select with a select of two loads.
1938
1939 assert(LI.isSimple() && "We only speculate simple loads");
1940
1941 IRB.SetInsertPoint(&LI);
1942
1943 // Casts may separate the load from the select.
1944 TV = IRB.CreateAddrSpaceCast(V: TV, DestTy: LI.getPointerOperandType(),
1945 Name: LI.getName() + ".sroa.speculate.cast.true");
1946 FV = IRB.CreateAddrSpaceCast(V: FV, DestTy: LI.getPointerOperandType(),
1947 Name: LI.getName() + ".sroa.speculate.cast.false");
1948
1949 LoadInst *TL =
1950 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: TV, Align: LI.getAlign(),
1951 Name: LI.getName() + ".sroa.speculate.load.true");
1952 LoadInst *FL =
1953 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: FV, Align: LI.getAlign(),
1954 Name: LI.getName() + ".sroa.speculate.load.false");
1955 NumLoadsSpeculated += 2;
1956
1957 // Transfer alignment and AA info if present.
1958 TL->setAlignment(LI.getAlign());
1959 FL->setAlignment(LI.getAlign());
1960
1961 AAMDNodes Tags = LI.getAAMetadata();
1962 if (Tags) {
1963 TL->setAAMetadata(Tags);
1964 FL->setAAMetadata(Tags);
1965 }
1966
1967 Value *V = IRB.CreateSelect(C: SI.getCondition(), True: TL, False: FL,
1968 Name: LI.getName() + ".sroa.speculated", MDFrom: &SI);
1969
1970 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1971 LI.replaceAllUsesWith(V);
1972}
1973
1974template <typename T>
1975static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1976 SelectHandSpeculativity Spec, IRBuilderTy &IRB,
1977 DomTreeUpdater &DTU) {
1978 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!");
1979 LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1980 BasicBlock *Head = I.getParent();
1981 Instruction *ThenTerm = nullptr;
1982 Instruction *ElseTerm = nullptr;
1983 if (Spec.areNoneSpeculatable())
1984 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1985 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU);
1986 else {
1987 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1988 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU,
1989 /*LI=*/nullptr, /*ThenBlock=*/nullptr);
1990 if (Spec.isSpeculatable(/*isTrueVal=*/true))
1991 cast<CondBrInst>(Val: Head->getTerminator())->swapSuccessors();
1992 }
1993 auto *HeadBI = cast<CondBrInst>(Val: Head->getTerminator());
1994 Spec = {}; // Do not use `Spec` beyond this point.
1995 BasicBlock *Tail = I.getParent();
1996 Tail->setName(Head->getName() + ".cont");
1997 PHINode *PN;
1998 if (isa<LoadInst>(I))
1999 PN = PHINode::Create(Ty: I.getType(), NumReservedValues: 2, NameStr: "", InsertBefore: I.getIterator());
2000 for (BasicBlock *SuccBB : successors(BB: Head)) {
2001 bool IsThen = SuccBB == HeadBI->getSuccessor(i: 0);
2002 int SuccIdx = IsThen ? 0 : 1;
2003 auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
2004 auto &CondMemOp = cast<T>(*I.clone());
2005 if (NewMemOpBB != Head) {
2006 NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
2007 if (isa<LoadInst>(I))
2008 ++NumLoadsPredicated;
2009 else
2010 ++NumStoresPredicated;
2011 } else {
2012 CondMemOp.dropUBImplyingAttrsAndMetadata();
2013 ++NumLoadsSpeculated;
2014 }
2015 CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
2016 Value *Ptr = SI.getOperand(i_nocapture: 1 + SuccIdx);
2017 // Casts may separate the memory operation from the select. Keep the
2018 // original access pointer type, e.g. its address space.
2019 IRB.SetInsertPoint(&CondMemOp);
2020 Ptr = IRB.CreateAddrSpaceCast(V: Ptr, DestTy: I.getPointerOperandType(),
2021 Name: I.getName() + (IsThen ? ".then" : ".else") +
2022 ".cast");
2023 CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
2024 if (isa<LoadInst>(I)) {
2025 CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
2026 PN->addIncoming(V: &CondMemOp, BB: NewMemOpBB);
2027 } else
2028 LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
2029 }
2030 if (isa<LoadInst>(I)) {
2031 PN->takeName(V: &I);
2032 LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
2033 I.replaceAllUsesWith(PN);
2034 }
2035}
2036
2037static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I,
2038 SelectHandSpeculativity Spec, IRBuilderTy &IRB,
2039 DomTreeUpdater &DTU) {
2040 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
2041 rewriteMemOpOfSelect(SI&: SelInst, I&: *LI, Spec, IRB, DTU);
2042 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
2043 rewriteMemOpOfSelect(SI&: SelInst, I&: *SI, Spec, IRB, DTU);
2044 else
2045 llvm_unreachable_internal(msg: "Only for load and store.");
2046}
2047
2048static bool rewriteSelectInstMemOps(SelectInst &SI,
2049 const RewriteableMemOps &Ops,
2050 IRBuilderTy &IRB, DomTreeUpdater *DTU) {
2051 bool CFGChanged = false;
2052 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
2053
2054 for (const RewriteableMemOp &Op : Ops) {
2055 SelectHandSpeculativity Spec;
2056 Instruction *I;
2057 if (auto *const *US = std::get_if<UnspeculatableStore>(ptr: &Op)) {
2058 I = *US;
2059 } else {
2060 auto PSL = std::get<PossiblySpeculatableLoad>(v: Op);
2061 I = PSL.getPointer();
2062 Spec = PSL.getInt();
2063 }
2064 if (Spec.areAllSpeculatable()) {
2065 speculateSelectInstLoads(SI, LI&: cast<LoadInst>(Val&: *I), IRB);
2066 } else {
2067 assert(DTU && "Should not get here when not allowed to modify the CFG!");
2068 rewriteMemOpOfSelect(SelInst&: SI, I&: *I, Spec, IRB, DTU&: *DTU);
2069 CFGChanged = true;
2070 }
2071 I->eraseFromParent();
2072 }
2073
2074 for (User *U : make_early_inc_range(Range: SI.users()))
2075 cast<AddrSpaceCastInst>(Val: U)->eraseFromParent();
2076 SI.eraseFromParent();
2077 return CFGChanged;
2078}
2079
2080/// Compute an adjusted pointer from Ptr by Offset bytes where the
2081/// resulting pointer has PointerTy.
2082static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
2083 APInt Offset, Type *PointerTy,
2084 const Twine &NamePrefix) {
2085 if (Offset != 0)
2086 Ptr = IRB.CreateInBoundsPtrAdd(Ptr, Offset: IRB.getInt(AI: Offset),
2087 Name: NamePrefix + "sroa_idx");
2088 return IRB.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: PointerTy,
2089 Name: NamePrefix + "sroa_cast");
2090}
2091
2092/// Compute the adjusted alignment for a load or store from an offset.
2093static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
2094 return commonAlignment(A: getLoadStoreAlignment(I), Offset);
2095}
2096
2097/// Test whether we can convert a value from the old to the new type.
2098///
2099/// This predicate should be used to guard calls to convertValue in order to
2100/// ensure that we only try to convert viable values. The strategy is that we
2101/// will peel off single element struct and array wrappings to get to an
2102/// underlying value, and convert that value.
2103static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
2104 unsigned VScale = 0) {
2105 if (OldTy == NewTy)
2106 return true;
2107
2108 // For integer types, we can't handle any bit-width differences. This would
2109 // break both vector conversions with extension and introduce endianness
2110 // issues when in conjunction with loads and stores.
2111 if (isa<IntegerType>(Val: OldTy) && isa<IntegerType>(Val: NewTy)) {
2112 assert(cast<IntegerType>(OldTy)->getBitWidth() !=
2113 cast<IntegerType>(NewTy)->getBitWidth() &&
2114 "We can't have the same bitwidth for different int types");
2115 return false;
2116 }
2117
2118 TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
2119 TypeSize OldSize = DL.getTypeSizeInBits(Ty: OldTy);
2120
2121 if ((isa<ScalableVectorType>(Val: NewTy) && isa<FixedVectorType>(Val: OldTy)) ||
2122 (isa<ScalableVectorType>(Val: OldTy) && isa<FixedVectorType>(Val: NewTy))) {
2123 // Conversion is only possible when the size of scalable vectors is known.
2124 if (!VScale)
2125 return false;
2126
2127 // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
2128 // a single domain (either fixed or scalable). Any additional conversion
2129 // between fixed and scalable types is handled through integer types.
2130 auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
2131 auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
2132
2133 if (isa<ScalableVectorType>(Val: NewTy)) {
2134 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: NewVTy), EltTy: OldVTy))
2135 return false;
2136
2137 NewSize = TypeSize::getFixed(ExactSize: NewSize.getKnownMinValue() * VScale);
2138 } else {
2139 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: OldVTy), EltTy: NewVTy))
2140 return false;
2141
2142 OldSize = TypeSize::getFixed(ExactSize: OldSize.getKnownMinValue() * VScale);
2143 }
2144 }
2145
2146 if (NewSize != OldSize)
2147 return false;
2148 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
2149 return false;
2150
2151 // We can convert pointers to integers and vice-versa. Same for vectors
2152 // of pointers and integers.
2153 OldTy = OldTy->getScalarType();
2154 NewTy = NewTy->getScalarType();
2155 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
2156 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
2157 unsigned OldAS = OldTy->getPointerAddressSpace();
2158 unsigned NewAS = NewTy->getPointerAddressSpace();
2159 // Convert pointers if they are pointers from the same address space or
2160 // different integral (not non-integral) address spaces with the same
2161 // pointer size.
2162 return OldAS == NewAS ||
2163 (!DL.isNonIntegralAddressSpace(AddrSpace: OldAS) &&
2164 !DL.isNonIntegralAddressSpace(AddrSpace: NewAS) &&
2165 DL.getPointerSize(AS: OldAS) == DL.getPointerSize(AS: NewAS));
2166 }
2167
2168 // We can convert integers to integral pointers, but not to non-integral
2169 // pointers.
2170 if (OldTy->isIntegerTy())
2171 return !DL.isNonIntegralPointerType(Ty: NewTy);
2172
2173 // We can convert integral pointers to integers, but non-integral pointers
2174 // need to remain pointers.
2175 if (!DL.isNonIntegralPointerType(Ty: OldTy))
2176 return NewTy->isIntegerTy();
2177
2178 return false;
2179 }
2180
2181 if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy())
2182 return false;
2183
2184 return true;
2185}
2186
2187/// Test whether the given slice use can be promoted to a vector.
2188///
2189/// This function is called to test each entry in a partition which is slated
2190/// for a single slice.
2191static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2192 VectorType *Ty,
2193 uint64_t ElementSize,
2194 const DataLayout &DL,
2195 unsigned VScale) {
2196 // First validate the slice offsets.
2197 uint64_t BeginOffset =
2198 std::max(a: S.beginOffset(), b: P.beginOffset()) - P.beginOffset();
2199 uint64_t BeginIndex = BeginOffset / ElementSize;
2200 if (BeginIndex * ElementSize != BeginOffset ||
2201 BeginIndex >= cast<FixedVectorType>(Val: Ty)->getNumElements())
2202 return false;
2203 uint64_t EndOffset = std::min(a: S.endOffset(), b: P.endOffset()) - P.beginOffset();
2204 uint64_t EndIndex = EndOffset / ElementSize;
2205 if (EndIndex * ElementSize != EndOffset ||
2206 EndIndex > cast<FixedVectorType>(Val: Ty)->getNumElements())
2207 return false;
2208
2209 assert(EndIndex > BeginIndex && "Empty vector!");
2210 uint64_t NumElements = EndIndex - BeginIndex;
2211 Type *SliceTy = (NumElements == 1)
2212 ? Ty->getElementType()
2213 : FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElements);
2214
2215 Type *SplitIntTy =
2216 Type::getIntNTy(C&: Ty->getContext(), N: NumElements * ElementSize * 8);
2217
2218 Use *U = S.getUse();
2219
2220 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2221 if (MI->isVolatile())
2222 return false;
2223 if (!S.isSplittable())
2224 return false; // Skip any unsplittable intrinsics.
2225 if (isa<MemSetInst>(Val: MI)) {
2226 Type *SplatTy = Type::getIntNTy(C&: Ty->getContext(), N: ElementSize * 8);
2227 if (!canConvertValue(DL, OldTy: SplatTy, NewTy: Ty->getElementType(), VScale))
2228 return false;
2229 }
2230 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2231 if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2232 return false;
2233 } else if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2234 if (LI->isVolatile())
2235 return false;
2236 Type *LTy = LI->getType();
2237 // Disable vector promotion when there are loads or stores of an FCA.
2238 if (LTy->isStructTy())
2239 return false;
2240 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2241 assert(LTy->isIntegerTy());
2242 LTy = SplitIntTy;
2243 }
2244 if (!canConvertValue(DL, OldTy: SliceTy, NewTy: LTy, VScale))
2245 return false;
2246 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2247 if (SI->isVolatile())
2248 return false;
2249 Type *STy = SI->getValueOperand()->getType();
2250 // Disable vector promotion when there are loads or stores of an FCA.
2251 if (STy->isStructTy())
2252 return false;
2253 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2254 assert(STy->isIntegerTy());
2255 STy = SplitIntTy;
2256 }
2257 if (!canConvertValue(DL, OldTy: STy, NewTy: SliceTy, VScale))
2258 return false;
2259 } else {
2260 return false;
2261 }
2262
2263 return true;
2264}
2265
2266/// Test whether any vector type in \p CandidateTys is viable for promotion.
2267///
2268/// This implements the necessary checking for \c isVectorPromotionViable over
2269/// all slices of the alloca for the given VectorType.
2270static VectorType *
2271checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
2272 SmallVectorImpl<VectorType *> &CandidateTys,
2273 bool HaveCommonEltTy, Type *CommonEltTy,
2274 bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2275 VectorType *CommonVecPtrTy, unsigned VScale) {
2276 // If we didn't find a vector type, nothing to do here.
2277 if (CandidateTys.empty())
2278 return nullptr;
2279
2280 // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2281 // then we should choose it, not some other alternative.
2282 // But, we can't perform a no-op pointer address space change via bitcast,
2283 // so if we didn't have a common pointer element type, bail.
2284 if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2285 return nullptr;
2286
2287 // Try to pick the "best" element type out of the choices.
2288 if (!HaveCommonEltTy && HaveVecPtrTy) {
2289 // If there was a pointer element type, there's really only one choice.
2290 CandidateTys.clear();
2291 CandidateTys.push_back(Elt: CommonVecPtrTy);
2292 } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2293 // Integer-ify vector types.
2294 for (VectorType *&VTy : CandidateTys) {
2295 if (!VTy->getElementType()->isIntegerTy())
2296 VTy = cast<VectorType>(Val: VTy->getWithNewType(EltTy: IntegerType::getIntNTy(
2297 C&: VTy->getContext(), N: VTy->getScalarSizeInBits())));
2298 }
2299
2300 // Rank the remaining candidate vector types. This is easy because we know
2301 // they're all integer vectors. We sort by ascending number of elements.
2302 auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2303 (void)DL;
2304 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2305 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2306 "Cannot have vector types of different sizes!");
2307 assert(RHSTy->getElementType()->isIntegerTy() &&
2308 "All non-integer types eliminated!");
2309 assert(LHSTy->getElementType()->isIntegerTy() &&
2310 "All non-integer types eliminated!");
2311 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() <
2312 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2313 };
2314 auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2315 (void)DL;
2316 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2317 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2318 "Cannot have vector types of different sizes!");
2319 assert(RHSTy->getElementType()->isIntegerTy() &&
2320 "All non-integer types eliminated!");
2321 assert(LHSTy->getElementType()->isIntegerTy() &&
2322 "All non-integer types eliminated!");
2323 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() ==
2324 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2325 };
2326 llvm::sort(C&: CandidateTys, Comp: RankVectorTypesComp);
2327 CandidateTys.erase(CS: llvm::unique(R&: CandidateTys, P: RankVectorTypesEq),
2328 CE: CandidateTys.end());
2329 } else {
2330// The only way to have the same element type in every vector type is to
2331// have the same vector type. Check that and remove all but one.
2332#ifndef NDEBUG
2333 for (VectorType *VTy : CandidateTys) {
2334 assert(VTy->getElementType() == CommonEltTy &&
2335 "Unaccounted for element type!");
2336 assert(VTy == CandidateTys[0] &&
2337 "Different vector types with the same element type!");
2338 }
2339#endif
2340 CandidateTys.resize(N: 1);
2341 }
2342
2343 // FIXME: hack. Do we have a named constant for this?
2344 // SDAG SDNode can't have more than 65535 operands.
2345 llvm::erase_if(C&: CandidateTys, P: [](VectorType *VTy) {
2346 return cast<FixedVectorType>(Val: VTy)->getNumElements() >
2347 std::numeric_limits<unsigned short>::max();
2348 });
2349
2350 // Find a vector type viable for promotion by iterating over all slices.
2351 auto *VTy = llvm::find_if(Range&: CandidateTys, P: [&](VectorType *VTy) -> bool {
2352 uint64_t ElementSize =
2353 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2354
2355 // While the definition of LLVM vectors is bitpacked, we don't support sizes
2356 // that aren't byte sized.
2357 if (ElementSize % 8)
2358 return false;
2359 assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&
2360 "vector size not a multiple of element size?");
2361 ElementSize /= 8;
2362
2363 for (const Slice &S : P)
2364 if (!isVectorPromotionViableForSlice(P, S, Ty: VTy, ElementSize, DL, VScale))
2365 return false;
2366
2367 for (const Slice *S : P.splitSliceTails())
2368 if (!isVectorPromotionViableForSlice(P, S: *S, Ty: VTy, ElementSize, DL, VScale))
2369 return false;
2370
2371 return true;
2372 });
2373 return VTy != CandidateTys.end() ? *VTy : nullptr;
2374}
2375
2376static VectorType *createAndCheckVectorTypesForPromotion(
2377 SetVector<Type *> &OtherTys, ArrayRef<VectorType *> CandidateTysCopy,
2378 function_ref<void(Type *)> CheckCandidateType, Partition &P,
2379 const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2380 bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
2381 bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
2382 [[maybe_unused]] VectorType *OriginalElt =
2383 CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
2384 // Consider additional vector types where the element type size is a
2385 // multiple of load/store element size.
2386 for (Type *Ty : OtherTys) {
2387 if (!VectorType::isValidElementType(ElemTy: Ty))
2388 continue;
2389 unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2390 // Make a copy of CandidateTys and iterate through it, because we
2391 // might append to CandidateTys in the loop.
2392 for (VectorType *const VTy : CandidateTysCopy) {
2393 // The elements in the copy should remain invariant throughout the loop
2394 assert(CandidateTysCopy[0] == OriginalElt && "Different Element");
2395 unsigned VectorSize = DL.getTypeSizeInBits(Ty: VTy).getFixedValue();
2396 unsigned ElementSize =
2397 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2398 if (TypeSize != VectorSize && TypeSize != ElementSize &&
2399 VectorSize % TypeSize == 0) {
2400 VectorType *NewVTy = VectorType::get(ElementType: Ty, NumElements: VectorSize / TypeSize, Scalable: false);
2401 CheckCandidateType(NewVTy);
2402 }
2403 }
2404 }
2405
2406 return checkVectorTypesForPromotion(
2407 P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2408 HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2409}
2410
2411/// Test whether the given alloca partitioning and range of slices can be
2412/// promoted to a vector.
2413///
2414/// This is a quick test to check whether we can rewrite a particular alloca
2415/// partition (and its newly formed alloca) into a vector alloca with only
2416/// whole-vector loads and stores such that it could be promoted to a vector
2417/// SSA value. We only can ensure this for a limited set of operations, and we
2418/// don't want to do the rewrites unless we are confident that the result will
2419/// be promotable, so we have an early test here.
2420static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL,
2421 unsigned VScale) {
2422 // Collect the candidate types for vector-based promotion. Also track whether
2423 // we have different element types.
2424 SmallVector<VectorType *, 4> CandidateTys;
2425 SetVector<Type *> LoadStoreTys;
2426 SetVector<Type *> DeferredTys;
2427 Type *CommonEltTy = nullptr;
2428 VectorType *CommonVecPtrTy = nullptr;
2429 bool HaveVecPtrTy = false;
2430 bool HaveCommonEltTy = true;
2431 bool HaveCommonVecPtrTy = true;
2432 auto CheckCandidateType = [&](Type *Ty) {
2433 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2434 // Return if bitcast to vectors is different for total size in bits.
2435 if (!CandidateTys.empty()) {
2436 VectorType *V = CandidateTys[0];
2437 if (DL.getTypeSizeInBits(Ty: VTy).getFixedValue() !=
2438 DL.getTypeSizeInBits(Ty: V).getFixedValue()) {
2439 CandidateTys.clear();
2440 return;
2441 }
2442 }
2443 CandidateTys.push_back(Elt: VTy);
2444 Type *EltTy = VTy->getElementType();
2445
2446 if (!CommonEltTy)
2447 CommonEltTy = EltTy;
2448 else if (CommonEltTy != EltTy)
2449 HaveCommonEltTy = false;
2450
2451 if (EltTy->isPointerTy()) {
2452 HaveVecPtrTy = true;
2453 if (!CommonVecPtrTy)
2454 CommonVecPtrTy = VTy;
2455 else if (CommonVecPtrTy != VTy)
2456 HaveCommonVecPtrTy = false;
2457 }
2458 }
2459 };
2460
2461 // Put load and store types into a set for de-duplication.
2462 for (const Slice &S : P) {
2463 Type *Ty;
2464 if (auto *LI = dyn_cast<LoadInst>(Val: S.getUse()->getUser()))
2465 Ty = LI->getType();
2466 else if (auto *SI = dyn_cast<StoreInst>(Val: S.getUse()->getUser()))
2467 Ty = SI->getValueOperand()->getType();
2468 else
2469 continue;
2470
2471 auto CandTy = Ty->getScalarType();
2472 if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() ||
2473 S.endOffset() != P.endOffset())) {
2474 DeferredTys.insert(X: Ty);
2475 continue;
2476 }
2477
2478 LoadStoreTys.insert(X: Ty);
2479 // Consider any loads or stores that are the exact size of the slice.
2480 if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2481 CheckCandidateType(Ty);
2482 }
2483
2484 SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
2485 if (auto *VTy = createAndCheckVectorTypesForPromotion(
2486 OtherTys&: LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2487 CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2488 HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2489 return VTy;
2490
2491 CandidateTys.clear();
2492 return createAndCheckVectorTypesForPromotion(
2493 OtherTys&: DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2494 HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2495 CommonVecPtrTy, VScale);
2496}
2497
2498/// Test whether a slice of an alloca is valid for integer widening.
2499///
2500/// This implements the necessary checking for the \c isIntegerWideningViable
2501/// test below on a single slice of the alloca.
2502static bool isIntegerWideningViableForSlice(const Slice &S,
2503 uint64_t AllocBeginOffset,
2504 Type *AllocaTy,
2505 const DataLayout &DL,
2506 bool &WholeAllocaOp) {
2507 uint64_t Size = DL.getTypeStoreSize(Ty: AllocaTy).getFixedValue();
2508
2509 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2510 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2511
2512 Use *U = S.getUse();
2513
2514 // Lifetime intrinsics operate over the whole alloca whose sizes are usually
2515 // larger than other load/store slices (RelEnd > Size). But lifetime are
2516 // always promotable and should not impact other slices' promotability of the
2517 // partition.
2518 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2519 if (II->isLifetimeStartOrEnd() || II->isDroppable())
2520 return true;
2521 }
2522
2523 // We can't reasonably handle cases where the load or store extends past
2524 // the end of the alloca's type and into its padding.
2525 if (RelEnd > Size)
2526 return false;
2527
2528 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2529 if (LI->isVolatile())
2530 return false;
2531 // We can't handle loads that extend past the allocated memory.
2532 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
2533 if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
2534 return false;
2535 // So far, AllocaSliceRewriter does not support widening split slice tails
2536 // in rewriteIntegerLoad.
2537 if (S.beginOffset() < AllocBeginOffset)
2538 return false;
2539 // Note that we don't count vector loads or stores as whole-alloca
2540 // operations which enable integer widening because we would prefer to use
2541 // vector widening instead.
2542 if (!isa<VectorType>(Val: LI->getType()) && RelBegin == 0 && RelEnd == Size)
2543 WholeAllocaOp = true;
2544 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: LI->getType())) {
2545 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2546 return false;
2547 } else if (RelBegin != 0 || RelEnd != Size ||
2548 !canConvertValue(DL, OldTy: AllocaTy, NewTy: LI->getType())) {
2549 // Non-integer loads need to be convertible from the alloca type so that
2550 // they are promotable.
2551 return false;
2552 }
2553 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2554 Type *ValueTy = SI->getValueOperand()->getType();
2555 if (SI->isVolatile())
2556 return false;
2557 // We can't handle stores that extend past the allocated memory.
2558 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValueTy);
2559 if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
2560 return false;
2561 // So far, AllocaSliceRewriter does not support widening split slice tails
2562 // in rewriteIntegerStore.
2563 if (S.beginOffset() < AllocBeginOffset)
2564 return false;
2565 // Note that we don't count vector loads or stores as whole-alloca
2566 // operations which enable integer widening because we would prefer to use
2567 // vector widening instead.
2568 if (!isa<VectorType>(Val: ValueTy) && RelBegin == 0 && RelEnd == Size)
2569 WholeAllocaOp = true;
2570 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: ValueTy)) {
2571 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2572 return false;
2573 } else if (RelBegin != 0 || RelEnd != Size ||
2574 !canConvertValue(DL, OldTy: ValueTy, NewTy: AllocaTy)) {
2575 // Non-integer stores need to be convertible to the alloca type so that
2576 // they are promotable.
2577 return false;
2578 }
2579 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2580 if (MI->isVolatile() || !isa<Constant>(Val: MI->getLength()))
2581 return false;
2582 if (!S.isSplittable())
2583 return false; // Skip any unsplittable intrinsics.
2584 } else {
2585 return false;
2586 }
2587
2588 return true;
2589}
2590
2591/// Test whether the given alloca partition's integer operations can be
2592/// widened to promotable ones.
2593///
2594/// This is a quick test to check whether we can rewrite the integer loads and
2595/// stores to a particular alloca into wider loads and stores and be able to
2596/// promote the resulting alloca.
2597static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2598 const DataLayout &DL) {
2599 uint64_t SizeInBits = DL.getTypeSizeInBits(Ty: AllocaTy).getFixedValue();
2600 // Don't create integer types larger than the maximum bitwidth.
2601 if (SizeInBits > IntegerType::MAX_INT_BITS)
2602 return false;
2603
2604 // Don't try to handle allocas with bit-padding.
2605 if (SizeInBits != DL.getTypeStoreSizeInBits(Ty: AllocaTy).getFixedValue())
2606 return false;
2607
2608 // We need to ensure that an integer type with the appropriate bitwidth can
2609 // be converted to the alloca type, whatever that is. We don't want to force
2610 // the alloca itself to have an integer type if there is a more suitable one.
2611 Type *IntTy = Type::getIntNTy(C&: AllocaTy->getContext(), N: SizeInBits);
2612 if (!canConvertValue(DL, OldTy: AllocaTy, NewTy: IntTy) ||
2613 !canConvertValue(DL, OldTy: IntTy, NewTy: AllocaTy))
2614 return false;
2615
2616 // While examining uses, we ensure that the alloca has a covering load or
2617 // store. We don't want to widen the integer operations only to fail to
2618 // promote due to some other unsplittable entry (which we may make splittable
2619 // later). However, if there are only splittable uses, go ahead and assume
2620 // that we cover the alloca.
2621 // FIXME: We shouldn't consider split slices that happen to start in the
2622 // partition here...
2623 bool WholeAllocaOp = P.empty() && DL.isLegalInteger(Width: SizeInBits);
2624
2625 for (const Slice &S : P)
2626 if (!isIntegerWideningViableForSlice(S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2627 WholeAllocaOp))
2628 return false;
2629
2630 for (const Slice *S : P.splitSliceTails())
2631 if (!isIntegerWideningViableForSlice(S: *S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2632 WholeAllocaOp))
2633 return false;
2634
2635 return WholeAllocaOp;
2636}
2637
2638static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
2639 IntegerType *Ty, uint64_t Offset,
2640 const Twine &Name) {
2641 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2642 IntegerType *IntTy = cast<IntegerType>(Val: V->getType());
2643 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2644 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2645 "Element extends past full value");
2646 uint64_t ShAmt = 8 * Offset;
2647 if (DL.isBigEndian())
2648 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2649 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2650 if (ShAmt) {
2651 V = IRB.CreateLShr(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2652 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2653 }
2654 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2655 "Cannot extract to a larger integer!");
2656 if (Ty != IntTy) {
2657 V = IRB.CreateTrunc(V, DestTy: Ty, Name: Name + ".trunc");
2658 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2659 }
2660 return V;
2661}
2662
2663static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2664 Value *V, uint64_t Offset, const Twine &Name) {
2665 IntegerType *IntTy = cast<IntegerType>(Val: Old->getType());
2666 IntegerType *Ty = cast<IntegerType>(Val: V->getType());
2667 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2668 "Cannot insert a larger integer!");
2669 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2670 if (Ty != IntTy) {
2671 V = IRB.CreateZExt(V, DestTy: IntTy, Name: Name + ".ext");
2672 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2673 }
2674 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2675 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2676 "Element store outside of alloca store");
2677 uint64_t ShAmt = 8 * Offset;
2678 if (DL.isBigEndian())
2679 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2680 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2681 if (ShAmt) {
2682 V = IRB.CreateShl(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2683 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2684 }
2685
2686 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2687 APInt Mask = ~Ty->getMask().zext(width: IntTy->getBitWidth()).shl(shiftAmt: ShAmt);
2688 Old = IRB.CreateAnd(LHS: Old, RHS: Mask, Name: Name + ".mask");
2689 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2690 V = IRB.CreateOr(LHS: Old, RHS: V, Name: Name + ".insert");
2691 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2692 }
2693 return V;
2694}
2695
2696static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2697 unsigned EndIndex, const Twine &Name) {
2698 auto *VecTy = cast<FixedVectorType>(Val: V->getType());
2699 unsigned NumElements = EndIndex - BeginIndex;
2700 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2701
2702 if (NumElements == VecTy->getNumElements())
2703 return V;
2704
2705 if (NumElements == 1) {
2706 V = IRB.CreateExtractElement(Vec: V, Idx: BeginIndex, Name: Name + ".extract");
2707 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2708 return V;
2709 }
2710
2711 auto Mask = llvm::to_vector<8>(Range: llvm::seq<int>(Begin: BeginIndex, End: EndIndex));
2712 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".extract");
2713 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2714 return V;
2715}
2716
2717static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
2718 unsigned BeginIndex, const Twine &Name) {
2719 VectorType *VecTy = cast<VectorType>(Val: Old->getType());
2720 assert(VecTy && "Can only insert a vector into a vector");
2721
2722 VectorType *Ty = dyn_cast<VectorType>(Val: V->getType());
2723 if (!Ty) {
2724 // Single element to insert.
2725 V = IRB.CreateInsertElement(Vec: Old, NewElt: V, Idx: BeginIndex, Name: Name + ".insert");
2726 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2727 return V;
2728 }
2729
2730 unsigned NumSubElements = cast<FixedVectorType>(Val: Ty)->getNumElements();
2731 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2732
2733 assert(NumSubElements <= NumElements && "Too many elements!");
2734 if (NumSubElements == NumElements) {
2735 assert(V->getType() == VecTy && "Vector type mismatch");
2736 return V;
2737 }
2738 unsigned EndIndex = BeginIndex + NumSubElements;
2739
2740 // When inserting a smaller vector into the larger to store, we first
2741 // use a shuffle vector to widen it with undef elements, and then
2742 // a second shuffle vector to select between the loaded vector and the
2743 // incoming vector.
2744 SmallVector<int, 8> Mask;
2745 Mask.reserve(N: NumElements);
2746 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2747 if (Idx >= BeginIndex && Idx < EndIndex)
2748 Mask.push_back(Elt: Idx - BeginIndex);
2749 else
2750 Mask.push_back(Elt: -1);
2751 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".expand");
2752 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2753
2754 Mask.clear();
2755 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2756 if (Idx >= BeginIndex && Idx < EndIndex)
2757 Mask.push_back(Elt: Idx);
2758 else
2759 Mask.push_back(Elt: Idx + NumElements);
2760 V = IRB.CreateShuffleVector(V1: V, V2: Old, Mask, Name: Name + "blend");
2761 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2762 return V;
2763}
2764
2765/// This function takes two vector values and combines them into a single vector
2766/// by concatenating their elements. The function handles:
2767///
2768/// 1. Element type mismatch: If either vector's element type differs from
2769/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2770/// preserving the total bit width (adjusting the number of elements
2771/// accordingly).
2772///
2773/// 2. Size mismatch: After transforming the vectors to have the desired element
2774/// type, if the two vectors have different numbers of elements, the smaller
2775/// vector is extended with poison values to match the size of the larger
2776/// vector before concatenation.
2777///
2778/// 3. Concatenation: The vectors are merged using a shuffle operation that
2779/// places all elements of V0 first, followed by all elements of V1.
2780///
2781/// \param V0 The first vector to merge (must be a vector type)
2782/// \param V1 The second vector to merge (must be a vector type)
2783/// \param DL The data layout for size calculations
2784/// \param NewAIEltTy The desired element type for the result vector
2785/// \param Builder IRBuilder for creating new instructions
2786/// \return A new vector containing all elements from V0 followed by all
2787/// elements from V1
2788static Value *mergeTwoVectors(Value *V0, Value *V1, const DataLayout &DL,
2789 Type *NewAIEltTy, IRBuilder<> &Builder) {
2790 // V0 and V1 are vectors
2791 // Create a new vector type with combined elements
2792 // Use ShuffleVector to concatenate the vectors
2793 auto *VecType0 = cast<FixedVectorType>(Val: V0->getType());
2794 auto *VecType1 = cast<FixedVectorType>(Val: V1->getType());
2795
2796 // If V0/V1 element types are different from NewAllocaElementType,
2797 // we need to introduce bitcasts before merging them
2798 auto BitcastIfNeeded = [&](Value *&V, FixedVectorType *&VecType,
2799 const char *DebugName) {
2800 Type *EltType = VecType->getElementType();
2801 if (EltType != NewAIEltTy) {
2802 // Calculate new number of elements to maintain same bit width
2803 unsigned TotalBits =
2804 VecType->getNumElements() * DL.getTypeSizeInBits(Ty: EltType);
2805 unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(Ty: NewAIEltTy);
2806
2807 auto *NewVecType = FixedVectorType::get(ElementType: NewAIEltTy, NumElts: NewNumElts);
2808 V = Builder.CreateBitCast(V, DestTy: NewVecType);
2809 VecType = NewVecType;
2810 LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2811 }
2812 };
2813
2814 BitcastIfNeeded(V0, VecType0, "V0");
2815 BitcastIfNeeded(V1, VecType1, "V1");
2816
2817 unsigned NumElts0 = VecType0->getNumElements();
2818 unsigned NumElts1 = VecType1->getNumElements();
2819
2820 SmallVector<int, 16> ShuffleMask;
2821
2822 if (NumElts0 == NumElts1) {
2823 for (unsigned i = 0; i < NumElts0 + NumElts1; ++i)
2824 ShuffleMask.push_back(Elt: i);
2825 } else {
2826 // If two vectors have different sizes, we need to extend
2827 // the smaller vector to the size of the larger vector.
2828 unsigned SmallSize = std::min(a: NumElts0, b: NumElts1);
2829 unsigned LargeSize = std::max(a: NumElts0, b: NumElts1);
2830 bool IsV0Smaller = NumElts0 < NumElts1;
2831 Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2832 SmallVector<int, 16> ExtendMask;
2833 for (unsigned i = 0; i < SmallSize; ++i)
2834 ExtendMask.push_back(Elt: i);
2835 for (unsigned i = SmallSize; i < LargeSize; ++i)
2836 ExtendMask.push_back(Elt: PoisonMaskElem);
2837 ExtendedVec = Builder.CreateShuffleVector(
2838 V1: ExtendedVec, V2: PoisonValue::get(T: ExtendedVec->getType()), Mask: ExtendMask);
2839 LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2840 for (unsigned i = 0; i < NumElts0; ++i)
2841 ShuffleMask.push_back(Elt: i);
2842 for (unsigned i = 0; i < NumElts1; ++i)
2843 ShuffleMask.push_back(Elt: LargeSize + i);
2844 }
2845
2846 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
2847}
2848
2849namespace {
2850
2851/// Visitor to rewrite instructions using p particular slice of an alloca
2852/// to use a new alloca.
2853///
2854/// Also implements the rewriting to vector-based accesses when the partition
2855/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2856/// lives here.
2857class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2858 // Befriend the base class so it can delegate to private visit methods.
2859 friend class InstVisitor<AllocaSliceRewriter, bool>;
2860
2861 using Base = InstVisitor<AllocaSliceRewriter, bool>;
2862
2863 const DataLayout &DL;
2864 AllocaSlices &AS;
2865 SROA &Pass;
2866 AllocaInst &OldAI, &NewAI;
2867 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2868 Type *NewAllocaTy;
2869
2870 // This is a convenience and flag variable that will be null unless the new
2871 // alloca's integer operations should be widened to this integer type due to
2872 // passing isIntegerWideningViable above. If it is non-null, the desired
2873 // integer type will be stored here for easy access during rewriting.
2874 IntegerType *IntTy;
2875
2876 // If we are rewriting an alloca partition which can be written as pure
2877 // vector operations, we stash extra information here. When VecTy is
2878 // non-null, we have some strict guarantees about the rewritten alloca:
2879 // - The new alloca is exactly the size of the vector type here.
2880 // - The accesses all either map to the entire vector or to a single
2881 // element.
2882 // - The set of accessing instructions is only one of those handled above
2883 // in isVectorPromotionViable. Generally these are the same access kinds
2884 // which are promotable via mem2reg.
2885 VectorType *VecTy;
2886 Type *ElementTy;
2887 uint64_t ElementSize;
2888
2889 // The original offset of the slice currently being rewritten relative to
2890 // the original alloca.
2891 uint64_t BeginOffset = 0;
2892 uint64_t EndOffset = 0;
2893
2894 // The new offsets of the slice currently being rewritten relative to the
2895 // original alloca.
2896 uint64_t NewBeginOffset = 0, NewEndOffset = 0;
2897
2898 uint64_t SliceSize = 0;
2899 bool IsSplittable = false;
2900 bool IsSplit = false;
2901 Use *OldUse = nullptr;
2902 Instruction *OldPtr = nullptr;
2903
2904 // Track post-rewrite users which are PHI nodes and Selects.
2905 SmallSetVector<PHINode *, 8> &PHIUsers;
2906 SmallSetVector<SelectInst *, 8> &SelectUsers;
2907
2908 // Utility IR builder, whose name prefix is setup for each visited use, and
2909 // the insertion point is set to point to the user.
2910 IRBuilderTy IRB;
2911
2912 // Return the new alloca, addrspacecasted if required to avoid changing the
2913 // addrspace of a volatile access.
2914 Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) {
2915 if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace())
2916 return &NewAI;
2917
2918 Type *AccessTy = IRB.getPtrTy(AddrSpace);
2919 return IRB.CreateAddrSpaceCast(V: &NewAI, DestTy: AccessTy);
2920 }
2921
2922public:
2923 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2924 AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2925 uint64_t NewAllocaBeginOffset,
2926 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2927 VectorType *PromotableVecTy,
2928 SmallSetVector<PHINode *, 8> &PHIUsers,
2929 SmallSetVector<SelectInst *, 8> &SelectUsers)
2930 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2931 NewAllocaBeginOffset(NewAllocaBeginOffset),
2932 NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2933 IntTy(IsIntegerPromotable
2934 ? Type::getIntNTy(
2935 C&: NewAI.getContext(),
2936 N: DL.getTypeSizeInBits(Ty: NewAllocaTy).getFixedValue())
2937 : nullptr),
2938 VecTy(PromotableVecTy),
2939 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2940 ElementSize(VecTy ? DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8
2941 : 0),
2942 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2943 IRB(*NewAI.getModule(), ConstantFolder()) {
2944 if (VecTy) {
2945 assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
2946 "Only multiple-of-8 sized vector elements are viable");
2947 ++NumVectorized;
2948 }
2949 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
2950 }
2951
2952 bool visit(AllocaSlices::const_iterator I) {
2953 bool CanSROA = true;
2954 BeginOffset = I->beginOffset();
2955 EndOffset = I->endOffset();
2956 IsSplittable = I->isSplittable();
2957 IsSplit =
2958 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
2959 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2960 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2961 LLVM_DEBUG(dbgs() << "\n");
2962
2963 // Compute the intersecting offset range.
2964 assert(BeginOffset < NewAllocaEndOffset);
2965 assert(EndOffset > NewAllocaBeginOffset);
2966 NewBeginOffset = std::max(a: BeginOffset, b: NewAllocaBeginOffset);
2967 NewEndOffset = std::min(a: EndOffset, b: NewAllocaEndOffset);
2968
2969 SliceSize = NewEndOffset - NewBeginOffset;
2970 LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2971 << ") NewBegin:(" << NewBeginOffset << ", "
2972 << NewEndOffset << ") NewAllocaBegin:("
2973 << NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2974 << ")\n");
2975 assert(IsSplit || NewBeginOffset == BeginOffset);
2976 OldUse = I->getUse();
2977 OldPtr = cast<Instruction>(Val: OldUse->get());
2978
2979 Instruction *OldUserI = cast<Instruction>(Val: OldUse->getUser());
2980 IRB.SetInsertPoint(OldUserI);
2981 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2982 // Avoid materializing the name prefix when it is discarded anyway.
2983 if (!IRB.getContext().shouldDiscardValueNames())
2984 IRB.getInserter().SetNamePrefix(Twine(NewAI.getName()) + "." +
2985 Twine(BeginOffset) + ".");
2986
2987 CanSROA &= visit(I: cast<Instruction>(Val: OldUse->getUser()));
2988 if (VecTy || IntTy)
2989 assert(CanSROA);
2990 return CanSROA;
2991 }
2992
2993 /// Attempts to rewrite a partition using tree-structured merge optimization.
2994 ///
2995 /// This function handles two patterns. Both produce an O(log n) tree of
2996 /// shufflevectors in place of the linear expand+blend chain that SROA would
2997 /// otherwise emit for each partial store.
2998 ///
2999 /// Pattern 1 (stores-only):
3000 /// Multiple non-overlapping partial stores completely fill the alloca
3001 /// and there is exactly one full-width load coming after the stores.
3002 /// The stores are tree-merged into a single vector and stored once.
3003 ///
3004 /// Example transformation:
3005 /// Before: (stores do not have to be in order)
3006 /// %alloca = alloca <8 x float>
3007 /// store <2 x float> %val0, ptr %alloca ; offset 0-1
3008 /// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
3009 /// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
3010 /// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
3011 /// %r = load <8 x float>, ptr %alloca
3012 ///
3013 /// After: tree of shufflevectors producing <8 x float> directly.
3014 ///
3015 /// Pattern 2 (init + RMW, possibly multi-round):
3016 /// A single full-width init store, followed by partial loads and
3017 /// partial stores that read-modify-write the alloca one or more
3018 /// times, optionally followed by a full-width load. The only
3019 /// structural requirement is that the distinct [begin, end) ranges
3020 /// touched by the partial loads and stores, taken together, tile
3021 /// the alloca disjointly.
3022 ///
3023 /// We keep a map from each slice range to the SSA value that
3024 /// currently lives there, `SliceValues[r] -> Value*`:
3025 /// - initialize each entry to the corresponding piece of the
3026 /// init store's value (via a shufflevector picking the
3027 /// range's elements out of the init value),
3028 /// - walk partial loads and stores in block order,
3029 /// - for a partial load at range r: RAUW with `SliceValues[r]`,
3030 /// - for a partial store at range r: update `SliceValues[r]` to
3031 /// the stored value and drop the store.
3032 /// At the end, the final `SliceValues[r]` entries are tree-merged
3033 /// (in range order) into a single store to the alloca, and the
3034 /// optional full-width load is replaced by a load of the alloca.
3035 ///
3036 /// Because the ranges are disjoint by construction, a store at one
3037 /// range cannot affect another range's tracked value, so a single
3038 /// block-order walk correctly tracks the memory state at each
3039 /// range. The algorithm handles multi-round RMW, partial loads
3040 /// and stores interleaved in any order, read-only slices (the
3041 /// tracked value stays at the init extract), and write-only
3042 /// slices (the tracked value never flows into a load).
3043 ///
3044 /// \param P The partition to analyze and potentially rewrite
3045 /// \return An optional vector of values that were deleted during the
3046 /// rewrite, or std::nullopt if the partition cannot be optimized.
3047 std::optional<SmallVector<Value *, 4>>
3048 rewriteTreeStructuredMerge(Partition &P) {
3049 // No tail slices that overlap with the partition
3050 if (P.splitSliceTails().size() > 0)
3051 return std::nullopt;
3052
3053 // Structure to hold store information
3054 struct StoreInfo {
3055 StoreInst *Store;
3056 uint64_t BeginOffset;
3057 uint64_t EndOffset;
3058 Value *StoredValue;
3059 StoreInfo(StoreInst *SI, uint64_t Begin, uint64_t End, Value *Val)
3060 : Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
3061 };
3062 struct LoadInfo {
3063 LoadInst *Load;
3064 uint64_t BeginOffset;
3065 uint64_t EndOffset;
3066 };
3067
3068 SmallVector<StoreInfo, 4> StoreInfos; // partial stores only
3069 SmallVector<LoadInfo, 4> LoadInfos; // partial loads only
3070 LoadInst *FullLoad = nullptr; // optional full-width load
3071 StoreInst *InitStore = nullptr; // optional full-width init store
3072
3073 // If the new alloca is a fixed vector type, we use its element type as the
3074 // allocated element type, otherwise we use i8 as the allocated element
3075 Type *AllocatedEltTy =
3076 isa<FixedVectorType>(Val: NewAllocaTy)
3077 ? cast<FixedVectorType>(Val: NewAllocaTy)->getElementType()
3078 : Type::getInt8Ty(C&: NewAI.getContext());
3079 unsigned AllocatedEltTySize = DL.getTypeSizeInBits(Ty: AllocatedEltTy);
3080
3081 // Helper to check if a type is
3082 // 1. A fixed vector type
3083 // 2. The element type is not a pointer
3084 // 3. The element type size is byte-aligned
3085 // We only handle the cases that the ld/st meet these conditions
3086 auto IsTypeValidForTreeStructuredMerge = [&](Type *Ty) -> bool {
3087 auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: Ty);
3088 return FixedVecTy &&
3089 DL.getTypeSizeInBits(Ty: FixedVecTy->getElementType()) % 8 == 0 &&
3090 !FixedVecTy->getElementType()->isPointerTy();
3091 };
3092
3093 for (Slice &S : P) {
3094 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
3095 // A "full-width" slice spans the entire alloca; it's either the single
3096 // init store (Pattern 2) or the single final load (both patterns).
3097 bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
3098 S.endOffset() == NewAllocaEndOffset);
3099 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
3100 // Only handle simple (non-volatile, non-atomic) loads.
3101 if (!LI->isSimple() ||
3102 !IsTypeValidForTreeStructuredMerge(LI->getType()))
3103 return std::nullopt;
3104 if (IsFullWidth) {
3105 // We accept at most one full-width load (the "final" load, after
3106 // all the partial stores).
3107 if (FullLoad)
3108 return std::nullopt;
3109 FullLoad = LI;
3110 } else {
3111 // Partial load (RMW pattern only).
3112 LoadInfos.push_back(Elt: {.Load: LI, .BeginOffset: S.beginOffset(), .EndOffset: S.endOffset()});
3113 }
3114 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
3115 // Do not handle the case if
3116 // 1. The store does not meet the conditions in the helper function
3117 // 2. The store is not simple — we drop stores as part of the
3118 // rewrite, so volatile stores (which must be kept) and atomic
3119 // stores (which carry memory-ordering semantics) are unsound
3120 // to replace with SSA bookkeeping.
3121 // 3. The total store size is not a multiple of the allocated
3122 // element type size (required so the tree merge can produce a
3123 // vector whose element type matches the alloca).
3124 if (!SI->isSimple() || !IsTypeValidForTreeStructuredMerge(
3125 SI->getValueOperand()->getType()))
3126 return std::nullopt;
3127 auto *StVecTy = cast<FixedVectorType>(Val: SI->getValueOperand()->getType());
3128 unsigned NumElts = StVecTy->getNumElements();
3129 unsigned EltSize = DL.getTypeSizeInBits(Ty: StVecTy->getElementType());
3130 if (NumElts * EltSize % AllocatedEltTySize != 0)
3131 return std::nullopt;
3132 if (IsFullWidth) {
3133 // At most one full-width store is allowed — it's the init store
3134 // for the RMW pattern.
3135 if (InitStore)
3136 return std::nullopt;
3137 InitStore = SI;
3138 } else {
3139 StoreInfos.emplace_back(Args&: SI, Args: S.beginOffset(), Args: S.endOffset(),
3140 Args: SI->getValueOperand());
3141 }
3142 } else {
3143 // If we have instructions other than load and store, we cannot do
3144 // the tree structured merge.
3145 return std::nullopt;
3146 }
3147 }
3148
3149 // Need at least two partial stores to benefit from tree-merging; a
3150 // single store is already optimal as-is. This applies to both patterns
3151 // below, so check it before classifying.
3152 if (StoreInfos.size() < 2)
3153 return std::nullopt;
3154
3155 // Classify the pattern by looking at what we collected:
3156 // Pattern 1 (stores-only): only partial stores + exactly one full load.
3157 // Pattern 2 (RMW): one full init store + partial loads + partial stores
3158 // (+ optional full final load). RMW also needs VecTy to be set
3159 // because we use getIndex() to convert byte offsets to element
3160 // indices, which requires a promoted vector alloca.
3161 bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
3162 bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
3163 if (!IsRMWPattern && !IsStoresOnlyPattern)
3164 return std::nullopt;
3165
3166 // All partial stores must live in the same basic block — the tree merge
3167 // is built in a single BB using block-order ordering (comesBefore).
3168 BasicBlock *StoreBB = StoreInfos[0].Store->getParent();
3169 for (auto &Info : StoreInfos)
3170 if (Info.Store->getParent() != StoreBB)
3171 return std::nullopt;
3172
3173 SmallVector<Value *, 4> DeletedValues;
3174
3175 // Helper: pairwise tree-merge a list of vectors into a single vector.
3176 // At each iteration we merge each adjacent pair via mergeTwoVectors,
3177 // collect the merged values into Next, and (if Vals had odd length)
3178 // carry the trailing element through unchanged. Loop until one value
3179 // remains — the fully-merged vector.
3180 auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3181 IRBuilder<> &B) -> Value * {
3182 LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3183 while (Vals.size() > 1) {
3184 SmallVector<Value *, 8> Next;
3185 for (unsigned I = 0, E = Vals.size(); I + 1 < E; I += 2) {
3186 Value *M =
3187 mergeTwoVectors(V0: Vals[I], V1: Vals[I + 1], DL, NewAIEltTy: AllocatedEltTy, Builder&: B);
3188 LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3189 Next.push_back(Elt: M);
3190 }
3191 if (Vals.size() % 2 == 1)
3192 Next.push_back(Elt: Vals.back());
3193 Vals = std::move(Next);
3194 }
3195 return Vals[0];
3196 };
3197
3198 // Replace a full-width load with a load of the freshly-merged alloca.
3199 // The merge stored a value of type Merged->getType() into NewAI; we load
3200 // that same type back so every access to NewAI stays consistently typed
3201 // (otherwise the alloca is no longer promotable).
3202 auto ReplaceFullLoad = [&](LoadInst *LoadToReplace, Value *Merged) {
3203 IRBuilder<> LoadBuilder(LoadToReplace);
3204 Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3205 Ty: Merged->getType(), Ptr: &NewAI, Align: getSliceAlign(),
3206 isVolatile: LoadToReplace->isVolatile(),
3207 Name: LoadToReplace->getName() + ".sroa.new.load");
3208 if (NewLoad->getType() != LoadToReplace->getType())
3209 NewLoad = LoadBuilder.CreateBitCast(V: NewLoad, DestTy: LoadToReplace->getType());
3210 LoadToReplace->replaceAllUsesWith(V: NewLoad);
3211 DeletedValues.push_back(Elt: LoadToReplace);
3212 };
3213
3214 if (IsStoresOnlyPattern) {
3215 // Stores should not overlap and should cover the whole alloca.
3216 // Sort by begin offset to verify this with a single linear scan.
3217 llvm::sort(C&: StoreInfos, Comp: [](const StoreInfo &A, const StoreInfo &B) {
3218 return A.BeginOffset < B.BeginOffset;
3219 });
3220 // Check for gap or overlap: each begin offset must equal the previous
3221 // end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3222 // NewAllocaEndOffset) exactly.
3223 uint64_t Expected = NewAllocaBeginOffset;
3224 for (auto &Info : StoreInfos) {
3225 if (Info.BeginOffset != Expected)
3226 return std::nullopt;
3227 Expected = Info.EndOffset;
3228 }
3229 // Stores cover the entire alloca (no trailing gap either).
3230 if (Expected != NewAllocaEndOffset)
3231 return std::nullopt;
3232
3233 // The load should not be in the middle of the stores.
3234 // Note:
3235 // If the load is in a different basic block from the stores, we can
3236 // still do the tree-structured merge. We don't have store->load
3237 // forwarding here — the merged vector is stored back to NewAI and
3238 // the new load loads from NewAI. The forwarding will be handled
3239 // later when NewAI is promoted.
3240 BasicBlock *LoadBB = FullLoad->getParent();
3241 if (LoadBB == StoreBB) {
3242 for (auto &Info : StoreInfos)
3243 if (!Info.Store->comesBefore(Other: FullLoad))
3244 return std::nullopt;
3245 }
3246
3247 LLVM_DEBUG({
3248 dbgs() << "Tree structured merge rewrite (stores-only):\n";
3249 dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3250 for (auto [I, Info] : enumerate(StoreInfos)) {
3251 dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3252 << Info.EndOffset << ") \tStore: " << *Info.Store
3253 << "\tValue: " << *Info.StoredValue << "\n";
3254 }
3255 });
3256
3257 // StoreInfos is sorted by offset, not by block order. Anchoring to
3258 // StoreInfos.back().Store (last by offset) can place shuffles before
3259 // operands that appear later in the block (invalid SSA). Insert before
3260 // FullLoad when it shares the store block (after all stores, before
3261 // any later IR in that block). Otherwise insert before the store
3262 // block's terminator so the merge runs after every store and any
3263 // trailing instructions in that block.
3264 IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(Val: FullLoad)
3265 : StoreBB->getTerminator());
3266 SmallVector<Value *, 8> Vals;
3267 for (const auto &Info : StoreInfos) {
3268 DeletedValues.push_back(Elt: Info.Store);
3269 Vals.push_back(Elt: Info.StoredValue);
3270 }
3271 // Merge all stored values and store the merged value into the alloca.
3272 Value *Merged = TreeMerge(Vals, Builder);
3273 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3274
3275 // Replace the original load with a load of the newly-merged alloca.
3276 ReplaceFullLoad(FullLoad, Merged);
3277 return DeletedValues;
3278 }
3279
3280 // RMW pattern handling starts from here.
3281 // Like StoreBB above: keep the init store, all partial loads and all
3282 // partial stores in one basic block so we can reason about ordering
3283 // with comesBefore and build SSA without PHIs.
3284 if (InitStore->getParent() != StoreBB)
3285 return std::nullopt;
3286 if (any_of(Range&: LoadInfos, P: [&](const LoadInfo &I) {
3287 return I.Load->getParent() != StoreBB;
3288 }))
3289 return std::nullopt;
3290 // FullLoad (if any) is allowed to live in a different basic block. See
3291 // the note on the stores-only path: we don't do store->load forwarding
3292 // directly — the merged vector is stored to NewAI and the new load
3293 // loads from NewAI, so cross-BB ordering is resolved later when NewAI
3294 // is promoted.
3295
3296 // Collect the combined partial-load/partial-store accesses sorted
3297 // by block order. Used both for ordering checks and for the rewrite
3298 // walk below.
3299 struct Access {
3300 Instruction *Inst;
3301 uint64_t BeginOffset, EndOffset;
3302 bool IsStore;
3303 };
3304 SmallVector<Access, 16> Accesses;
3305 Accesses.reserve(N: LoadInfos.size() + StoreInfos.size());
3306 for (const auto &L : LoadInfos)
3307 Accesses.push_back(Elt: {.Inst: L.Load, .BeginOffset: L.BeginOffset, .EndOffset: L.EndOffset, .IsStore: false});
3308 for (const auto &S : StoreInfos)
3309 Accesses.push_back(Elt: {.Inst: S.Store, .BeginOffset: S.BeginOffset, .EndOffset: S.EndOffset, .IsStore: true});
3310 llvm::sort(C&: Accesses, Comp: [](const Access &A, const Access &B) {
3311 return A.Inst->comesBefore(Other: B.Inst);
3312 });
3313
3314 // Ordering constraint 1: InitStore must come before every partial
3315 // access — they read/write the RMW state initialised by InitStore.
3316 // Accesses is sorted by block order, so the first element is the
3317 // earliest; checking it is enough.
3318 if (!InitStore->comesBefore(Other: Accesses.front().Inst))
3319 return std::nullopt;
3320 // Ordering constraint 2: when FullLoad shares the block with the
3321 // partial accesses, it must come after every one of them — otherwise
3322 // it could read a stale value. Accesses is sorted, so the last
3323 // element is the latest; checking it is enough. If FullLoad is in
3324 // another block, mem2reg forwards the merged store to it.
3325 if (FullLoad && FullLoad->getParent() == StoreBB &&
3326 !Accesses.back().Inst->comesBefore(Other: FullLoad))
3327 return std::nullopt;
3328
3329 // Coverage check: the distinct [begin, end) ranges touched by the
3330 // partial loads and stores must tile the alloca disjointly. That is
3331 // the only precondition the per-range SliceValues tracking below
3332 // needs — a disjoint tile guarantees the entries don't alias each
3333 // other. We don't check per-range load/store counts: a range with
3334 // only loads ends with SliceValues[r] = the init extract
3335 // (contributed to the final tree-merge), and a range with only
3336 // stores ends with SliceValues[r] = its last stored value. Both are
3337 // correct.
3338 using SliceRange = std::pair<uint64_t, uint64_t>;
3339 SmallVector<SliceRange, 8> SortedRanges;
3340 SortedRanges.reserve(N: Accesses.size());
3341 for (auto &Acc : Accesses)
3342 SortedRanges.emplace_back(Args&: Acc.BeginOffset, Args&: Acc.EndOffset);
3343 llvm::sort(C&: SortedRanges);
3344 SortedRanges.erase(CS: llvm::unique(R&: SortedRanges), CE: SortedRanges.end());
3345 // Disjoint + contiguous tile of the whole alloca.
3346 uint64_t Expected = NewAllocaBeginOffset;
3347 for (auto &Range : SortedRanges) {
3348 if (Range.first != Expected)
3349 return std::nullopt;
3350 Expected = Range.second;
3351 }
3352 if (Expected != NewAllocaEndOffset)
3353 return std::nullopt;
3354
3355 LLVM_DEBUG({
3356 dbgs() << "Tree structured merge rewrite (RMW):\n";
3357 dbgs() << " Init store: " << *InitStore << "\n";
3358 if (FullLoad)
3359 dbgs() << " Final load: " << *FullLoad << "\n";
3360 dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3361 for (auto &Range : SortedRanges)
3362 dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3363 });
3364
3365 // Initialize SliceValues: one SSA value per slice range, tracking
3366 // the value the alloca currently holds at that range. Each entry
3367 // starts at the corresponding piece of the init store, obtained by
3368 // bitcasting the init value to the alloca's vector type (if needed)
3369 // and extracting the slice's sub-range.
3370 IRB.SetInsertPoint(InitStore->getNextNode());
3371 Value *InitVec = InitStore->getValueOperand();
3372 if (InitVec->getType() != NewAllocaTy)
3373 InitVec = IRB.CreateBitCast(V: InitVec, DestTy: NewAllocaTy, Name: "init.cast");
3374 DenseMap<SliceRange, Value *> SliceValues;
3375 for (auto &Range : SortedRanges) {
3376 unsigned BeginIdx = getIndex(Offset: Range.first);
3377 unsigned EndIdx = getIndex(Offset: Range.second);
3378 SliceValues[Range] = IRB.CreateShuffleVector(
3379 V: InitVec, Mask: createSequentialMask(Start: BeginIdx, NumInts: EndIdx - BeginIdx, NumUndefs: 0),
3380 Name: "init.extract");
3381 }
3382 // The init store itself becomes dead — its value is consumed via the
3383 // extracts above.
3384 DeletedValues.push_back(Elt: InitStore);
3385
3386 // Walk accesses in block order:
3387 // - partial load at range r: replace with SliceValues[r] (bitcast
3388 // if the load's type differs from the current tracked value's
3389 // type, e.g. because a previous store wrote a vector with a
3390 // different element type);
3391 // - partial store at range r: update SliceValues[r] to the stored
3392 // value and drop the store.
3393 for (auto &Acc : Accesses) {
3394 SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3395 if (!Acc.IsStore) {
3396 Value *V = SliceValues[Range];
3397 if (V->getType() != Acc.Inst->getType()) {
3398 IRB.SetInsertPoint(cast<LoadInst>(Val: Acc.Inst));
3399 V = IRB.CreateBitCast(V, DestTy: Acc.Inst->getType());
3400 }
3401 Acc.Inst->replaceAllUsesWith(V);
3402 } else {
3403 SliceValues[Range] = cast<StoreInst>(Val: Acc.Inst)->getValueOperand();
3404 }
3405 DeletedValues.push_back(Elt: Acc.Inst);
3406 }
3407
3408 // Tree-merge the final per-range values (in range order) into the
3409 // alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3410 // shares the partial-access block) or otherwise to the block's
3411 // terminator — never to a partial access, since those are queued for
3412 // deletion. Both anchors are guaranteed to dominate every SliceValues
3413 // entry: each one is either an init extract (before any access) or a
3414 // stored value defined before its (now-deleted) store.
3415 IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3416 ? cast<Instruction>(Val: FullLoad)
3417 : StoreBB->getTerminator());
3418 SmallVector<Value *, 8> Vals;
3419 for (auto &Range : SortedRanges)
3420 Vals.push_back(Elt: SliceValues[Range]);
3421 Value *Merged = TreeMerge(Vals, Builder);
3422 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3423
3424 // Replace the optional final full-width load with a load of the newly
3425 // merged alloca. Later promotion will forward the store above to it.
3426 if (FullLoad)
3427 ReplaceFullLoad(FullLoad, Merged);
3428
3429 return DeletedValues;
3430 }
3431
3432private:
3433 // Make sure the other visit overloads are visible.
3434 using Base::visit;
3435
3436 // Every instruction which can end up as a user must have a rewrite rule.
3437 bool visitInstruction(Instruction &I) {
3438 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3439 llvm_unreachable("No rewrite rule for this instruction!");
3440 }
3441
3442 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
3443 // Note that the offset computation can use BeginOffset or NewBeginOffset
3444 // interchangeably for unsplit slices.
3445 assert(IsSplit || BeginOffset == NewBeginOffset);
3446 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3447
3448 StringRef OldName = OldPtr->getName();
3449 // Skip through the last '.sroa.' component of the name.
3450 size_t LastSROAPrefix = OldName.rfind(Str: ".sroa.");
3451 if (LastSROAPrefix != StringRef::npos) {
3452 OldName = OldName.substr(Start: LastSROAPrefix + strlen(s: ".sroa."));
3453 // Look for an SROA slice index.
3454 size_t IndexEnd = OldName.find_first_not_of(Chars: "0123456789");
3455 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
3456 // Strip the index and look for the offset.
3457 OldName = OldName.substr(Start: IndexEnd + 1);
3458 size_t OffsetEnd = OldName.find_first_not_of(Chars: "0123456789");
3459 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
3460 // Strip the offset.
3461 OldName = OldName.substr(Start: OffsetEnd + 1);
3462 }
3463 }
3464 // Strip any SROA suffixes as well.
3465 OldName = OldName.substr(Start: 0, N: OldName.find(Str: ".sroa_"));
3466
3467 return getAdjustedPtr(
3468 IRB, DL, Ptr: &NewAI,
3469 Offset: APInt(DL.getIndexTypeSizeInBits(Ty: NewAI.getType()), Offset), PointerTy,
3470 NamePrefix: Twine(OldName) + ".");
3471 }
3472
3473 /// Compute suitable alignment to access this slice of the *new*
3474 /// alloca.
3475 ///
3476 /// You can optionally pass a type to this routine and if that type's ABI
3477 /// alignment is itself suitable, this will return zero.
3478 Align getSliceAlign() {
3479 return commonAlignment(A: NewAI.getAlign(),
3480 Offset: NewBeginOffset - NewAllocaBeginOffset);
3481 }
3482
3483 unsigned getIndex(uint64_t Offset) {
3484 assert(VecTy && "Can only call getIndex when rewriting a vector");
3485 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3486 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3487 uint32_t Index = RelOffset / ElementSize;
3488 assert(Index * ElementSize == RelOffset);
3489 return Index;
3490 }
3491
3492 void deleteIfTriviallyDead(Value *V) {
3493 Instruction *I = cast<Instruction>(Val: V);
3494 if (isInstructionTriviallyDead(I))
3495 Pass.DeadInsts.push_back(Elt: I);
3496 }
3497
3498 Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3499 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3500 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3501 assert(EndIndex > BeginIndex && "Empty vector!");
3502
3503 LoadInst *Load =
3504 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3505
3506 Load->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3507 LLVMContext::MD_access_group});
3508 return extractVector(IRB, V: Load, BeginIndex, EndIndex, Name: "vec");
3509 }
3510
3511 Value *rewriteIntegerLoad(LoadInst &LI) {
3512 assert(IntTy && "We cannot insert an integer to the alloca");
3513 assert(!LI.isVolatile());
3514 Value *V =
3515 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3516 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: IntTy);
3517 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3518 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3519 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
3520 IntegerType *ExtractTy = Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8);
3521 V = extractInteger(DL, IRB, V, Ty: ExtractTy, Offset, Name: "extract");
3522 }
3523 // It is possible that the extracted type is not the load type. This
3524 // happens if there is a load past the end of the alloca, and as
3525 // a consequence the slice is narrower but still a candidate for integer
3526 // lowering. To handle this case, we just zero extend the extracted
3527 // integer.
3528 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
3529 "Can only handle an extract for an overly wide load");
3530 if (cast<IntegerType>(Val: LI.getType())->getBitWidth() > SliceSize * 8)
3531 V = IRB.CreateZExt(V, DestTy: LI.getType());
3532 return V;
3533 }
3534
3535 bool visitLoadInst(LoadInst &LI) {
3536 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3537 Value *OldOp = LI.getOperand(i_nocapture: 0);
3538 assert(OldOp == OldPtr);
3539
3540 AAMDNodes AATags = LI.getAAMetadata();
3541
3542 unsigned AS = LI.getPointerAddressSpace();
3543
3544 Type *TargetTy = IsSplit ? Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8)
3545 : LI.getType();
3546 bool IsPtrAdjusted = false;
3547 Value *V;
3548 if (VecTy) {
3549 V = rewriteVectorizedLoadInst(LI);
3550 } else if (IntTy && LI.getType()->isIntegerTy()) {
3551 V = rewriteIntegerLoad(LI);
3552 } else if (NewBeginOffset == NewAllocaBeginOffset &&
3553 NewEndOffset == NewAllocaEndOffset &&
3554 (canConvertValue(DL, OldTy: NewAllocaTy, NewTy: TargetTy) ||
3555 (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3556 DL.getTypeStoreSize(Ty: TargetTy).getFixedValue() > SliceSize &&
3557 !LI.isVolatile()))) {
3558 Value *NewPtr =
3559 getPtrToNewAI(AddrSpace: LI.getPointerAddressSpace(), IsVolatile: LI.isVolatile());
3560 LoadInst *NewLI = IRB.CreateAlignedLoad(
3561 Ty: NewAllocaTy, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3562 if (LI.isVolatile())
3563 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3564 if (NewLI->isAtomic())
3565 NewLI->setAlignment(LI.getAlign());
3566
3567 // Copy any metadata that is valid for the new load. This may require
3568 // conversion to a different kind of metadata, e.g. !nonnull might change
3569 // to !range or vice versa.
3570 copyMetadataForLoad(Dest&: *NewLI, Source: LI);
3571
3572 // Do this after copyMetadataForLoad() to preserve the TBAA shift.
3573 if (AATags)
3574 NewLI->setAAMetadata(AATags.adjustForAccess(
3575 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3576
3577 // Try to preserve nonnull metadata
3578 V = NewLI;
3579
3580 // If this is an integer load past the end of the slice (which means the
3581 // bytes outside the slice are undef or this load is dead) just forcibly
3582 // fix the integer size with correct handling of endianness.
3583 if (auto *AITy = dyn_cast<IntegerType>(Val: NewAllocaTy))
3584 if (auto *TITy = dyn_cast<IntegerType>(Val: TargetTy))
3585 if (AITy->getBitWidth() < TITy->getBitWidth()) {
3586 V = IRB.CreateZExt(V, DestTy: TITy, Name: "load.ext");
3587 if (DL.isBigEndian())
3588 V = IRB.CreateShl(LHS: V, RHS: TITy->getBitWidth() - AITy->getBitWidth(),
3589 Name: "endian_shift");
3590 }
3591 } else {
3592 Type *LTy = IRB.getPtrTy(AddrSpace: AS);
3593 LoadInst *NewLI =
3594 IRB.CreateAlignedLoad(Ty: TargetTy, Ptr: getNewAllocaSlicePtr(IRB, PointerTy: LTy),
3595 Align: getSliceAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3596
3597 if (AATags)
3598 NewLI->setAAMetadata(AATags.adjustForAccess(
3599 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3600
3601 if (LI.isVolatile())
3602 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3603 NewLI->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3604 LLVMContext::MD_access_group});
3605
3606 V = NewLI;
3607 IsPtrAdjusted = true;
3608 }
3609 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: TargetTy);
3610
3611 if (IsSplit) {
3612 assert(!LI.isVolatile());
3613 assert(LI.getType()->isIntegerTy() &&
3614 "Only integer type loads and stores are split");
3615 assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3616 "Split load isn't smaller than original load");
3617 assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3618 "Non-byte-multiple bit width");
3619 // Move the insertion point just past the load so that we can refer to it.
3620 BasicBlock::iterator LIIt = std::next(x: LI.getIterator());
3621 // Ensure the insertion point comes before any debug-info immediately
3622 // after the load, so that variable values referring to the load are
3623 // dominated by it.
3624 LIIt.setHeadBit(true);
3625 IRB.SetInsertPoint(LIIt);
3626 // Create a placeholder value with the same type as LI to use as the
3627 // basis for the new value. This allows us to replace the uses of LI with
3628 // the computed value, and then replace the placeholder with LI, leaving
3629 // LI only used for this computation.
3630 Value *Placeholder =
3631 new LoadInst(LI.getType(), PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AS)), "",
3632 false, Align(1));
3633 V = insertInteger(DL, IRB, Old: Placeholder, V, Offset: NewBeginOffset - BeginOffset,
3634 Name: "insert");
3635 LI.replaceAllUsesWith(V);
3636 Placeholder->replaceAllUsesWith(V: &LI);
3637 Placeholder->deleteValue();
3638 } else {
3639 LI.replaceAllUsesWith(V);
3640 }
3641
3642 Pass.DeadInsts.push_back(Elt: &LI);
3643 deleteIfTriviallyDead(V: OldOp);
3644 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3645 return !LI.isVolatile() && !IsPtrAdjusted;
3646 }
3647
3648 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
3649 AAMDNodes AATags) {
3650 // Capture V for the purpose of debug-info accounting once it's converted
3651 // to a vector store.
3652 Value *OrigV = V;
3653 if (V->getType() != VecTy) {
3654 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3655 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3656 assert(EndIndex > BeginIndex && "Empty vector!");
3657 unsigned NumElements = EndIndex - BeginIndex;
3658 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3659 "Too many elements!");
3660 Type *SliceTy = (NumElements == 1)
3661 ? ElementTy
3662 : FixedVectorType::get(ElementType: ElementTy, NumElts: NumElements);
3663 if (V->getType() != SliceTy)
3664 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: SliceTy);
3665
3666 // Mix in the existing elements.
3667 Value *Old =
3668 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3669 V = insertVector(IRB, Old, V, BeginIndex, Name: "vec");
3670 }
3671 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3672 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3673 LLVMContext::MD_access_group});
3674 if (AATags)
3675 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3676 AccessTy: V->getType(), DL));
3677 Pass.DeadInsts.push_back(Elt: &SI);
3678
3679 // NOTE: Careful to use OrigV rather than V.
3680 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3681 Inst: Store, Dest: Store->getPointerOperand(), Value: OrigV, DL);
3682 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3683 return true;
3684 }
3685
3686 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3687 assert(IntTy && "We cannot extract an integer from the alloca");
3688 assert(!SI.isVolatile());
3689 if (DL.getTypeSizeInBits(Ty: V->getType()).getFixedValue() !=
3690 IntTy->getBitWidth()) {
3691 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3692 Name: "oldload");
3693 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3694 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3695 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3696 V = insertInteger(DL, IRB, Old, V: SI.getValueOperand(), Offset, Name: "insert");
3697 }
3698 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3699 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3700 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3701 LLVMContext::MD_access_group});
3702 if (AATags)
3703 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3704 AccessTy: V->getType(), DL));
3705
3706 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3707 Inst: Store, Dest: Store->getPointerOperand(),
3708 Value: Store->getValueOperand(), DL);
3709
3710 Pass.DeadInsts.push_back(Elt: &SI);
3711 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3712 return true;
3713 }
3714
3715 bool visitStoreInst(StoreInst &SI) {
3716 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3717 Value *OldOp = SI.getOperand(i_nocapture: 1);
3718 assert(OldOp == OldPtr);
3719
3720 AAMDNodes AATags = SI.getAAMetadata();
3721 Value *V = SI.getValueOperand();
3722
3723 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3724 // alloca that should be re-examined after promoting this alloca.
3725 if (V->getType()->isPointerTy())
3726 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V->stripInBoundsOffsets()))
3727 Pass.PostPromotionWorklist.insert(X: AI);
3728
3729 TypeSize StoreSize = DL.getTypeStoreSize(Ty: V->getType());
3730 if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3731 assert(!SI.isVolatile());
3732 assert(V->getType()->isIntegerTy() &&
3733 "Only integer type loads and stores are split");
3734 assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3735 "Non-byte-multiple bit width");
3736 IntegerType *NarrowTy = Type::getIntNTy(C&: SI.getContext(), N: SliceSize * 8);
3737 V = extractInteger(DL, IRB, V, Ty: NarrowTy, Offset: NewBeginOffset - BeginOffset,
3738 Name: "extract");
3739 }
3740
3741 if (VecTy)
3742 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3743 if (IntTy && V->getType()->isIntegerTy())
3744 return rewriteIntegerStore(V, SI, AATags);
3745
3746 StoreInst *NewSI;
3747 if (NewBeginOffset == NewAllocaBeginOffset &&
3748 NewEndOffset == NewAllocaEndOffset &&
3749 canConvertValue(DL, OldTy: V->getType(), NewTy: NewAllocaTy)) {
3750 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3751 Value *NewPtr =
3752 getPtrToNewAI(AddrSpace: SI.getPointerAddressSpace(), IsVolatile: SI.isVolatile());
3753
3754 NewSI =
3755 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: SI.isVolatile());
3756 } else {
3757 unsigned AS = SI.getPointerAddressSpace();
3758 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: IRB.getPtrTy(AddrSpace: AS));
3759 NewSI =
3760 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: getSliceAlign(), isVolatile: SI.isVolatile());
3761 }
3762 NewSI->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3763 LLVMContext::MD_access_group});
3764 if (AATags)
3765 NewSI->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3766 AccessTy: V->getType(), DL));
3767 if (SI.isVolatile())
3768 NewSI->setAtomic(Ordering: SI.getOrdering(), SSID: SI.getSyncScopeID());
3769 if (NewSI->isAtomic())
3770 NewSI->setAlignment(SI.getAlign());
3771
3772 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3773 Inst: NewSI, Dest: NewSI->getPointerOperand(),
3774 Value: NewSI->getValueOperand(), DL);
3775
3776 Pass.DeadInsts.push_back(Elt: &SI);
3777 deleteIfTriviallyDead(V: OldOp);
3778
3779 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3780 return NewSI->getPointerOperand() == &NewAI &&
3781 NewSI->getValueOperand()->getType() == NewAllocaTy &&
3782 !SI.isVolatile();
3783 }
3784
3785 /// Compute an integer value from splatting an i8 across the given
3786 /// number of bytes.
3787 ///
3788 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3789 /// call this routine.
3790 /// FIXME: Heed the advice above.
3791 ///
3792 /// \param V The i8 value to splat.
3793 /// \param Size The number of bytes in the output (assuming i8 is one byte)
3794 Value *getIntegerSplat(Value *V, unsigned Size) {
3795 assert(Size > 0 && "Expected a positive number of bytes.");
3796 IntegerType *VTy = cast<IntegerType>(Val: V->getType());
3797 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
3798 if (Size == 1)
3799 return V;
3800
3801 Type *SplatIntTy = Type::getIntNTy(C&: VTy->getContext(), N: Size * 8);
3802 V = IRB.CreateMul(
3803 LHS: IRB.CreateZExt(V, DestTy: SplatIntTy, Name: "zext"),
3804 RHS: IRB.CreateUDiv(LHS: Constant::getAllOnesValue(Ty: SplatIntTy),
3805 RHS: IRB.CreateZExt(V: Constant::getAllOnesValue(Ty: V->getType()),
3806 DestTy: SplatIntTy)),
3807 Name: "isplat");
3808 return V;
3809 }
3810
3811 /// Compute a vector splat for a given element value.
3812 Value *getVectorSplat(Value *V, unsigned NumElements) {
3813 V = IRB.CreateVectorSplat(NumElts: NumElements, V, Name: "vsplat");
3814 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3815 return V;
3816 }
3817
3818 bool visitMemSetInst(MemSetInst &II) {
3819 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3820 assert(II.getRawDest() == OldPtr);
3821
3822 AAMDNodes AATags = II.getAAMetadata();
3823
3824 // If the memset has a variable size, it cannot be split, just adjust the
3825 // pointer to the new alloca.
3826 if (!isa<ConstantInt>(Val: II.getLength())) {
3827 assert(!IsSplit);
3828 assert(NewBeginOffset == BeginOffset);
3829 II.setDest(getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()));
3830 II.setDestAlignment(getSliceAlign());
3831 // In theory we should call migrateDebugInfo here. However, we do not
3832 // emit dbg.assign intrinsics for mem intrinsics storing through non-
3833 // constant geps, or storing a variable number of bytes.
3834 assert(at::getDVRAssignmentMarkers(&II).empty() &&
3835 "AT: Unexpected link to non-const GEP");
3836 deleteIfTriviallyDead(V: OldPtr);
3837 return false;
3838 }
3839
3840 // Record this instruction for deletion.
3841 Pass.DeadInsts.push_back(Elt: &II);
3842
3843 Type *ScalarTy = NewAllocaTy->getScalarType();
3844
3845 const bool CanContinue = [&]() {
3846 if (VecTy || IntTy)
3847 return true;
3848 if (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset)
3849 return false;
3850 // Length must be in range for FixedVectorType.
3851 auto *C = cast<ConstantInt>(Val: II.getLength());
3852 const uint64_t Len = C->getLimitedValue();
3853 if (Len > std::numeric_limits<unsigned>::max())
3854 return false;
3855 auto *Int8Ty = IntegerType::getInt8Ty(C&: NewAI.getContext());
3856 auto *SrcTy = FixedVectorType::get(ElementType: Int8Ty, NumElts: Len);
3857 return canConvertValue(DL, OldTy: SrcTy, NewTy: NewAllocaTy) &&
3858 DL.isLegalInteger(Width: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue());
3859 }();
3860
3861 // If this doesn't map cleanly onto the alloca type, and that type isn't
3862 // a single value type, just emit a memset.
3863 if (!CanContinue) {
3864 Type *SizeTy = II.getLength()->getType();
3865 unsigned Sz = NewEndOffset - NewBeginOffset;
3866 Constant *Size = ConstantInt::get(Ty: SizeTy, V: Sz);
3867 MemIntrinsic *New = cast<MemIntrinsic>(Val: IRB.CreateMemSet(
3868 Ptr: getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()), Val: II.getValue(), Size,
3869 Align: MaybeAlign(getSliceAlign()), isVolatile: II.isVolatile()));
3870 if (AATags)
3871 New->setAAMetadata(
3872 AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset, AccessSize: Sz));
3873
3874 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3875 Inst: New, Dest: New->getRawDest(), Value: nullptr, DL);
3876
3877 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3878 return false;
3879 }
3880
3881 // If we can represent this as a simple value, we have to build the actual
3882 // value to store, which requires expanding the byte present in memset to
3883 // a sensible representation for the alloca type. This is essentially
3884 // splatting the byte to a sufficiently wide integer, splatting it across
3885 // any desired vector width, and bitcasting to the final type.
3886 Value *V;
3887
3888 if (VecTy) {
3889 // If this is a memset of a vectorized alloca, insert it.
3890 assert(ElementTy == ScalarTy);
3891
3892 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3893 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3894 assert(EndIndex > BeginIndex && "Empty vector!");
3895 unsigned NumElements = EndIndex - BeginIndex;
3896 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3897 "Too many elements!");
3898
3899 Value *Splat = getIntegerSplat(
3900 V: II.getValue(), Size: DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8);
3901 Splat = IRB.CreateBitPreservingCastChain(DL, V: Splat, NewTy: ElementTy);
3902 if (NumElements > 1)
3903 Splat = getVectorSplat(V: Splat, NumElements);
3904
3905 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3906 Name: "oldload");
3907 V = insertVector(IRB, Old, V: Splat, BeginIndex, Name: "vec");
3908 } else if (IntTy) {
3909 // If this is a memset on an alloca where we can widen stores, insert the
3910 // set integer.
3911 assert(!II.isVolatile());
3912
3913 uint64_t Size = NewEndOffset - NewBeginOffset;
3914 V = getIntegerSplat(V: II.getValue(), Size);
3915
3916 if (IntTy && (NewBeginOffset != NewAllocaBeginOffset ||
3917 NewEndOffset != NewAllocaEndOffset)) {
3918 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI,
3919 Align: NewAI.getAlign(), Name: "oldload");
3920 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3921 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3922 V = insertInteger(DL, IRB, Old, V, Offset, Name: "insert");
3923 } else {
3924 assert(V->getType() == IntTy &&
3925 "Wrong type for an alloca wide integer!");
3926 }
3927 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3928 } else {
3929 // Established these invariants above.
3930 assert(NewBeginOffset == NewAllocaBeginOffset);
3931 assert(NewEndOffset == NewAllocaEndOffset);
3932
3933 V = getIntegerSplat(V: II.getValue(),
3934 Size: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue() / 8);
3935 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(Val: NewAllocaTy))
3936 V = getVectorSplat(
3937 V, NumElements: cast<FixedVectorType>(Val: AllocaVecTy)->getNumElements());
3938
3939 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3940 }
3941
3942 Value *NewPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3943 StoreInst *New =
3944 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: II.isVolatile());
3945 New->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3946 LLVMContext::MD_access_group});
3947 if (AATags)
3948 New->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3949 AccessTy: V->getType(), DL));
3950
3951 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3952 Inst: New, Dest: New->getPointerOperand(), Value: V, DL);
3953
3954 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3955 return !II.isVolatile();
3956 }
3957
3958 bool visitMemTransferInst(MemTransferInst &II) {
3959 // Rewriting of memory transfer instructions can be a bit tricky. We break
3960 // them into two categories: split intrinsics and unsplit intrinsics.
3961
3962 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3963
3964 AAMDNodes AATags = II.getAAMetadata();
3965
3966 bool IsDest = &II.getRawDestUse() == OldUse;
3967 assert((IsDest && II.getRawDest() == OldPtr) ||
3968 (!IsDest && II.getRawSource() == OldPtr));
3969
3970 Align SliceAlign = getSliceAlign();
3971 // For unsplit intrinsics, we simply modify the source and destination
3972 // pointers in place. This isn't just an optimization, it is a matter of
3973 // correctness. With unsplit intrinsics we may be dealing with transfers
3974 // within a single alloca before SROA ran, or with transfers that have
3975 // a variable length. We may also be dealing with memmove instead of
3976 // memcpy, and so simply updating the pointers is the necessary for us to
3977 // update both source and dest of a single call.
3978 if (!IsSplittable) {
3979 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3980 if (IsDest) {
3981 // Update the address component of linked dbg.assigns.
3982 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(Inst: &II)) {
3983 if (llvm::is_contained(Range: DbgAssign->location_ops(), Element: II.getDest()) ||
3984 DbgAssign->getAddress() == II.getDest())
3985 DbgAssign->replaceVariableLocationOp(OldValue: II.getDest(), NewValue: AdjustedPtr);
3986 }
3987 II.setDest(AdjustedPtr);
3988 II.setDestAlignment(SliceAlign);
3989 } else {
3990 II.setSource(AdjustedPtr);
3991 II.setSourceAlignment(SliceAlign);
3992 }
3993
3994 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3995 deleteIfTriviallyDead(V: OldPtr);
3996 return false;
3997 }
3998 // For split transfer intrinsics we have an incredibly useful assurance:
3999 // the source and destination do not reside within the same alloca, and at
4000 // least one of them does not escape. This means that we can replace
4001 // memmove with memcpy, and we don't need to worry about all manner of
4002 // downsides to splitting and transforming the operations.
4003
4004 // If this doesn't map cleanly onto the alloca type, and that type isn't
4005 // a single value type, just emit a memcpy.
4006 bool EmitMemCpy =
4007 !VecTy && !IntTy &&
4008 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
4009 SliceSize != DL.getTypeStoreSize(Ty: NewAllocaTy).getFixedValue() ||
4010 !DL.typeSizeEqualsStoreSize(Ty: NewAllocaTy) ||
4011 !NewAllocaTy->isSingleValueType());
4012
4013 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
4014 // size hasn't been shrunk based on analysis of the viable range, this is
4015 // a no-op.
4016 if (EmitMemCpy && &OldAI == &NewAI) {
4017 // Ensure the start lines up.
4018 assert(NewBeginOffset == BeginOffset);
4019
4020 // Rewrite the size as needed.
4021 if (NewEndOffset != EndOffset)
4022 II.setLength(NewEndOffset - NewBeginOffset);
4023 return false;
4024 }
4025 // Record this instruction for deletion.
4026 Pass.DeadInsts.push_back(Elt: &II);
4027
4028 // Strip all inbounds GEPs and pointer casts to try to dig out any root
4029 // alloca that should be re-examined after rewriting this instruction.
4030 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
4031 if (AllocaInst *AI =
4032 dyn_cast<AllocaInst>(Val: OtherPtr->stripInBoundsOffsets())) {
4033 assert(AI != &OldAI && AI != &NewAI &&
4034 "Splittable transfers cannot reach the same alloca on both ends.");
4035 Pass.Worklist.insert(X: AI);
4036 }
4037
4038 Type *OtherPtrTy = OtherPtr->getType();
4039 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
4040
4041 // Compute the relative offset for the other pointer within the transfer.
4042 unsigned OffsetWidth = DL.getIndexSizeInBits(AS: OtherAS);
4043 APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
4044 Align OtherAlign =
4045 (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
4046 OtherAlign =
4047 commonAlignment(A: OtherAlign, Offset: OtherOffset.zextOrTrunc(width: 64).getZExtValue());
4048
4049 if (EmitMemCpy) {
4050 // Compute the other pointer, folding as much as possible to produce
4051 // a single, simple GEP in most cases.
4052 OtherPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
4053 NamePrefix: OtherPtr->getName() + ".");
4054
4055 Value *OurPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4056 Type *SizeTy = II.getLength()->getType();
4057 Constant *Size = ConstantInt::get(Ty: SizeTy, V: NewEndOffset - NewBeginOffset);
4058
4059 Value *DestPtr, *SrcPtr;
4060 MaybeAlign DestAlign, SrcAlign;
4061 // Note: IsDest is true iff we're copying into the new alloca slice
4062 if (IsDest) {
4063 DestPtr = OurPtr;
4064 DestAlign = SliceAlign;
4065 SrcPtr = OtherPtr;
4066 SrcAlign = OtherAlign;
4067 } else {
4068 DestPtr = OtherPtr;
4069 DestAlign = OtherAlign;
4070 SrcPtr = OurPtr;
4071 SrcAlign = SliceAlign;
4072 }
4073 CallInst *New = IRB.CreateMemCpy(Dst: DestPtr, DstAlign: DestAlign, Src: SrcPtr, SrcAlign,
4074 Size, isVolatile: II.isVolatile());
4075 if (AATags)
4076 New->setAAMetadata(AATags.shift(Offset: NewBeginOffset - BeginOffset));
4077
4078 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DestPtr->getType()), 0);
4079 if (IsDest) {
4080 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8,
4081 OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
4082 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4083 Val: DestPtr->stripAndAccumulateConstantOffsets(
4084 DL, Offset, /*AllowNonInbounds*/ true))) {
4085 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
4086 SliceSizeInBits: SliceSize * 8, OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
4087 }
4088 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4089 return false;
4090 }
4091
4092 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
4093 NewEndOffset == NewAllocaEndOffset;
4094 uint64_t Size = NewEndOffset - NewBeginOffset;
4095 unsigned BeginIndex = VecTy ? getIndex(Offset: NewBeginOffset) : 0;
4096 unsigned EndIndex = VecTy ? getIndex(Offset: NewEndOffset) : 0;
4097 unsigned NumElements = EndIndex - BeginIndex;
4098 IntegerType *SubIntTy =
4099 IntTy ? Type::getIntNTy(C&: IntTy->getContext(), N: Size * 8) : nullptr;
4100
4101 // Reset the other pointer type to match the register type we're going to
4102 // use, but using the address space of the original other pointer.
4103 Type *OtherTy;
4104 if (VecTy && !IsWholeAlloca) {
4105 if (NumElements == 1)
4106 OtherTy = VecTy->getElementType();
4107 else
4108 OtherTy = FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: NumElements);
4109 } else if (IntTy && !IsWholeAlloca) {
4110 OtherTy = SubIntTy;
4111 } else {
4112 OtherTy = NewAllocaTy;
4113 }
4114
4115 Value *AdjPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
4116 NamePrefix: OtherPtr->getName() + ".");
4117 MaybeAlign SrcAlign = OtherAlign;
4118 MaybeAlign DstAlign = SliceAlign;
4119 if (!IsDest)
4120 std::swap(a&: SrcAlign, b&: DstAlign);
4121
4122 Value *SrcPtr;
4123 Value *DstPtr;
4124
4125 if (IsDest) {
4126 DstPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
4127 SrcPtr = AdjPtr;
4128 } else {
4129 DstPtr = AdjPtr;
4130 SrcPtr = getPtrToNewAI(AddrSpace: II.getSourceAddressSpace(), IsVolatile: II.isVolatile());
4131 }
4132
4133 Value *Src;
4134 if (VecTy && !IsWholeAlloca && !IsDest) {
4135 Src =
4136 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
4137 Src = extractVector(IRB, V: Src, BeginIndex, EndIndex, Name: "vec");
4138 } else if (IntTy && !IsWholeAlloca && !IsDest) {
4139 Src =
4140 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
4141 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: IntTy);
4142 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4143 Src = extractInteger(DL, IRB, V: Src, Ty: SubIntTy, Offset, Name: "extract");
4144 } else {
4145 LoadInst *Load = IRB.CreateAlignedLoad(Ty: OtherTy, Ptr: SrcPtr, Align: SrcAlign,
4146 isVolatile: II.isVolatile(), Name: "copyload");
4147 Load->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4148 LLVMContext::MD_access_group});
4149 if (AATags)
4150 Load->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4151 AccessTy: Load->getType(), DL));
4152 Src = Load;
4153 }
4154
4155 if (VecTy && !IsWholeAlloca && IsDest) {
4156 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4157 Name: "oldload");
4158 Src = insertVector(IRB, Old, V: Src, BeginIndex, Name: "vec");
4159 } else if (IntTy && !IsWholeAlloca && IsDest) {
4160 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4161 Name: "oldload");
4162 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
4163 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4164 Src = insertInteger(DL, IRB, Old, V: Src, Offset, Name: "insert");
4165 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: NewAllocaTy);
4166 }
4167
4168 StoreInst *Store = cast<StoreInst>(
4169 Val: IRB.CreateAlignedStore(Val: Src, Ptr: DstPtr, Align: DstAlign, isVolatile: II.isVolatile()));
4170 Store->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4171 LLVMContext::MD_access_group});
4172 if (AATags)
4173 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4174 AccessTy: Src->getType(), DL));
4175
4176 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DstPtr->getType()), 0);
4177 if (IsDest) {
4178
4179 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
4180 Inst: Store, Dest: DstPtr, Value: Src, DL);
4181 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4182 Val: DstPtr->stripAndAccumulateConstantOffsets(
4183 DL, Offset, /*AllowNonInbounds*/ true))) {
4184 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8, SliceSizeInBits: SliceSize * 8,
4185 OldInst: &II, Inst: Store, Dest: DstPtr, Value: Src, DL);
4186 }
4187
4188 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4189 return !II.isVolatile();
4190 }
4191
4192 bool visitIntrinsicInst(IntrinsicInst &II) {
4193 assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
4194 "Unexpected intrinsic!");
4195 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4196
4197 // Record this instruction for deletion.
4198 Pass.DeadInsts.push_back(Elt: &II);
4199
4200 if (II.isDroppable()) {
4201 assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4202 // TODO For now we forget assumed information, this can be improved.
4203 OldPtr->dropDroppableUsesIn(Usr&: II);
4204 return true;
4205 }
4206
4207 assert(II.getArgOperand(0) == OldPtr);
4208 Type *PointerTy = IRB.getPtrTy(AddrSpace: OldPtr->getType()->getPointerAddressSpace());
4209 Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4210 Value *New;
4211 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4212 New = IRB.CreateLifetimeStart(Ptr);
4213 else
4214 New = IRB.CreateLifetimeEnd(Ptr);
4215
4216 (void)New;
4217 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4218
4219 return true;
4220 }
4221
4222 void fixLoadStoreAlign(Instruction &Root) {
4223 // This algorithm implements the same visitor loop as
4224 // hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4225 // or store found.
4226 SmallPtrSet<Instruction *, 4> Visited;
4227 SmallVector<Instruction *, 4> Uses;
4228 Visited.insert(Ptr: &Root);
4229 Uses.push_back(Elt: &Root);
4230 do {
4231 Instruction *I = Uses.pop_back_val();
4232
4233 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
4234 LI->setAlignment(std::min(a: LI->getAlign(), b: getSliceAlign()));
4235 continue;
4236 }
4237 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
4238 SI->setAlignment(std::min(a: SI->getAlign(), b: getSliceAlign()));
4239 continue;
4240 }
4241
4242 assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||
4243 isa<PHINode>(I) || isa<SelectInst>(I) ||
4244 isa<GetElementPtrInst>(I));
4245 for (User *U : I->users())
4246 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
4247 Uses.push_back(Elt: cast<Instruction>(Val: U));
4248 } while (!Uses.empty());
4249 }
4250
4251 bool visitPHINode(PHINode &PN) {
4252 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4253 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4254 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4255
4256 // We would like to compute a new pointer in only one place, but have it be
4257 // as local as possible to the PHI. To do that, we re-use the location of
4258 // the old pointer, which necessarily must be in the right position to
4259 // dominate the PHI.
4260 IRBuilderBase::InsertPointGuard Guard(IRB);
4261 if (isa<PHINode>(Val: OldPtr))
4262 IRB.SetInsertPoint(OldPtr->getParent()->getFirstInsertionPt());
4263 else
4264 IRB.SetInsertPoint(OldPtr);
4265 IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4266
4267 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4268 // Replace the operands which were using the old pointer.
4269 std::replace(first: PN.op_begin(), last: PN.op_end(), old_value: cast<Value>(Val: OldPtr), new_value: NewPtr);
4270
4271 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4272 deleteIfTriviallyDead(V: OldPtr);
4273
4274 // Fix the alignment of any loads or stores using this PHI node.
4275 fixLoadStoreAlign(Root&: PN);
4276
4277 // PHIs can't be promoted on their own, but often can be speculated. We
4278 // check the speculation outside of the rewriter so that we see the
4279 // fully-rewritten alloca.
4280 PHIUsers.insert(X: &PN);
4281 return true;
4282 }
4283
4284 bool visitSelectInst(SelectInst &SI) {
4285 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4286 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
4287 "Pointer isn't an operand!");
4288 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4289 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4290
4291 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4292 // Replace the operands which were using the old pointer.
4293 if (SI.getOperand(i_nocapture: 1) == OldPtr)
4294 SI.setOperand(i_nocapture: 1, Val_nocapture: NewPtr);
4295 if (SI.getOperand(i_nocapture: 2) == OldPtr)
4296 SI.setOperand(i_nocapture: 2, Val_nocapture: NewPtr);
4297
4298 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4299 deleteIfTriviallyDead(V: OldPtr);
4300
4301 // Fix the alignment of any loads or stores using this select.
4302 fixLoadStoreAlign(Root&: SI);
4303
4304 // Selects can't be promoted on their own, but often can be speculated. We
4305 // check the speculation outside of the rewriter so that we see the
4306 // fully-rewritten alloca.
4307 SelectUsers.insert(X: &SI);
4308 return true;
4309 }
4310};
4311
4312/// Visitor to rewrite aggregate loads and stores as scalar.
4313///
4314/// This pass aggressively rewrites all aggregate loads and stores on
4315/// a particular pointer (or any pointer derived from it which we can identify)
4316/// with scalar loads and stores.
4317class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4318 // Befriend the base class so it can delegate to private visit methods.
4319 friend class InstVisitor<AggLoadStoreRewriter, bool>;
4320
4321 /// Queue of pointer uses to analyze and potentially rewrite.
4322 SmallVector<Use *, 8> Queue;
4323
4324 /// Set to prevent us from cycling with phi nodes and loops.
4325 SmallPtrSet<User *, 8> Visited;
4326
4327 /// The current pointer use being rewritten. This is used to dig up the used
4328 /// value (as opposed to the user).
4329 Use *U = nullptr;
4330
4331 /// Used to calculate offsets, and hence alignment, of subobjects.
4332 const DataLayout &DL;
4333
4334 IRBuilderTy &IRB;
4335
4336public:
4337 AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4338 : DL(DL), IRB(IRB) {}
4339
4340 /// Rewrite loads and stores through a pointer and all pointers derived from
4341 /// it.
4342 bool rewrite(Instruction &I) {
4343 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4344 enqueueUsers(I);
4345 bool Changed = false;
4346 while (!Queue.empty()) {
4347 U = Queue.pop_back_val();
4348 Changed |= visit(I: cast<Instruction>(Val: U->getUser()));
4349 }
4350 return Changed;
4351 }
4352
4353private:
4354 /// Enqueue all the users of the given instruction for further processing.
4355 /// This uses a set to de-duplicate users.
4356 void enqueueUsers(Instruction &I) {
4357 for (Use &U : I.uses())
4358 if (Visited.insert(Ptr: U.getUser()).second)
4359 Queue.push_back(Elt: &U);
4360 }
4361
4362 // Conservative default is to not rewrite anything.
4363 bool visitInstruction(Instruction &I) { return false; }
4364
4365 /// Generic recursive split emission class.
4366 template <typename Derived> class OpSplitter {
4367 protected:
4368 /// The builder used to form new instructions.
4369 IRBuilderTy &IRB;
4370
4371 /// The indices which to be used with insert- or extractvalue to select the
4372 /// appropriate value within the aggregate.
4373 SmallVector<unsigned, 4> Indices;
4374
4375 /// The indices to a GEP instruction which will move Ptr to the correct slot
4376 /// within the aggregate.
4377 SmallVector<Value *, 4> GEPIndices;
4378
4379 /// The base pointer of the original op, used as a base for GEPing the
4380 /// split operations.
4381 Value *Ptr;
4382
4383 /// The base pointee type being GEPed into.
4384 Type *BaseTy;
4385
4386 /// Known alignment of the base pointer.
4387 Align BaseAlign;
4388
4389 /// To calculate offset of each component so we can correctly deduce
4390 /// alignments.
4391 const DataLayout &DL;
4392
4393 /// Initialize the splitter with an insertion point, Ptr and start with a
4394 /// single zero GEP index.
4395 OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4396 Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4397 : IRB(IRB), GEPIndices(1, IRB.getInt32(C: 0)), Ptr(Ptr), BaseTy(BaseTy),
4398 BaseAlign(BaseAlign), DL(DL) {
4399 IRB.SetInsertPoint(InsertionPoint);
4400 }
4401
4402 public:
4403 /// Generic recursive split emission routine.
4404 ///
4405 /// This method recursively splits an aggregate op (load or store) into
4406 /// scalar or vector ops. It splits recursively until it hits a single value
4407 /// and emits that single value operation via the template argument.
4408 ///
4409 /// The logic of this routine relies on GEPs and insertvalue and
4410 /// extractvalue all operating with the same fundamental index list, merely
4411 /// formatted differently (GEPs need actual values).
4412 ///
4413 /// \param Ty The type being split recursively into smaller ops.
4414 /// \param Agg The aggregate value being built up or stored, depending on
4415 /// whether this is splitting a load or a store respectively.
4416 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
4417 if (Ty->isSingleValueType()) {
4418 unsigned Offset = DL.getIndexedOffsetInType(ElemTy: BaseTy, Indices: GEPIndices);
4419 return static_cast<Derived *>(this)->emitFunc(
4420 Ty, Agg, commonAlignment(A: BaseAlign, Offset), Name);
4421 }
4422
4423 if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
4424 unsigned OldSize = Indices.size();
4425 (void)OldSize;
4426 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
4427 ++Idx) {
4428 assert(Indices.size() == OldSize && "Did not return to the old size");
4429 Indices.push_back(Elt: Idx);
4430 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4431 emitSplitOps(Ty: ATy->getElementType(), Agg, Name: Name + "." + Twine(Idx));
4432 GEPIndices.pop_back();
4433 Indices.pop_back();
4434 }
4435 return;
4436 }
4437
4438 if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4439 unsigned OldSize = Indices.size();
4440 (void)OldSize;
4441 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
4442 ++Idx) {
4443 assert(Indices.size() == OldSize && "Did not return to the old size");
4444 Indices.push_back(Elt: Idx);
4445 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4446 emitSplitOps(Ty: STy->getElementType(N: Idx), Agg, Name: Name + "." + Twine(Idx));
4447 GEPIndices.pop_back();
4448 Indices.pop_back();
4449 }
4450 return;
4451 }
4452
4453 llvm_unreachable("Only arrays and structs are aggregate loadable types");
4454 }
4455 };
4456
4457 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4458 AAMDNodes AATags;
4459 // A vector to hold the split components that we want to emit
4460 // separate fake uses for.
4461 SmallVector<Value *, 4> Components;
4462 // A vector to hold all the fake uses of the struct that we are splitting.
4463 // Usually there should only be one, but we are handling the general case.
4464 SmallVector<Instruction *, 1> FakeUses;
4465
4466 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4467 AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4468 IRBuilderTy &IRB)
4469 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4470 IRB),
4471 AATags(AATags) {}
4472
4473 /// Emit a leaf load of a single value. This is called at the leaves of the
4474 /// recursive emission to actually load values.
4475 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4476 assert(Ty->isSingleValueType());
4477 // Load the single value and insert it using the indices.
4478 Value *GEP =
4479 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4480 LoadInst *Load =
4481 IRB.CreateAlignedLoad(Ty, Ptr: GEP, Align: Alignment, Name: Name + ".load");
4482
4483 APInt Offset(
4484 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4485 if (AATags &&
4486 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset))
4487 Load->setAAMetadata(
4488 AATags.adjustForAccess(Offset: Offset.getZExtValue(), AccessTy: Load->getType(), DL));
4489 // Record the load so we can generate a fake use for this aggregate
4490 // component.
4491 Components.push_back(Elt: Load);
4492
4493 Agg = IRB.CreateInsertValue(Agg, Val: Load, Idxs: Indices, Name: Name + ".insert");
4494 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4495 }
4496
4497 // Stash the fake uses that use the value generated by this instruction.
4498 void recordFakeUses(LoadInst &LI) {
4499 for (Use &U : LI.uses())
4500 if (auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser()))
4501 if (II->getIntrinsicID() == Intrinsic::fake_use)
4502 FakeUses.push_back(Elt: II);
4503 }
4504
4505 // Replace all fake uses of the aggregate with a series of fake uses, one
4506 // for each split component.
4507 void emitFakeUses() {
4508 for (Instruction *I : FakeUses) {
4509 IRB.SetInsertPoint(I);
4510 for (auto *V : Components)
4511 IRB.CreateIntrinsic(ID: Intrinsic::fake_use, Args: {V});
4512 I->eraseFromParent();
4513 }
4514 }
4515 };
4516
4517 bool visitLoadInst(LoadInst &LI) {
4518 assert(LI.getPointerOperand() == *U);
4519 if (!LI.isSimple() || LI.getType()->isSingleValueType())
4520 return false;
4521
4522 // We have an aggregate being loaded, split it apart.
4523 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4524 LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4525 getAdjustedAlignment(I: &LI, Offset: 0), DL, IRB);
4526 Splitter.recordFakeUses(LI);
4527 Value *V = PoisonValue::get(T: LI.getType());
4528 Splitter.emitSplitOps(Ty: LI.getType(), Agg&: V, Name: LI.getName() + ".fca");
4529 Splitter.emitFakeUses();
4530 Visited.erase(Ptr: &LI);
4531 LI.replaceAllUsesWith(V);
4532 LI.eraseFromParent();
4533 return true;
4534 }
4535
4536 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4537 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4538 AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4539 const DataLayout &DL, IRBuilderTy &IRB)
4540 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4541 DL, IRB),
4542 AATags(AATags), AggStore(AggStore) {}
4543 AAMDNodes AATags;
4544 StoreInst *AggStore;
4545 /// Emit a leaf store of a single value. This is called at the leaves of the
4546 /// recursive emission to actually produce stores.
4547 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4548 assert(Ty->isSingleValueType());
4549 // Extract the single value and store it using the indices.
4550 //
4551 // The gep and extractvalue values are factored out of the CreateStore
4552 // call to make the output independent of the argument evaluation order.
4553 Value *ExtractValue =
4554 IRB.CreateExtractValue(Agg, Idxs: Indices, Name: Name + ".extract");
4555 Value *InBoundsGEP =
4556 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4557 StoreInst *Store =
4558 IRB.CreateAlignedStore(Val: ExtractValue, Ptr: InBoundsGEP, Align: Alignment);
4559
4560 APInt Offset(
4561 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4562 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset);
4563 if (AATags) {
4564 Store->setAAMetadata(AATags.adjustForAccess(
4565 Offset: Offset.getZExtValue(), AccessTy: ExtractValue->getType(), DL));
4566 }
4567
4568 // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4569 // If we cannot (because there's an intervening non-const or unbounded
4570 // gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4571 // this instruction.
4572 Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets();
4573 if (auto *OldAI = dyn_cast<AllocaInst>(Val: Base)) {
4574 uint64_t SizeInBits =
4575 DL.getTypeSizeInBits(Ty: Store->getValueOperand()->getType());
4576 migrateDebugInfo(OldAlloca: OldAI, /*IsSplit*/ true, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
4577 SliceSizeInBits: SizeInBits, OldInst: AggStore, Inst: Store,
4578 Dest: Store->getPointerOperand(), Value: Store->getValueOperand(),
4579 DL);
4580 } else {
4581 assert(at::getDVRAssignmentMarkers(Store).empty() &&
4582 "AT: unexpected debug.assign linked to store through "
4583 "unbounded GEP");
4584 }
4585 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4586 }
4587 };
4588
4589 bool visitStoreInst(StoreInst &SI) {
4590 if (!SI.isSimple() || SI.getPointerOperand() != *U)
4591 return false;
4592 Value *V = SI.getValueOperand();
4593 if (V->getType()->isSingleValueType())
4594 return false;
4595
4596 // We have an aggregate being stored, split it apart.
4597 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4598 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4599 getAdjustedAlignment(I: &SI, Offset: 0), DL, IRB);
4600 Splitter.emitSplitOps(Ty: V->getType(), Agg&: V, Name: V->getName() + ".fca");
4601 Visited.erase(Ptr: &SI);
4602 // The stores replacing SI each have markers describing fragments of the
4603 // assignment so delete the assignment markers linked to SI.
4604 at::deleteAssignmentMarkers(Inst: &SI);
4605 SI.eraseFromParent();
4606 return true;
4607 }
4608
4609 bool visitBitCastInst(BitCastInst &BC) {
4610 enqueueUsers(I&: BC);
4611 return false;
4612 }
4613
4614 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4615 enqueueUsers(I&: ASC);
4616 return false;
4617 }
4618
4619 // Unfold gep (select cond, ptr1, ptr2), idx
4620 // => select cond, gep(ptr1, idx), gep(ptr2, idx)
4621 // and gep ptr, (select cond, idx1, idx2)
4622 // => select cond, gep(ptr, idx1), gep(ptr, idx2)
4623 // We also allow for i1 zext indices, which are equivalent to selects.
4624 bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4625 // Check whether the GEP has exactly one select operand and all indices
4626 // will become constant after the transform.
4627 Instruction *Sel = dyn_cast<SelectInst>(Val: GEPI.getPointerOperand());
4628 for (Value *Op : GEPI.indices()) {
4629 if (auto *SI = dyn_cast<SelectInst>(Val: Op)) {
4630 if (Sel)
4631 return false;
4632
4633 Sel = SI;
4634 if (!isa<ConstantInt>(Val: SI->getTrueValue()) ||
4635 !isa<ConstantInt>(Val: SI->getFalseValue()))
4636 return false;
4637 continue;
4638 }
4639 if (auto *ZI = dyn_cast<ZExtInst>(Val: Op)) {
4640 if (Sel)
4641 return false;
4642 Sel = ZI;
4643 if (!ZI->getSrcTy()->isIntegerTy(BitWidth: 1))
4644 return false;
4645 continue;
4646 }
4647
4648 if (!isa<ConstantInt>(Val: Op))
4649 return false;
4650 }
4651
4652 if (!Sel)
4653 return false;
4654
4655 LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4656 dbgs() << " original: " << *Sel << "\n";
4657 dbgs() << " " << GEPI << "\n";);
4658
4659 auto GetNewOps = [&](Value *SelOp) {
4660 SmallVector<Value *> NewOps;
4661 for (Value *Op : GEPI.operands())
4662 if (Op == Sel)
4663 NewOps.push_back(Elt: SelOp);
4664 else
4665 NewOps.push_back(Elt: Op);
4666 return NewOps;
4667 };
4668
4669 Value *Cond, *True, *False;
4670 Instruction *MDFrom = nullptr;
4671 if (auto *SI = dyn_cast<SelectInst>(Val: Sel)) {
4672 Cond = SI->getCondition();
4673 True = SI->getTrueValue();
4674 False = SI->getFalseValue();
4675 MDFrom = SI;
4676 } else {
4677 Cond = Sel->getOperand(i: 0);
4678 True = ConstantInt::get(Ty: Sel->getType(), V: 1);
4679 False = ConstantInt::get(Ty: Sel->getType(), V: 0);
4680 }
4681 SmallVector<Value *> TrueOps = GetNewOps(True);
4682 SmallVector<Value *> FalseOps = GetNewOps(False);
4683
4684 IRB.SetInsertPoint(&GEPI);
4685 GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4686
4687 Type *Ty = GEPI.getSourceElementType();
4688 Value *NTrue = IRB.CreateGEP(Ty, Ptr: TrueOps[0], IdxList: ArrayRef(TrueOps).drop_front(),
4689 Name: True->getName() + ".sroa.gep", NW);
4690
4691 Value *NFalse =
4692 IRB.CreateGEP(Ty, Ptr: FalseOps[0], IdxList: ArrayRef(FalseOps).drop_front(),
4693 Name: False->getName() + ".sroa.gep", NW);
4694
4695 Value *NSel = MDFrom
4696 ? IRB.CreateSelect(C: Cond, True: NTrue, False: NFalse,
4697 Name: Sel->getName() + ".sroa.sel", MDFrom)
4698 : IRB.CreateSelectWithUnknownProfile(
4699 C: Cond, True: NTrue, False: NFalse, DEBUG_TYPE,
4700 Name: Sel->getName() + ".sroa.sel");
4701 Visited.erase(Ptr: &GEPI);
4702 GEPI.replaceAllUsesWith(V: NSel);
4703 GEPI.eraseFromParent();
4704 Instruction *NSelI = cast<Instruction>(Val: NSel);
4705 Visited.insert(Ptr: NSelI);
4706 enqueueUsers(I&: *NSelI);
4707
4708 LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4709 dbgs() << " " << *NFalse << "\n";
4710 dbgs() << " " << *NSel << "\n";);
4711
4712 return true;
4713 }
4714
4715 // Unfold gep (phi ptr1, ptr2), idx
4716 // => phi ((gep ptr1, idx), (gep ptr2, idx))
4717 // and gep ptr, (phi idx1, idx2)
4718 // => phi ((gep ptr, idx1), (gep ptr, idx2))
4719 bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4720 // To prevent infinitely expanding recursive phis, bail if the GEP pointer
4721 // operand (looking through the phi if it is the phi we want to unfold) is
4722 // an instruction besides a static alloca.
4723 PHINode *Phi = dyn_cast<PHINode>(Val: GEPI.getPointerOperand());
4724 auto IsInvalidPointerOperand = [](Value *V) {
4725 if (!isa<Instruction>(Val: V))
4726 return false;
4727 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
4728 return !AI->isStaticAlloca();
4729 return true;
4730 };
4731 if (Phi) {
4732 if (any_of(Range: Phi->operands(), P: IsInvalidPointerOperand))
4733 return false;
4734 } else {
4735 if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4736 return false;
4737 }
4738 // Check whether the GEP has exactly one phi operand (including the pointer
4739 // operand) and all indices will become constant after the transform.
4740 for (Value *Op : GEPI.indices()) {
4741 if (auto *SI = dyn_cast<PHINode>(Val: Op)) {
4742 if (Phi)
4743 return false;
4744
4745 Phi = SI;
4746 if (!all_of(Range: Phi->incoming_values(),
4747 P: [](Value *V) { return isa<ConstantInt>(Val: V); }))
4748 return false;
4749 continue;
4750 }
4751
4752 if (!isa<ConstantInt>(Val: Op))
4753 return false;
4754 }
4755
4756 if (!Phi)
4757 return false;
4758
4759 LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4760 dbgs() << " original: " << *Phi << "\n";
4761 dbgs() << " " << GEPI << "\n";);
4762
4763 auto GetNewOps = [&](Value *PhiOp) {
4764 SmallVector<Value *> NewOps;
4765 for (Value *Op : GEPI.operands())
4766 if (Op == Phi)
4767 NewOps.push_back(Elt: PhiOp);
4768 else
4769 NewOps.push_back(Elt: Op);
4770 return NewOps;
4771 };
4772
4773 IRB.SetInsertPoint(Phi);
4774 PHINode *NewPhi = IRB.CreatePHI(Ty: GEPI.getType(), NumReservedValues: Phi->getNumIncomingValues(),
4775 Name: Phi->getName() + ".sroa.phi");
4776
4777 Type *SourceTy = GEPI.getSourceElementType();
4778 // We only handle arguments, constants, and static allocas here, so we can
4779 // insert GEPs at the end of the entry block.
4780 IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4781 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4782 Value *Op = Phi->getIncomingValue(i: I);
4783 BasicBlock *BB = Phi->getIncomingBlock(i: I);
4784 Value *NewGEP;
4785 if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) {
4786 NewGEP = NewPhi->getIncomingValue(i: NI);
4787 } else {
4788 SmallVector<Value *> NewOps = GetNewOps(Op);
4789 NewGEP =
4790 IRB.CreateGEP(Ty: SourceTy, Ptr: NewOps[0], IdxList: ArrayRef(NewOps).drop_front(),
4791 Name: Phi->getName() + ".sroa.gep", NW: GEPI.getNoWrapFlags());
4792 }
4793 NewPhi->addIncoming(V: NewGEP, BB);
4794 }
4795
4796 Visited.erase(Ptr: &GEPI);
4797 GEPI.replaceAllUsesWith(V: NewPhi);
4798 GEPI.eraseFromParent();
4799 Visited.insert(Ptr: NewPhi);
4800 enqueueUsers(I&: *NewPhi);
4801
4802 LLVM_DEBUG(dbgs() << " to: ";
4803 for (Value *In
4804 : NewPhi->incoming_values()) dbgs()
4805 << "\n " << *In;
4806 dbgs() << "\n " << *NewPhi << '\n');
4807
4808 return true;
4809 }
4810
4811 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4812 if (unfoldGEPSelect(GEPI))
4813 return true;
4814
4815 if (unfoldGEPPhi(GEPI))
4816 return true;
4817
4818 enqueueUsers(I&: GEPI);
4819 return false;
4820 }
4821
4822 bool visitPHINode(PHINode &PN) {
4823 enqueueUsers(I&: PN);
4824 return false;
4825 }
4826
4827 bool visitSelectInst(SelectInst &SI) {
4828 enqueueUsers(I&: SI);
4829 return false;
4830 }
4831};
4832
4833} // end anonymous namespace
4834
4835/// Strip aggregate type wrapping.
4836///
4837/// This removes no-op aggregate types wrapping an underlying type. It will
4838/// strip as many layers of types as it can without changing either the type
4839/// size or the allocated size.
4840static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
4841 if (Ty->isSingleValueType())
4842 return Ty;
4843
4844 uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4845 uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4846
4847 Type *InnerTy;
4848 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Val: Ty)) {
4849 InnerTy = ArrTy->getElementType();
4850 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4851 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4852 unsigned Index = SL->getElementContainingOffset(FixedOffset: 0);
4853 InnerTy = STy->getElementType(N: Index);
4854 } else {
4855 return Ty;
4856 }
4857
4858 if (AllocSize > DL.getTypeAllocSize(Ty: InnerTy).getFixedValue() ||
4859 TypeSize > DL.getTypeSizeInBits(Ty: InnerTy).getFixedValue())
4860 return Ty;
4861
4862 return stripAggregateTypeWrapping(DL, Ty: InnerTy);
4863}
4864
4865/// Try to find a partition of the aggregate type passed in for a given
4866/// offset and size.
4867///
4868/// This recurses through the aggregate type and tries to compute a subtype
4869/// based on the offset and size. When the offset and size span a sub-section
4870/// of an array, it will even compute a new array type for that sub-section,
4871/// and the same for structs.
4872///
4873/// Note that this routine is very strict and tries to find a partition of the
4874/// type which produces the *exact* right offset and size. It is not forgiving
4875/// when the size or offset cause either end of type-based partition to be off.
4876/// Also, this is a best-effort routine. It is reasonable to give up and not
4877/// return a type if necessary.
4878static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
4879 uint64_t Size) {
4880 if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4881 return stripAggregateTypeWrapping(DL, Ty);
4882 if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() ||
4883 (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4884 return nullptr;
4885
4886 if (isa<ArrayType>(Val: Ty) || isa<VectorType>(Val: Ty)) {
4887 Type *ElementTy;
4888 uint64_t TyNumElements;
4889 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
4890 ElementTy = AT->getElementType();
4891 TyNumElements = AT->getNumElements();
4892 } else {
4893 // FIXME: This isn't right for vectors with non-byte-sized or
4894 // non-power-of-two sized elements.
4895 auto *VT = cast<FixedVectorType>(Val: Ty);
4896 ElementTy = VT->getElementType();
4897 TyNumElements = VT->getNumElements();
4898 }
4899 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4900 uint64_t NumSkippedElements = Offset / ElementSize;
4901 if (NumSkippedElements >= TyNumElements)
4902 return nullptr;
4903 Offset -= NumSkippedElements * ElementSize;
4904
4905 // First check if we need to recurse.
4906 if (Offset > 0 || Size < ElementSize) {
4907 // Bail if the partition ends in a different array element.
4908 if ((Offset + Size) > ElementSize)
4909 return nullptr;
4910 // Recurse through the element type trying to peel off offset bytes.
4911 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4912 }
4913 assert(Offset == 0);
4914
4915 if (Size == ElementSize)
4916 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4917 assert(Size > ElementSize);
4918 uint64_t NumElements = Size / ElementSize;
4919 if (NumElements * ElementSize != Size)
4920 return nullptr;
4921 return ArrayType::get(ElementType: ElementTy, NumElements);
4922 }
4923
4924 StructType *STy = dyn_cast<StructType>(Val: Ty);
4925 if (!STy)
4926 return nullptr;
4927
4928 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4929
4930 if (SL->getSizeInBits().isScalable())
4931 return nullptr;
4932
4933 if (Offset >= SL->getSizeInBytes())
4934 return nullptr;
4935 uint64_t EndOffset = Offset + Size;
4936 if (EndOffset > SL->getSizeInBytes())
4937 return nullptr;
4938
4939 unsigned Index = SL->getElementContainingOffset(FixedOffset: Offset);
4940 Offset -= SL->getElementOffset(Idx: Index);
4941
4942 Type *ElementTy = STy->getElementType(N: Index);
4943 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4944 if (Offset >= ElementSize)
4945 return nullptr; // The offset points into alignment padding.
4946
4947 // See if any partition must be contained by the element.
4948 if (Offset > 0 || Size < ElementSize) {
4949 if ((Offset + Size) > ElementSize)
4950 return nullptr;
4951 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4952 }
4953 assert(Offset == 0);
4954
4955 if (Size == ElementSize)
4956 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4957
4958 StructType::element_iterator EI = STy->element_begin() + Index,
4959 EE = STy->element_end();
4960 if (EndOffset < SL->getSizeInBytes()) {
4961 unsigned EndIndex = SL->getElementContainingOffset(FixedOffset: EndOffset);
4962 if (Index == EndIndex)
4963 return nullptr; // Within a single element and its padding.
4964
4965 // Don't try to form "natural" types if the elements don't line up with the
4966 // expected size.
4967 // FIXME: We could potentially recurse down through the last element in the
4968 // sub-struct to find a natural end point.
4969 if (SL->getElementOffset(Idx: EndIndex) != EndOffset)
4970 return nullptr;
4971
4972 assert(Index < EndIndex);
4973 EE = STy->element_begin() + EndIndex;
4974 }
4975
4976 // Try to build up a sub-structure.
4977 StructType *SubTy =
4978 StructType::get(Context&: STy->getContext(), Elements: ArrayRef(EI, EE), isPacked: STy->isPacked());
4979 const StructLayout *SubSL = DL.getStructLayout(Ty: SubTy);
4980 if (Size != SubSL->getSizeInBytes())
4981 return nullptr; // The sub-struct doesn't have quite the size needed.
4982
4983 return SubTy;
4984}
4985
4986/// This is a helper class that's used by presplitLoadsAndStores to split a
4987/// slice into smaller slices, including splitting the corresponding load/store
4988/// instruction.
4989class SliceSplitter {
4990 IRBuilderTy &IRB;
4991 const DataLayout &DL;
4992
4993 /// The split points that the slice is being split at. These are byte offsets
4994 /// within the slice, and are used to form the start/end offsets of the slices
4995 /// that we generate, plus the implicit offsets from the start and end of the
4996 /// slice that's being split.
4997 const std::vector<uint64_t> &Splits;
4998
4999 /// The size of the slice that's being split.
5000 unsigned SliceSize;
5001
5002public:
5003 SliceSplitter(IRBuilderTy &IRB, const DataLayout &DL,
5004 std::vector<uint64_t> &Splits, unsigned SliceSize)
5005 : IRB(IRB), DL(DL), Splits(Splits), SliceSize(SliceSize) {}
5006
5007 /// An iterator which iterates over the sub-slices that we want to split the
5008 /// slice into.
5009 struct Iterator
5010 : public iterator_facade_base<Iterator, std::forward_iterator_tag,
5011 const Iterator> {
5012 SliceSplitter &SS;
5013 unsigned Idx;
5014 Iterator(SliceSplitter &SS, bool IsEnd) : SS(SS) {
5015 Idx = IsEnd ? SS.Splits.size() + 1 : 0;
5016 }
5017 bool operator==(const Iterator &Other) const { return Idx == Other.Idx; }
5018 uint64_t getOffset() const { return Idx == 0 ? 0 : SS.Splits[Idx - 1]; }
5019 uint64_t getSize() const {
5020 uint64_t NextOffset =
5021 Idx < SS.Splits.size() ? SS.Splits[Idx] : SS.SliceSize;
5022 return NextOffset - getOffset();
5023 }
5024 Iterator &operator++() {
5025 ++Idx;
5026 return *this;
5027 }
5028 const Iterator &operator*() const { return *this; }
5029 };
5030
5031 Iterator begin() { return Iterator(*this, false); }
5032
5033 Iterator end() { return Iterator(*this, true); }
5034
5035 LoadInst *createLoad(Value *BasePtr, Iterator &It, Instruction *SplitI) {
5036 Type *PartTy = Type::getIntNTy(C&: SplitI->getContext(), N: It.getSize() * 8);
5037 Type *PartPtrTy = BasePtr->getType();
5038 unsigned AS = getLoadStoreAddressSpace(I: SplitI);
5039 LoadInst *PLoad = IRB.CreateAlignedLoad(
5040 Ty: PartTy,
5041 Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5042 Offset: APInt(DL.getIndexSizeInBits(AS), It.getOffset()),
5043 PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5044 Align: getAdjustedAlignment(I: SplitI, Offset: It.getOffset()),
5045 /*IsVolatile*/ isVolatile: false, Name: SplitI->getName());
5046 PLoad->copyMetadata(SrcInst: *SplitI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5047 LLVMContext::MD_access_group});
5048 return PLoad;
5049 }
5050
5051 StoreInst *createStore(Value *StoreVal, Value *BasePtr, Iterator &It,
5052 Instruction *SplitI) {
5053 Type *PartPtrTy = BasePtr->getType();
5054 unsigned AS = getLoadStoreAddressSpace(I: SplitI);
5055 StoreInst *PStore = IRB.CreateAlignedStore(
5056 Val: StoreVal,
5057 Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5058 Offset: APInt(DL.getIndexSizeInBits(AS), It.getOffset()),
5059 PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5060 Align: getAdjustedAlignment(I: SplitI, Offset: It.getOffset()),
5061 /*IsVolatile*/ isVolatile: false);
5062 PStore->copyMetadata(SrcInst: *SplitI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5063 LLVMContext::MD_access_group,
5064 LLVMContext::MD_DIAssignID});
5065 AAMDNodes AATags = SplitI->getAAMetadata();
5066 if (AATags)
5067 PStore->setAAMetadata(
5068 AATags.adjustForAccess(Offset: It.getOffset(), AccessTy: StoreVal->getType(), DL));
5069
5070 return PStore;
5071 }
5072
5073 Slice createSlice(Instruction *I, uint64_t BaseOffset, Iterator &It) {
5074 unsigned PtrIdx = 0;
5075 if (auto *LI = dyn_cast<LoadInst>(Val: I))
5076 PtrIdx = LI->getPointerOperandIndex();
5077 else if (auto *SI = dyn_cast<StoreInst>(Val: I))
5078 PtrIdx = SI->getPointerOperandIndex();
5079 else
5080 llvm_unreachable("Slice must be for a load or store");
5081 // BaseOffset is the offset of the original slice within the alloca, so the
5082 // start and end offsets of this slice are offset from that.
5083 Slice S(BaseOffset + It.getOffset(),
5084 BaseOffset + It.getOffset() + It.getSize(),
5085 &I->getOperandUse(i: PtrIdx), /*IsSplittable*/ false);
5086 LLVM_DEBUG(dbgs() << " new slice [" << S.beginOffset() << ", "
5087 << S.endOffset() << "): " << *I << "\n");
5088 return S;
5089 }
5090};
5091
5092/// Try to find a slice in the map that partially overlaps with S, i.e. some but
5093/// not all of its range is contained within the range of S, and which forms a
5094/// load/store pair with S (i.e. the two slices are used to perform a
5095/// memmove-like copy). SliceMap is expected to contain the slices for a single
5096/// Partition.
5097static Slice *findOverlappingCopySlice(Slice &S,
5098 InstructionSliceMap &SliceMap) {
5099 // Single byte slices can't partially overlap anything
5100 if (S.endOffset() - S.beginOffset() == 1)
5101 return nullptr;
5102 // The source/destination of this slice needs to be a memory instruction
5103 // whose slice overlaps this one.
5104 Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
5105 Instruction *J = nullptr;
5106 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5107 if (!LI->hasOneUser())
5108 return nullptr;
5109 J = cast<Instruction>(Val: *LI->user_begin());
5110 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5111 J = dyn_cast<Instruction>(Val: SI->getValueOperand());
5112 if (!J || !J->hasOneUser())
5113 return nullptr;
5114 } else {
5115 return nullptr;
5116 }
5117 // Check if there's a slice that corresponds to J that overlaps this slice
5118 for (Slice *JS : SliceMap[J]) {
5119 if (S.beginOffset() > JS->beginOffset() &&
5120 S.beginOffset() < JS->endOffset())
5121 return JS;
5122 if (JS->beginOffset() > S.beginOffset() &&
5123 JS->beginOffset() < S.endOffset())
5124 return JS;
5125 }
5126 return nullptr;
5127}
5128
5129/// Pre-split loads and stores to simplify rewriting.
5130///
5131/// We want to break up the splittable load+store pairs as much as
5132/// possible. This is important to do as a preprocessing step, as once we
5133/// start rewriting the accesses to partitions of the alloca we lose the
5134/// necessary information to correctly split apart paired loads and stores
5135/// which both point into this alloca. The case to consider is something like
5136/// the following:
5137///
5138/// %a = alloca [12 x i8]
5139/// %gep1 = getelementptr i8, ptr %a, i32 0
5140/// %gep2 = getelementptr i8, ptr %a, i32 4
5141/// %gep3 = getelementptr i8, ptr %a, i32 8
5142/// store float 0.0, ptr %gep1
5143/// store float 1.0, ptr %gep2
5144/// %v = load i64, ptr %gep1
5145/// store i64 %v, ptr %gep2
5146/// %f1 = load float, ptr %gep2
5147/// %f2 = load float, ptr %gep3
5148///
5149/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
5150/// promote everything so we recover the 2 SSA values that should have been
5151/// there all along.
5152///
5153/// \returns true if any changes are made.
5154bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
5155 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
5156
5157 // Track the loads and stores which are candidates for pre-splitting here, in
5158 // the order they first appear during the partition scan. These give stable
5159 // iteration order and a basis for tracking which loads and stores we
5160 // actually split.
5161 SmallVector<LoadInst *, 4> Loads;
5162 SmallVector<StoreInst *, 4> Stores;
5163
5164 // We need to accumulate the splits required of each load or store where we
5165 // can find them via a direct lookup. This is important to cross-check loads
5166 // and stores against each other. We also track the slice so that we can kill
5167 // all the slices that end up split.
5168 struct SplitOffsets {
5169 Slice *S;
5170 std::vector<uint64_t> Splits;
5171 };
5172 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
5173
5174 // Track loads out of this alloca which cannot, for any reason, be pre-split.
5175 // This is important as we also cannot pre-split stores of those loads!
5176 // FIXME: This is all pretty gross. It means that we can be more aggressive
5177 // in pre-splitting when the load feeding the store happens to come from
5178 // a separate alloca. Put another way, the effectiveness of SROA would be
5179 // decreased by a frontend which just concatenated all of its local allocas
5180 // into one big flat alloca. But defeating such patterns is exactly the job
5181 // SROA is tasked with! Sadly, to not have this discrepancy we would have
5182 // change store pre-splitting to actually force pre-splitting of the load
5183 // that feeds it *and all stores*. That makes pre-splitting much harder, but
5184 // maybe it would make it more principled?
5185 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
5186
5187 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
5188 for (auto &P : AS.partitions()) {
5189 InstructionSliceMap SliceMap;
5190 for (Slice &S : P) {
5191 SliceMap[cast<Instruction>(Val: S.getUse()->getUser())].insert(Ptr: &S);
5192 }
5193 for (Slice &S : P) {
5194 Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
5195 bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
5196 Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
5197 if (!S.isSplittable() || (!ExtendsPastPartitionEnd && !CopyOverlap)) {
5198 // If this is a load we have to track that it can't participate in any
5199 // pre-splitting. If this is a store of a load we have to track that
5200 // that load also can't participate in any pre-splitting.
5201 if (auto *LI = dyn_cast<LoadInst>(Val: I))
5202 UnsplittableLoads.insert(Ptr: LI);
5203 else if (auto *SI = dyn_cast<StoreInst>(Val: I))
5204 if (auto *LI = dyn_cast<LoadInst>(Val: SI->getValueOperand()))
5205 UnsplittableLoads.insert(Ptr: LI);
5206 continue;
5207 }
5208 assert(P.endOffset() > S.beginOffset() &&
5209 "Empty or backwards partition!");
5210
5211 // Determine if this is a pre-splittable slice.
5212 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5213 assert(!LI->isVolatile() && "Cannot split volatile loads!");
5214
5215 // The load must be used exclusively to store into other pointers for
5216 // us to be able to arbitrarily pre-split it. The stores must also be
5217 // simple to avoid changing semantics.
5218 auto IsLoadSimplyStored = [](LoadInst *LI) {
5219 for (User *LU : LI->users()) {
5220 auto *SI = dyn_cast<StoreInst>(Val: LU);
5221 if (!SI || !SI->isSimple())
5222 return false;
5223 }
5224 return true;
5225 };
5226 if (!IsLoadSimplyStored(LI)) {
5227 UnsplittableLoads.insert(Ptr: LI);
5228 continue;
5229 }
5230
5231 Loads.push_back(Elt: LI);
5232 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5233 if (S.getUse() != &SI->getOperandUse(i: SI->getPointerOperandIndex()))
5234 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
5235 continue;
5236 auto *StoredLoad = dyn_cast<LoadInst>(Val: SI->getValueOperand());
5237 if (!StoredLoad || !StoredLoad->isSimple())
5238 continue;
5239 assert(!SI->isVolatile() && "Cannot split volatile stores!");
5240
5241 Stores.push_back(Elt: SI);
5242 } else {
5243 // Other uses cannot be pre-split.
5244 continue;
5245 }
5246
5247 // Record the initial split.
5248 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
5249 auto &Offsets = SplitOffsetsMap[I];
5250 assert(Offsets.Splits.empty() &&
5251 "Should not have splits the first time we see an instruction!");
5252 Offsets.S = &S;
5253 if (CopyOverlap) {
5254 // S is being moved to CopyOverlap, which overlaps with S, so we split S
5255 // into three parts:
5256 // * Initially outside the overlap and copied into the overlap.
5257 // * Either initially inside the overlap and remains inside it after
5258 // the copy, or initially outside and remains outside, depending on
5259 // if the overlap or non-overlap area is larger, and which will be
5260 // empty if they are equal.
5261 // * Initially inside the overlap and copied outside the overlap.
5262 // This should result in the first and last parts being promoted to
5263 // scalars. If the middle part is an overlapping copy then we repeat
5264 // this process until it isn't.
5265 uint64_t OverlapStart =
5266 std::max(a: S.beginOffset(), b: CopyOverlap->beginOffset());
5267 uint64_t OverlapEnd = std::min(a: S.endOffset(), b: CopyOverlap->endOffset());
5268 uint64_t OverlapSize = OverlapEnd - OverlapStart;
5269 uint64_t SliceSize = S.endOffset() - S.beginOffset();
5270 uint64_t NonOverlapSize = SliceSize - OverlapSize;
5271 if (OverlapSize < NonOverlapSize) {
5272 // When the overlap area is smaller the middle part is initially and
5273 // remains outside the overlap, so splitting once is enough. The size
5274 // of the first and last part both equal the overlap size, so the
5275 // split points are that amount from the start and end of the slice.
5276 Offsets.Splits.push_back(x: OverlapSize);
5277 Offsets.Splits.push_back(x: SliceSize - OverlapSize);
5278 } else if (OverlapSize > NonOverlapSize) {
5279 // When the overlap area is larger the middle part is initially and
5280 // remains inside the overlap, so we repeatedly split it.
5281 for (uint64_t Split = NonOverlapSize; Split <= SliceSize / 2;
5282 Split += NonOverlapSize) {
5283 Offsets.Splits.push_back(x: Split);
5284 if (Split != SliceSize - Split) {
5285 Offsets.Splits.push_back(x: SliceSize - Split);
5286 }
5287 }
5288 // Sort the splits as they need to be in ascending order.
5289 llvm::sort(C&: Offsets.Splits);
5290 } else {
5291 // Here the overlap and non-overlap size are the same, so the middle
5292 // part is empty and the slice is split exactly in the center.
5293 Offsets.Splits.push_back(x: OverlapSize);
5294 }
5295 } else {
5296 Offsets.Splits.push_back(x: P.endOffset() - S.beginOffset());
5297 }
5298 }
5299 }
5300
5301 // We may have split loads where some of their stores are split stores. For
5302 // such loads and stores, we can only pre-split them if their splits exactly
5303 // match relative to their starting offset. We have to verify this prior to
5304 // any rewriting.
5305 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
5306 // Lookup the load we are storing in our map of split
5307 // offsets.
5308 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5309 // If it was completely unsplittable, then we're done,
5310 // and this store can't be pre-split.
5311 if (UnsplittableLoads.count(Ptr: LI))
5312 return true;
5313
5314 auto LoadOffsetsI = SplitOffsetsMap.find(Val: LI);
5315 if (LoadOffsetsI == SplitOffsetsMap.end())
5316 return false; // Unrelated loads are definitely safe.
5317 auto &LoadOffsets = LoadOffsetsI->second;
5318
5319 // Now lookup the store's offsets.
5320 auto &StoreOffsets = SplitOffsetsMap[SI];
5321
5322 // If the relative offsets of each split in the load and
5323 // store match exactly, then we can split them and we
5324 // don't need to remove them here.
5325 if (LoadOffsets.Splits == StoreOffsets.Splits)
5326 return false;
5327
5328 LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
5329 << " " << *LI << "\n"
5330 << " " << *SI << "\n");
5331
5332 // We've found a store and load that we need to split
5333 // with mismatched relative splits. Just give up on them
5334 // and remove both instructions from our list of
5335 // candidates.
5336 UnsplittableLoads.insert(Ptr: LI);
5337 return true;
5338 });
5339 // Now we have to go *back* through all the stores, because a later store may
5340 // have caused an earlier store's load to become unsplittable and if it is
5341 // unsplittable for the later store, then we can't rely on it being split in
5342 // the earlier store either.
5343 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads](StoreInst *SI) {
5344 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5345 return UnsplittableLoads.count(Ptr: LI);
5346 });
5347 // Once we've established all the loads that can't be split for some reason,
5348 // filter any that made it into our list out.
5349 llvm::erase_if(C&: Loads, P: [&UnsplittableLoads](LoadInst *LI) {
5350 return UnsplittableLoads.count(Ptr: LI);
5351 });
5352
5353 // If no loads or stores are left, there is no pre-splitting to be done for
5354 // this alloca.
5355 if (Loads.empty() && Stores.empty())
5356 return false;
5357
5358 // From here on, we can't fail and will be building new accesses, so rig up
5359 // an IR builder.
5360 IRBuilderTy IRB(&AI);
5361 IRB.disableGEPCanonicalization();
5362
5363 // Collect the new slices which we will merge into the alloca slices.
5364 SmallVector<Slice, 4> NewSlices;
5365
5366 // Track any allocas we end up splitting loads and stores for so we iterate
5367 // on them.
5368 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
5369
5370 // At this point, we have collected all of the loads and stores we can
5371 // pre-split, and the specific splits needed for them. We actually do the
5372 // splitting in a specific order in order to handle when one of the loads in
5373 // the value operand to one of the stores.
5374 //
5375 // First, we rewrite all of the split loads, and just accumulate each split
5376 // load in a parallel structure. We also build the slices for them and append
5377 // them to the alloca slices.
5378 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
5379 std::vector<LoadInst *> SplitLoads;
5380 const DataLayout &DL = AI.getDataLayout();
5381 for (LoadInst *LI : Loads) {
5382 SplitLoads.clear();
5383
5384 auto &Offsets = SplitOffsetsMap[LI];
5385 unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5386 assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&
5387 "Load must have type size equal to store size");
5388 assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&
5389 "Load must be >= slice size");
5390
5391 uint64_t BaseOffset = Offsets.S->beginOffset();
5392 assert(BaseOffset + SliceSize > BaseOffset &&
5393 "Cannot represent alloca access size using 64-bit integers!");
5394
5395 Instruction *BasePtr = cast<Instruction>(Val: LI->getPointerOperand());
5396 IRB.SetInsertPoint(LI);
5397
5398 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5399
5400 SliceSplitter Splitter(IRB, DL, Offsets.Splits, SliceSize);
5401 for (auto Part : Splitter) {
5402 LoadInst *PLoad = Splitter.createLoad(BasePtr, It&: Part, SplitI: LI);
5403
5404 // Append this load onto the list of split loads so we can find it later
5405 // to rewrite the stores.
5406 SplitLoads.push_back(x: PLoad);
5407
5408 // Now build a new slice for the alloca.
5409 NewSlices.push_back(Elt: Splitter.createSlice(I: PLoad, BaseOffset, It&: Part));
5410 }
5411
5412 // Now that we have the split loads, do the slow walk over all uses of the
5413 // load and rewrite them as split stores, or save the split loads to use
5414 // below if the store is going to be split there anyways.
5415 bool DeferredStores = false;
5416 for (User *LU : LI->users()) {
5417 StoreInst *SI = cast<StoreInst>(Val: LU);
5418 if (!Stores.empty() && SplitOffsetsMap.count(Val: SI)) {
5419 DeferredStores = true;
5420 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5421 << "\n");
5422 continue;
5423 }
5424
5425 Value *StoreBasePtr = SI->getPointerOperand();
5426 IRB.SetInsertPoint(SI);
5427
5428 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5429
5430 for (auto Part : Splitter) {
5431 LoadInst *PLoad = SplitLoads[Part.Idx];
5432 StoreInst *PStore = Splitter.createStore(StoreVal: PLoad, BasePtr: StoreBasePtr, It&: Part, SplitI: SI);
5433 (void)PStore;
5434 LLVM_DEBUG(dbgs() << " +" << Part.getOffset() << ":" << *PStore
5435 << "\n");
5436 }
5437
5438 // We want to immediately iterate on any allocas impacted by splitting
5439 // this store, and we have to track any promotable alloca (indicated by
5440 // a direct store) as needing to be resplit because it is no longer
5441 // promotable.
5442 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: StoreBasePtr)) {
5443 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5444 Worklist.insert(X: OtherAI);
5445 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5446 Val: StoreBasePtr->stripInBoundsOffsets())) {
5447 Worklist.insert(X: OtherAI);
5448 }
5449
5450 // Mark the original store as dead.
5451 DeadInsts.push_back(Elt: SI);
5452 }
5453
5454 // Save the split loads if there are deferred stores among the users.
5455 if (DeferredStores)
5456 SplitLoadsMap.insert(KV: std::make_pair(x&: LI, y: std::move(SplitLoads)));
5457
5458 // Mark the original load as dead and kill the original slice.
5459 DeadInsts.push_back(Elt: LI);
5460 Offsets.S->kill();
5461 }
5462
5463 // Second, we rewrite all of the split stores. At this point, we know that
5464 // all loads from this alloca have been split already. For stores of such
5465 // loads, we can simply look up the pre-existing split loads. For stores of
5466 // other loads, we split those loads first and then write split stores of
5467 // them.
5468 for (StoreInst *SI : Stores) {
5469 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5470 IntegerType *Ty = cast<IntegerType>(Val: LI->getType());
5471 assert(Ty->getBitWidth() % 8 == 0);
5472 uint64_t StoreSize = Ty->getBitWidth() / 8;
5473 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
5474
5475 auto &Offsets = SplitOffsetsMap[SI];
5476 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5477 "Slice size should always match load size exactly!");
5478 uint64_t BaseOffset = Offsets.S->beginOffset();
5479 assert(BaseOffset + StoreSize > BaseOffset &&
5480 "Cannot represent alloca access size using 64-bit integers!");
5481
5482 Value *LoadBasePtr = LI->getPointerOperand();
5483 Instruction *StoreBasePtr = cast<Instruction>(Val: SI->getPointerOperand());
5484
5485 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5486
5487 // Check whether we have an already split load.
5488 auto SplitLoadsMapI = SplitLoadsMap.find(Val: LI);
5489 std::vector<LoadInst *> *SplitLoads = nullptr;
5490 if (SplitLoadsMapI != SplitLoadsMap.end()) {
5491 SplitLoads = &SplitLoadsMapI->second;
5492 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
5493 "Too few split loads for the number of splits in the store!");
5494 } else {
5495 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5496 }
5497
5498 SliceSplitter Splitter(IRB, DL, Offsets.Splits, StoreSize);
5499 for (auto Part : Splitter) {
5500 // Either lookup a split load or create one.
5501 LoadInst *PLoad;
5502 if (SplitLoads) {
5503 PLoad = (*SplitLoads)[Part.Idx];
5504 } else {
5505 IRB.SetInsertPoint(LI);
5506 PLoad = Splitter.createLoad(BasePtr: LoadBasePtr, It&: Part, SplitI: LI);
5507 }
5508
5509 // And store this partition.
5510 IRB.SetInsertPoint(SI);
5511 StoreInst *PStore = Splitter.createStore(StoreVal: PLoad, BasePtr: StoreBasePtr, It&: Part, SplitI: SI);
5512
5513 // Now build a new slice for the alloca.
5514 NewSlices.push_back(Elt: Splitter.createSlice(I: PStore, BaseOffset, It&: Part));
5515 if (!SplitLoads) {
5516 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5517 }
5518 }
5519
5520 // We want to immediately iterate on any allocas impacted by splitting
5521 // this load, which is only relevant if it isn't a load of this alloca and
5522 // thus we didn't already split the loads above. We also have to keep track
5523 // of any promotable allocas we split loads on as they can no longer be
5524 // promoted.
5525 if (!SplitLoads) {
5526 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: LoadBasePtr)) {
5527 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5528 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5529 Worklist.insert(X: OtherAI);
5530 } else if (AllocaInst *StrippedAI = dyn_cast<AllocaInst>(
5531 Val: LoadBasePtr->stripInBoundsOffsets())) {
5532 assert(StrippedAI != &AI && "We can't re-split our own alloca!");
5533 Worklist.insert(X: StrippedAI);
5534 }
5535 }
5536
5537 // Mark the original store as dead now that we've split it up and kill its
5538 // slice. Note that we leave the original load in place unless this store
5539 // was its only use. It may in turn be split up if it is an alloca load
5540 // for some other alloca, but it may be a normal load. This may introduce
5541 // redundant loads, but where those can be merged the rest of the optimizer
5542 // should handle the merging, and this uncovers SSA splits which is more
5543 // important. In practice, the original loads will almost always be fully
5544 // split and removed eventually, and the splits will be merged by any
5545 // trivial CSE, including instcombine.
5546 if (LI->hasOneUse()) {
5547 assert(*LI->user_begin() == SI && "Single use isn't this store!");
5548 DeadInsts.push_back(Elt: LI);
5549 }
5550 DeadInsts.push_back(Elt: SI);
5551 Offsets.S->kill();
5552 }
5553
5554 // Remove the killed slices that have ben pre-split.
5555 llvm::erase_if(C&: AS, P: [](const Slice &S) { return S.isDead(); });
5556
5557 // Insert our new slices. This will sort and merge them into the sorted
5558 // sequence.
5559 AS.insert(NewSlices);
5560
5561 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5562#ifndef NDEBUG
5563 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5564 LLVM_DEBUG(AS.print(dbgs(), I, " "));
5565#endif
5566
5567 // Finally, don't try to promote any allocas that new require re-splitting.
5568 // They have already been added to the worklist above.
5569 PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5570
5571 return true;
5572}
5573
5574/// Try to canonicalize a homogeneous struct partition to a vector type.
5575///
5576/// We can do this if all the elements of the struct are the same and the
5577/// corresponding vector has the same byte-level layout. This can sometimes
5578/// eliminate allocas because structs cannot get promoted to LLVM values, but
5579/// vectors can.
5580///
5581/// We only apply this transformation when all users of the partition are memory
5582/// intrinsics. Otherwise, if there is a load or store of some other type to the
5583/// partition, SROA would select that type.
5584///
5585/// Applying this transformation too early may hinder memcpyopt, which may
5586/// generate better code when eliminating allocas. For example, see
5587/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5588/// this before memcpyopt can initialize previously uninitialized memory when
5589/// the alloca gets promoted to an SSA value. For another example, see
5590/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5591/// this before memcpyopt can result in promoting an alloca so that we load a
5592/// temporary value instead of copying the temporary value into memory, whereas
5593/// memcpyopt eliminates the temporary altogether.
5594///
5595/// As such, we only apply this transformation after memcpyopt has run. We gate
5596/// this transformation by the "AggregateToVector" pass option.
5597static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
5598 Partition &P,
5599 const DataLayout &DL) {
5600 unsigned NumElts = STy->getNumElements();
5601
5602 Type *EltTy = STy->getElementType(N: 0);
5603 if (!llvm::all_equal(Range: STy->elements()))
5604 return nullptr;
5605
5606 bool IsIntegralPointerTy =
5607 EltTy->isPointerTy() && !DL.isNonIntegralPointerType(Ty: EltTy);
5608 if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5609 !IsIntegralPointerTy)
5610 return nullptr;
5611
5612 // Ensure the struct is tightly packed so that the bit-layout is the same as
5613 // the corresponding vector. For example, this prevents a miscompile for
5614 // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5615 // tightly packed elements and trailing padding.
5616 if (DL.getTypeSizeInBits(Ty: EltTy) != DL.getTypeAllocSizeInBits(Ty: EltTy))
5617 return nullptr;
5618
5619 auto *VTy = FixedVectorType::get(ElementType: EltTy, NumElts);
5620 TypeSize StructSize = DL.getStructLayout(Ty: STy)->getSizeInBytes();
5621 TypeSize VectorSize = DL.getTypeStoreSize(Ty: VTy);
5622 // After ruling out per-element padding, make sure a vector load/store
5623 // covers the same number of bytes as the struct layout.
5624 if (StructSize != VectorSize)
5625 return nullptr;
5626
5627 auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5628 if (S.isDead())
5629 return true;
5630 auto *U = S.getUse();
5631 return !U || isa<LifetimeIntrinsic, MemIntrinsic>(Val: U->getUser());
5632 };
5633
5634 for (const Slice &S : P)
5635 if (!IsIgnorableOrMemIntrinsicSlice(S))
5636 return nullptr;
5637
5638 for (const Slice *S : P.splitSliceTails())
5639 if (!IsIgnorableOrMemIntrinsicSlice(*S))
5640 return nullptr;
5641
5642 return VTy;
5643}
5644
5645/// Select a partition type for an alloca partition.
5646///
5647/// Try to compute a friendly type for this partition of the alloca. This
5648/// won't always succeed, in which case we fall back to a legal integer type
5649/// or an i8 array of an appropriate size.
5650///
5651/// \returns A tuple with the following elements:
5652/// - PartitionType: The computed type for this partition.
5653/// - IsIntegerWideningViable: True if integer widening promotion is used.
5654/// - VectorType: The vector type if vector promotion is used, otherwise
5655/// nullptr.
5656static std::tuple<Type *, bool, VectorType *>
5657selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
5658 LLVMContext &C, bool AggregateToVector) {
5659 auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5660 VectorType *SelectedVecTy, bool SelectedIntWidening) {
5661 LLVM_DEBUG({
5662 dbgs() << "selectPartitionType path=" << Path
5663 << " func=" << AI.getFunction()->getName() << " alloca=";
5664 if (AI.hasName())
5665 dbgs() << AI.getName();
5666 else
5667 dbgs() << "<unnamed>";
5668 dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5669 << ") size=" << P.size();
5670 if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5671 dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5672 if (SelectedTy)
5673 dbgs() << " chosen=" << *SelectedTy;
5674 if (SelectedVecTy)
5675 dbgs() << " vec=" << *SelectedVecTy;
5676 dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5677 });
5678 };
5679 // First check if the partition is viable for vector promotion.
5680 //
5681 // We prefer vector promotion over integer widening promotion when:
5682 // - The vector element type is a floating-point type.
5683 // - All the loads/stores to the alloca are vector loads/stores to the
5684 // entire alloca or load/store a single element of the vector.
5685 //
5686 // Otherwise when there is an integer vector with mixed type loads/stores we
5687 // prefer integer widening promotion because it's more likely the user is
5688 // doing bitwise arithmetic and we generate better code.
5689 VectorType *VecTy =
5690 isVectorPromotionViable(P, DL, VScale: AI.getFunction()->getVScaleValue());
5691 // If the vector element type is a floating-point type, we prefer vector
5692 // promotion. If the vector has one element, let the below code select
5693 // whether we promote with the vector or scalar.
5694 if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5695 VecTy->getElementCount().getFixedValue() > 1) {
5696 LogSelection("direct-fp-vecty", VecTy, VecTy, false);
5697 return {VecTy, false, VecTy};
5698 }
5699
5700 // Check if there is a common type that all slices of the partition use that
5701 // spans the partition.
5702 auto [CommonUseTy, LargestIntTy] =
5703 findCommonType(B: P.begin(), E: P.end(), EndOffset: P.endOffset());
5704 if (CommonUseTy) {
5705 TypeSize CommonUseSize = DL.getTypeAllocSize(Ty: CommonUseTy);
5706 if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5707 // We prefer vector promotion here because if vector promotion is viable
5708 // and there is a common type used, then it implies the second listed
5709 // condition for preferring vector promotion is true.
5710 if (VecTy) {
5711 LogSelection("common-type-vecty", VecTy, VecTy, false);
5712 return {VecTy, false, VecTy};
5713 }
5714 bool IntWiden = isIntegerWideningViable(P, AllocaTy: CommonUseTy, DL);
5715 LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
5716 return {CommonUseTy, IntWiden, nullptr};
5717 }
5718 }
5719
5720 // Can we find an appropriate subtype in the original allocated
5721 // type?
5722 if (Type *TypePartitionTy = getTypePartition(DL, Ty: AI.getAllocatedType(),
5723 Offset: P.beginOffset(), Size: P.size())) {
5724 // If the partition is an integer array that can be spanned by a legal
5725 // integer type, prefer to represent it as a legal integer type because
5726 // it's more likely to be promotable.
5727 if (TypePartitionTy->isArrayTy() &&
5728 TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5729 DL.isLegalInteger(Width: P.size() * 8))
5730 TypePartitionTy = Type::getIntNTy(C, N: P.size() * 8);
5731 // There was no common type used, so we prefer integer widening promotion.
5732 if (isIntegerWideningViable(P, AllocaTy: TypePartitionTy, DL)) {
5733 LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
5734 return {TypePartitionTy, true, nullptr};
5735 }
5736 if (VecTy) {
5737 LogSelection("type-partition-vecty", VecTy, VecTy, false);
5738 return {VecTy, false, VecTy};
5739 }
5740 // If we couldn't promote with TypePartitionTy, try with the largest
5741 // integer type used.
5742 if (LargestIntTy &&
5743 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size() &&
5744 isIntegerWideningViable(P, AllocaTy: LargestIntTy, DL)) {
5745 LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
5746 return {LargestIntTy, true, nullptr};
5747 }
5748
5749 // Try homogeneous struct to vector canonicalization when requested. Running
5750 // this too early can hide memcpy chains from MemCpyOpt.
5751 if (AggregateToVector) {
5752 if (auto *STy = dyn_cast<StructType>(Val: TypePartitionTy)) {
5753 if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5754 LogSelection("struct-fallback-vecty", VTy, nullptr, false);
5755 return {VTy, false, nullptr};
5756 }
5757 }
5758 }
5759
5760 // Fallback to TypePartitionTy and we probably won't promote.
5761 LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
5762 return {TypePartitionTy, false, nullptr};
5763 }
5764
5765 // Select the largest integer type used if it spans the partition.
5766 if (LargestIntTy &&
5767 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size()) {
5768 LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
5769 return {LargestIntTy, false, nullptr};
5770 }
5771
5772 // Select a legal integer type if it spans the partition.
5773 if (DL.isLegalInteger(Width: P.size() * 8)) {
5774 Type *IntTy = Type::getIntNTy(C, N: P.size() * 8);
5775 LogSelection("legal-int-fallback", IntTy, nullptr, false);
5776 return {IntTy, false, nullptr};
5777 }
5778
5779 // Fallback to an i8 array.
5780 Type *ArrayTy = ArrayType::get(ElementType: Type::getInt8Ty(C), NumElements: P.size());
5781 LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
5782 return {ArrayTy, false, nullptr};
5783}
5784
5785/// Rewrite an alloca partition's users.
5786///
5787/// This routine drives both of the rewriting goals of the SROA pass. It tries
5788/// to rewrite uses of an alloca partition to be conducive for SSA value
5789/// promotion. If the partition needs a new, more refined alloca, this will
5790/// build that new alloca, preserving as much type information as possible, and
5791/// rewrite the uses of the old alloca to point at the new one and have the
5792/// appropriate new offsets. It also evaluates how successful the rewrite was
5793/// at enabling promotion and if it was successful queues the alloca to be
5794/// promoted.
5795std::pair<AllocaInst *, uint64_t>
5796SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5797 const DataLayout &DL = AI.getDataLayout();
5798 // Select the type for the new alloca that spans the partition.
5799 auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5800 selectPartitionType(P, DL, AI, C&: *C, AggregateToVector);
5801
5802 // Check for the case where we're going to rewrite to a new alloca of the
5803 // exact same type as the original, and with the same access offsets. In that
5804 // case, re-use the existing alloca, but still run through the rewriter to
5805 // perform phi and select speculation.
5806 // P.beginOffset() can be non-zero even with the same type in a case with
5807 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5808 AllocaInst *NewAI;
5809 if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == 0) {
5810 NewAI = &AI;
5811 // FIXME: We should be able to bail at this point with "nothing changed".
5812 // FIXME: We might want to defer PHI speculation until after here.
5813 // FIXME: return nullptr;
5814 } else {
5815 // Make sure the alignment is compatible with P.beginOffset().
5816 const Align Alignment = commonAlignment(A: AI.getAlign(), Offset: P.beginOffset());
5817 NewAI =
5818 new AllocaInst(PartitionTy, AI.getAddressSpace(), nullptr, Alignment,
5819 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()),
5820 AI.getIterator());
5821 tryEnforceAlignment(V: NewAI, PrefAlign: DL.getPrefTypeAlign(Ty: PartitionTy), DL);
5822 // Copy the old AI debug location over to the new one.
5823 NewAI->setDebugLoc(AI.getDebugLoc());
5824 ++NumNewAllocas;
5825 }
5826
5827 LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5828 << "," << P.endOffset() << ") to: " << *NewAI << "\n");
5829
5830 // Track the high watermark on the worklist as it is only relevant for
5831 // promoted allocas. We will reset it to this point if the alloca is not in
5832 // fact scheduled for promotion.
5833 unsigned PPWOldSize = PostPromotionWorklist.size();
5834 unsigned NumUses = 0;
5835 SmallSetVector<PHINode *, 8> PHIUsers;
5836 SmallSetVector<SelectInst *, 8> SelectUsers;
5837
5838 AllocaSliceRewriter Rewriter(
5839 DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5840 IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5841 bool Promotable = true;
5842 // Check whether we can have tree-structured merge.
5843 if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5844 NumUses += DeletedValues->size() + 1;
5845 for (Value *V : *DeletedValues)
5846 DeadInsts.push_back(Elt: V);
5847 } else {
5848 for (Slice *S : P.splitSliceTails()) {
5849 Promotable &= Rewriter.visit(I: S);
5850 ++NumUses;
5851 }
5852 for (Slice &S : P) {
5853 Promotable &= Rewriter.visit(I: &S);
5854 ++NumUses;
5855 }
5856 }
5857
5858 NumAllocaPartitionUses += NumUses;
5859 MaxUsesPerAllocaPartition.updateMax(V: NumUses);
5860
5861 // Now that we've processed all the slices in the new partition, check if any
5862 // PHIs or Selects would block promotion.
5863 SmallVector<PHINode *, 8> NewSpeculatablePHIs;
5864 SmallVector<PHINode *, 2> NewPHIsWithStoreToRewrite;
5865 for (PHINode *PHI : PHIUsers) {
5866 if (isSafePHIToSpeculate(PN&: *PHI)) {
5867 NewSpeculatablePHIs.push_back(Elt: PHI);
5868 continue;
5869 }
5870 if (getPHIStoreToRewrite(PN&: *PHI, PreserveCFG, DT&: DTU->getDomTree())) {
5871 NewPHIsWithStoreToRewrite.push_back(Elt: PHI);
5872 continue;
5873 }
5874
5875 Promotable = false;
5876 SelectUsers.clear();
5877 break;
5878 }
5879
5880 SmallVector<std::pair<SelectInst *, RewriteableMemOps>, 2>
5881 NewSelectsToRewrite;
5882 NewSelectsToRewrite.reserve(N: SelectUsers.size());
5883 for (SelectInst *Sel : SelectUsers) {
5884 std::optional<RewriteableMemOps> Ops =
5885 isSafeSelectToSpeculateOrPredicate(SI&: *Sel, PreserveCFG);
5886 if (!Ops) {
5887 Promotable = false;
5888 break;
5889 }
5890 NewSelectsToRewrite.emplace_back(Args: std::make_pair(x&: Sel, y&: *Ops));
5891 }
5892
5893 if (Promotable) {
5894 for (Use *U : AS.getDeadUsesIfPromotable()) {
5895 auto *OldInst = dyn_cast<Instruction>(Val: U->get());
5896 Value::dropDroppableUse(U&: *U);
5897 if (OldInst)
5898 if (isInstructionTriviallyDead(I: OldInst))
5899 DeadInsts.push_back(Elt: OldInst);
5900 }
5901 if (NewSpeculatablePHIs.empty() && NewPHIsWithStoreToRewrite.empty() &&
5902 SelectUsers.empty()) {
5903 // Promote the alloca.
5904 PromotableAllocas.insert(X: NewAI);
5905 } else {
5906 // If we have either PHIs or Selects to rewrite, add them to those
5907 // worklists and re-queue the new alloca so that we promote it on the next
5908 // iteration.
5909 SpeculatablePHIs.insert_range(R&: NewSpeculatablePHIs);
5910 PHIsWithStoreToRewrite.insert_range(R&: NewPHIsWithStoreToRewrite);
5911 SelectsToRewrite.reserve(NumEntries: SelectsToRewrite.size() +
5912 NewSelectsToRewrite.size());
5913 for (auto &&KV : llvm::make_range(
5914 x: std::make_move_iterator(i: NewSelectsToRewrite.begin()),
5915 y: std::make_move_iterator(i: NewSelectsToRewrite.end())))
5916 SelectsToRewrite.insert(KV: std::move(KV));
5917 Worklist.insert(X: NewAI);
5918 }
5919 } else {
5920 // Drop any post-promotion work items if promotion didn't happen.
5921 while (PostPromotionWorklist.size() > PPWOldSize)
5922 PostPromotionWorklist.pop_back();
5923
5924 // We couldn't promote and we didn't create a new partition, nothing
5925 // happened.
5926 if (NewAI == &AI)
5927 return {nullptr, 0};
5928
5929 // If we can't promote the alloca, iterate on it to check for new
5930 // refinements exposed by splitting the current alloca. Don't iterate on an
5931 // alloca which didn't actually change and didn't get promoted.
5932 Worklist.insert(X: NewAI);
5933 }
5934
5935 return {NewAI, DL.getTypeSizeInBits(Ty: PartitionTy).getFixedValue()};
5936}
5937
5938// There isn't a shared interface to get the "address" parts out of a
5939// dbg.declare and dbg.assign, so provide some wrappers.
5940bool isKillAddress(const DbgVariableRecord *DVR) {
5941 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5942 return DVR->isKillAddress();
5943 return DVR->isKillLocation();
5944}
5945
5946const DIExpression *getAddressExpression(const DbgVariableRecord *DVR) {
5947 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5948 return DVR->getAddressExpression();
5949 return DVR->getExpression();
5950}
5951
5952/// Create or replace an existing fragment in a DIExpression with \p Frag.
5953/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5954/// operation, add \p BitExtractOffset to the offset part.
5955///
5956/// Returns the new expression, or nullptr if this fails (see details below).
5957///
5958/// This function is similar to DIExpression::createFragmentExpression except
5959/// for 3 important distinctions:
5960/// 1. The new fragment isn't relative to an existing fragment.
5961/// 2. It assumes the computed location is a memory location. This means we
5962/// don't need to perform checks that creating the fragment preserves the
5963/// expression semantics.
5964/// 3. Existing extract_bits are modified independently of fragment changes
5965/// using \p BitExtractOffset. A change to the fragment offset or size
5966/// may affect a bit extract. But a bit extract offset can change
5967/// independently of the fragment dimensions.
5968///
5969/// Returns the new expression, or nullptr if one couldn't be created.
5970/// Ideally this is only used to signal that a bit-extract has become
5971/// zero-sized (and thus the new debug record has no size and can be
5972/// dropped), however, it fails for other reasons too - see the FIXME below.
5973///
5974/// FIXME: To keep the change that introduces this function NFC it bails
5975/// in some situations unecessarily, e.g. when fragment and bit extract
5976/// sizes differ.
5977static DIExpression *createOrReplaceFragment(const DIExpression *Expr,
5978 DIExpression::FragmentInfo Frag,
5979 int64_t BitExtractOffset) {
5980 SmallVector<uint64_t, 8> Ops;
5981 bool HasFragment = false;
5982 bool HasBitExtract = false;
5983
5984 for (auto &Op : Expr->expr_ops()) {
5985 if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5986 HasFragment = true;
5987 continue;
5988 }
5989 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val: Op)) {
5990 HasBitExtract = true;
5991 int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5992 int64_t ExtractSizeInBits = Extract.getSizeInBits();
5993
5994 // DIExpression::createFragmentExpression doesn't know how to handle
5995 // a fragment that is smaller than the extract. Copy the behaviour
5996 // (bail) to avoid non-NFC changes.
5997 // FIXME: Don't do this.
5998 if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5999 return nullptr;
6000
6001 assert(BitExtractOffset <= 0);
6002 int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
6003
6004 // DIExpression::createFragmentExpression doesn't know what to do
6005 // if the new extract starts "outside" the existing one. Copy the
6006 // behaviour (bail) to avoid non-NFC changes.
6007 // FIXME: Don't do this.
6008 if (AdjustedOffset < 0)
6009 return nullptr;
6010
6011 Ops.push_back(Elt: Op.getOp());
6012 Ops.push_back(Elt: std::max<int64_t>(a: 0, b: AdjustedOffset));
6013 Ops.push_back(Elt: ExtractSizeInBits);
6014 continue;
6015 }
6016 Op.appendToVector(V&: Ops);
6017 }
6018
6019 // Unsupported by createFragmentExpression, so don't support it here yet to
6020 // preserve NFC-ness.
6021 if (HasFragment && HasBitExtract)
6022 return nullptr;
6023
6024 if (!HasBitExtract) {
6025 Ops.push_back(Elt: dwarf::DW_OP_LLVM_fragment);
6026 Ops.push_back(Elt: Frag.OffsetInBits);
6027 Ops.push_back(Elt: Frag.SizeInBits);
6028 }
6029 return DIExpression::get(Context&: Expr->getContext(), Elements: Ops);
6030}
6031
6032/// Insert a new DbgRecord.
6033/// \p Orig Original to copy record type, debug loc and variable from, and
6034/// additionally value and value expression for dbg_assign records.
6035/// \p NewAddr Location's new base address.
6036/// \p NewAddrExpr New expression to apply to address.
6037/// \p BeforeInst Insert position.
6038/// \p NewFragment New fragment (absolute, non-relative).
6039/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
6040static void
6041insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr,
6042 DIExpression *NewAddrExpr, Instruction *BeforeInst,
6043 std::optional<DIExpression::FragmentInfo> NewFragment,
6044 int64_t BitExtractAdjustment) {
6045 (void)DIB;
6046
6047 // A dbg_assign puts fragment info in the value expression only. The address
6048 // expression has already been built: NewAddrExpr. A dbg_declare puts the
6049 // new fragment info into NewAddrExpr (as it only has one expression).
6050 DIExpression *NewFragmentExpr =
6051 Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
6052 if (NewFragment)
6053 NewFragmentExpr = createOrReplaceFragment(Expr: NewFragmentExpr, Frag: *NewFragment,
6054 BitExtractOffset: BitExtractAdjustment);
6055 if (!NewFragmentExpr)
6056 return;
6057
6058 if (Orig->isDbgDeclare()) {
6059 DbgVariableRecord *DVR = DbgVariableRecord::createDVRDeclare(
6060 Address: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
6061 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
6062 Here: BeforeInst->getIterator());
6063 return;
6064 }
6065
6066 if (Orig->isDbgValue()) {
6067 DbgVariableRecord *DVR = DbgVariableRecord::createDbgVariableRecord(
6068 Location: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
6069 // Drop debug information if the expression doesn't start with a
6070 // DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
6071 // describes the address of alloca rather than the value inside the alloca.
6072 if (!NewFragmentExpr->startsWithDeref())
6073 DVR->setKillAddress();
6074 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
6075 Here: BeforeInst->getIterator());
6076 return;
6077 }
6078
6079 // Apply a DIAssignID to the store if it doesn't already have it.
6080 if (!NewAddr->hasMetadata(KindID: LLVMContext::MD_DIAssignID)) {
6081 NewAddr->setMetadata(KindID: LLVMContext::MD_DIAssignID,
6082 Node: DIAssignID::getDistinct(Context&: NewAddr->getContext()));
6083 }
6084
6085 DbgVariableRecord *NewAssign = DbgVariableRecord::createLinkedDVRAssign(
6086 LinkedInstr: NewAddr, Val: Orig->getValue(), Variable: Orig->getVariable(), Expression: NewFragmentExpr, Address: NewAddr,
6087 AddressExpression: NewAddrExpr, DI: Orig->getDebugLoc());
6088 LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
6089 (void)NewAssign;
6090}
6091
6092/// Walks the slices of an alloca and form partitions based on them,
6093/// rewriting each of their uses.
6094bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
6095 if (AS.begin() == AS.end())
6096 return false;
6097
6098 unsigned NumPartitions = 0;
6099 bool Changed = false;
6100 const DataLayout &DL = AI.getDataLayout();
6101
6102 // First try to pre-split loads and stores.
6103 Changed |= presplitLoadsAndStores(AI, AS);
6104
6105 // Now that we have identified any pre-splitting opportunities,
6106 // mark loads and stores unsplittable except for the following case.
6107 // We leave a slice splittable if all other slices are disjoint or fully
6108 // included in the slice, such as whole-alloca loads and stores.
6109 // If we fail to split these during pre-splitting, we want to force them
6110 // to be rewritten into a partition.
6111 bool IsSorted = true;
6112
6113 uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
6114 // We can split at the begin and end offsets of each slice, but only if those
6115 // offsets don't lie inside another slice. Because slices are ordered by
6116 // increasing begin offset, and then decreasing end offset, we can consider
6117 // the slices as being split up into sets with the same begin offset where we
6118 // can ignore every slice except the first (the begin offset will already be
6119 // handled as the begin offset of the set, and the end offset we know is not a
6120 // splittable offset as it's inside the first slice of the set).
6121 SparseBitVector<> SplittableOffset;
6122 uint64_t CurBegin = 0, CurEnd = 0;
6123 for (Slice &S : AS) {
6124 // Check if we have a new set of slices
6125 if (S.beginOffset() > CurBegin || S.endOffset() > CurEnd) {
6126 // If the start isn't inside the previous set it's splittable
6127 if (S.beginOffset() >= CurEnd) {
6128 SplittableOffset.set(S.beginOffset());
6129 }
6130 // If the previous end is inside this slice then remove it
6131 if (CurEnd > S.beginOffset() && CurEnd < S.endOffset()) {
6132 SplittableOffset.reset(Idx: CurEnd);
6133 }
6134 CurBegin = S.beginOffset();
6135 // If the end offset isn't inside the previous set it's splittable. We
6136 // also don't update the end offset in that case, as the next set may also
6137 // be inside the previous set.
6138 if (S.endOffset() > CurEnd) {
6139 CurEnd = S.endOffset();
6140 SplittableOffset.set(CurEnd);
6141 }
6142 }
6143 }
6144
6145 for (Slice &S : AS) {
6146 if (!S.isSplittable())
6147 continue;
6148
6149 if ((S.beginOffset() > AllocaSize ||
6150 SplittableOffset.test(Idx: S.beginOffset())) &&
6151 (S.endOffset() > AllocaSize || SplittableOffset.test(Idx: S.endOffset())))
6152 continue;
6153
6154 if (isa<LoadInst>(Val: S.getUse()->getUser()) ||
6155 isa<StoreInst>(Val: S.getUse()->getUser())) {
6156 S.makeUnsplittable();
6157 IsSorted = false;
6158 }
6159 }
6160
6161 if (!IsSorted)
6162 llvm::stable_sort(Range&: AS);
6163
6164 /// Describes the allocas introduced by rewritePartition in order to migrate
6165 /// the debug info.
6166 struct Fragment {
6167 AllocaInst *Alloca;
6168 uint64_t Offset;
6169 uint64_t Size;
6170 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
6171 : Alloca(AI), Offset(O), Size(S) {}
6172 };
6173 SmallVector<Fragment, 4> Fragments;
6174
6175 // Rewrite each partition.
6176 for (auto &P : AS.partitions()) {
6177 auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
6178 if (NewAI) {
6179 Changed = true;
6180 if (NewAI != &AI) {
6181 uint64_t SizeOfByte = 8;
6182 // Don't include any padding.
6183 uint64_t Size = std::min(a: ActiveBits, b: P.size() * SizeOfByte);
6184 Fragments.push_back(
6185 Elt: Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
6186 }
6187 }
6188 ++NumPartitions;
6189 }
6190
6191 NumAllocaPartitions += NumPartitions;
6192 MaxPartitionsPerAlloca.updateMax(V: NumPartitions);
6193
6194 // Migrate debug information from the old alloca to the new alloca(s)
6195 // and the individual partitions.
6196 auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
6197 // Can't overlap with undef memory.
6198 if (isKillAddress(DVR: DbgVariable))
6199 return;
6200
6201 const Value *DbgPtr = DbgVariable->getAddress();
6202 DIExpression::FragmentInfo VarFrag =
6203 DbgVariable->getFragmentOrEntireVariable();
6204 // Get the address expression constant offset if one exists and the ops
6205 // that come after it.
6206 int64_t CurrentExprOffsetInBytes = 0;
6207 SmallVector<uint64_t> PostOffsetOps;
6208 if (!getAddressExpression(DVR: DbgVariable)
6209 ->extractLeadingOffset(OffsetInBytes&: CurrentExprOffsetInBytes, RemainingOps&: PostOffsetOps))
6210 return; // Couldn't interpret this DIExpression - drop the var.
6211
6212 // Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
6213 int64_t ExtractOffsetInBits = 0;
6214 for (auto Op : getAddressExpression(DVR: DbgVariable)->expr_ops()) {
6215 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val&: Op)) {
6216 ExtractOffsetInBits = Extract.getOffsetInBits();
6217 break;
6218 }
6219 }
6220
6221 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
6222 for (auto Fragment : Fragments) {
6223 int64_t OffsetFromLocationInBits;
6224 std::optional<DIExpression::FragmentInfo> NewDbgFragment;
6225 // Find the variable fragment that the new alloca slice covers.
6226 // Drop debug info for this variable fragment if we can't compute an
6227 // intersect between it and the alloca slice.
6228 if (!DIExpression::calculateFragmentIntersect(
6229 DL, SliceStart: &AI, SliceOffsetInBits: Fragment.Offset, SliceSizeInBits: Fragment.Size, DbgPtr,
6230 DbgPtrOffsetInBits: CurrentExprOffsetInBytes * 8, DbgExtractOffsetInBits: ExtractOffsetInBits, VarFrag,
6231 Result&: NewDbgFragment, OffsetFromLocationInBits))
6232 continue; // Do not migrate this fragment to this slice.
6233
6234 // Zero sized fragment indicates there's no intersect between the variable
6235 // fragment and the alloca slice. Skip this slice for this variable
6236 // fragment.
6237 if (NewDbgFragment && !NewDbgFragment->SizeInBits)
6238 continue; // Do not migrate this fragment to this slice.
6239
6240 // No fragment indicates DbgVariable's variable or fragment exactly
6241 // overlaps the slice; copy its fragment (or nullopt if there isn't one).
6242 if (!NewDbgFragment)
6243 NewDbgFragment = DbgVariable->getFragment();
6244
6245 // Reduce the new expression offset by the bit-extract offset since
6246 // we'll be keeping that.
6247 int64_t OffestFromNewAllocaInBits =
6248 OffsetFromLocationInBits - ExtractOffsetInBits;
6249 // We need to adjust an existing bit extract if the offset expression
6250 // can't eat the slack (i.e., if the new offset would be negative).
6251 int64_t BitExtractOffset =
6252 std::min<int64_t>(a: 0, b: OffestFromNewAllocaInBits);
6253 // The magnitude of a negative value indicates the number of bits into
6254 // the existing variable fragment that the memory region begins. The new
6255 // variable fragment already excludes those bits - the new DbgPtr offset
6256 // only needs to be applied if it's positive.
6257 OffestFromNewAllocaInBits =
6258 std::max(a: int64_t(0), b: OffestFromNewAllocaInBits);
6259
6260 // Rebuild the expression:
6261 // {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
6262 // Add NewDbgFragment later, because dbg.assigns don't want it in the
6263 // address expression but the value expression instead.
6264 DIExpression *NewExpr = DIExpression::get(Context&: AI.getContext(), Elements: PostOffsetOps);
6265 if (OffestFromNewAllocaInBits > 0) {
6266 int64_t OffsetInBytes = (OffestFromNewAllocaInBits + 7) / 8;
6267 NewExpr = DIExpression::prepend(Expr: NewExpr, /*flags=*/Flags: 0, Offset: OffsetInBytes);
6268 }
6269
6270 // Remove any existing intrinsics on the new alloca describing
6271 // the variable fragment.
6272 auto RemoveOne = [DbgVariable](auto *OldDII) {
6273 auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
6274 return LHS->getVariable() == RHS->getVariable() &&
6275 LHS->getDebugLoc()->getInlinedAt() ==
6276 RHS->getDebugLoc()->getInlinedAt();
6277 };
6278 if (SameVariableFragment(OldDII, DbgVariable))
6279 OldDII->eraseFromParent();
6280 };
6281 for_each(Range: findDVRDeclares(V: Fragment.Alloca), F: RemoveOne);
6282 for_each(Range: findDVRValues(V: Fragment.Alloca), F: RemoveOne);
6283 insertNewDbgInst(DIB, Orig: DbgVariable, NewAddr: Fragment.Alloca, NewAddrExpr: NewExpr, BeforeInst: &AI,
6284 NewFragment: NewDbgFragment, BitExtractAdjustment: BitExtractOffset);
6285 }
6286 };
6287
6288 // Migrate debug information from the old alloca to the new alloca(s)
6289 // and the individual partitions.
6290 for_each(Range: findDVRDeclares(V: &AI), F: MigrateOne);
6291 for_each(Range: findDVRValues(V: &AI), F: MigrateOne);
6292 for_each(Range: at::getDVRAssignmentMarkers(Inst: &AI), F: MigrateOne);
6293
6294 return Changed;
6295}
6296
6297/// Clobber a use with poison, deleting the used value if it becomes dead.
6298void SROA::clobberUse(Use &U) {
6299 Value *OldV = U;
6300 // Replace the use with an poison value.
6301 U = PoisonValue::get(T: OldV->getType());
6302
6303 // Check for this making an instruction dead. We have to garbage collect
6304 // all the dead instructions to ensure the uses of any alloca end up being
6305 // minimal.
6306 if (Instruction *OldI = dyn_cast<Instruction>(Val: OldV))
6307 if (isInstructionTriviallyDead(I: OldI)) {
6308 DeadInsts.push_back(Elt: OldI);
6309 }
6310}
6311
6312/// A basic LoadAndStorePromoter that does not remove store nodes.
6313class BasicLoadAndStorePromoter : public LoadAndStorePromoter {
6314public:
6315 BasicLoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S,
6316 Type *ZeroType)
6317 : LoadAndStorePromoter(Insts, S), ZeroType(ZeroType) {}
6318 bool shouldDelete(Instruction *I) const override {
6319 return !isa<StoreInst>(Val: I) && !isa<AllocaInst>(Val: I);
6320 }
6321
6322 Value *getValueToUseForAlloca(Instruction *I) const override {
6323 return UndefValue::get(T: ZeroType);
6324 }
6325
6326private:
6327 Type *ZeroType;
6328};
6329
6330bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6331 // Look through each "partition", looking for slices with the same start/end
6332 // that do not overlap with any before them. The slices are sorted by
6333 // increasing beginOffset. We don't use AS.partitions(), as it will use a more
6334 // sophisticated algorithm that takes splittable slices into account.
6335 LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6336 bool AllSameAndValid = true;
6337 Type *PartitionType = nullptr;
6338 SmallVector<Instruction *> Insts;
6339 uint64_t BeginOffset = 0;
6340 uint64_t EndOffset = 0;
6341
6342 auto Flush = [&]() {
6343 if (AllSameAndValid && !Insts.empty()) {
6344 LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6345 << EndOffset << ")\n");
6346 SmallVector<PHINode *, 4> NewPHIs;
6347 SSAUpdater SSA(&NewPHIs);
6348 Insts.push_back(Elt: &AI);
6349 BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6350 Promoter.run(Insts);
6351 }
6352 AllSameAndValid = true;
6353 PartitionType = nullptr;
6354 Insts.clear();
6355 };
6356
6357 for (Slice &S : AS) {
6358 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
6359 if (isAssumeLikeIntrinsic(I: User)) {
6360 LLVM_DEBUG({
6361 dbgs() << "Ignoring slice: ";
6362 AS.print(dbgs(), &S);
6363 });
6364 continue;
6365 }
6366 if (S.beginOffset() >= EndOffset) {
6367 Flush();
6368 BeginOffset = S.beginOffset();
6369 EndOffset = S.endOffset();
6370 } else if (S.beginOffset() != BeginOffset || S.endOffset() != EndOffset) {
6371 if (AllSameAndValid) {
6372 LLVM_DEBUG({
6373 dbgs() << "Slice does not match range [" << BeginOffset << ", "
6374 << EndOffset << ")";
6375 AS.print(dbgs(), &S);
6376 });
6377 AllSameAndValid = false;
6378 }
6379 EndOffset = std::max(a: EndOffset, b: S.endOffset());
6380 continue;
6381 }
6382
6383 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
6384 Type *UserTy = LI->getType();
6385 // LoadAndStorePromoter requires all the types to be the same.
6386 if (!LI->isSimple() || (PartitionType && UserTy != PartitionType))
6387 AllSameAndValid = false;
6388 PartitionType = UserTy;
6389 Insts.push_back(Elt: User);
6390 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
6391 Type *UserTy = SI->getValueOperand()->getType();
6392 if (!SI->isSimple() || (PartitionType && UserTy != PartitionType))
6393 AllSameAndValid = false;
6394 PartitionType = UserTy;
6395 Insts.push_back(Elt: User);
6396 } else {
6397 AllSameAndValid = false;
6398 }
6399 }
6400
6401 Flush();
6402 return true;
6403}
6404
6405/// Analyze an alloca for SROA.
6406///
6407/// This analyzes the alloca to ensure we can reason about it, builds
6408/// the slices of the alloca, and then hands it off to be split and
6409/// rewritten as needed.
6410std::pair<bool /*Changed*/, bool /*CFGChanged*/>
6411SROA::runOnAlloca(AllocaInst &AI) {
6412 bool Changed = false;
6413 bool CFGChanged = false;
6414
6415 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6416 ++NumAllocasAnalyzed;
6417
6418 // Special case dead allocas, as they're trivial.
6419 if (AI.use_empty()) {
6420 DeadInsts.push_back(Elt: &AI);
6421 Changed = true;
6422 return {Changed, CFGChanged};
6423 }
6424 const DataLayout &DL = AI.getDataLayout();
6425
6426 // Skip alloca forms that this analysis can't handle.
6427 std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6428 if (AI.isArrayAllocation() || !Size || Size->isScalable() || Size->isZero())
6429 return {Changed, CFGChanged};
6430
6431 // First, split any FCA loads and stores touching this alloca to promote
6432 // better splitting and promotion opportunities.
6433 IRBuilderTy IRB(&AI);
6434 IRB.disableGEPCanonicalization();
6435 AggLoadStoreRewriter AggRewriter(DL, IRB);
6436 Changed |= AggRewriter.rewrite(I&: AI);
6437
6438 // Build the slices using a recursive instruction-visiting builder.
6439 AllocaSlices AS(DL, AI);
6440 LLVM_DEBUG(AS.print(dbgs()));
6441 if (AS.isEscaped())
6442 return {Changed, CFGChanged};
6443
6444 if (AS.isEscapedReadOnly()) {
6445 Changed |= propagateStoredValuesToLoads(AI, AS);
6446 return {Changed, CFGChanged};
6447 }
6448
6449 // Delete all the dead users of this alloca before splitting and rewriting it.
6450 for (Instruction *DeadUser : AS.getDeadUsers()) {
6451 // Free up everything used by this instruction.
6452 for (Use &DeadOp : DeadUser->operands())
6453 clobberUse(U&: DeadOp);
6454
6455 // Now replace the uses of this instruction.
6456 DeadUser->replaceAllUsesWith(V: PoisonValue::get(T: DeadUser->getType()));
6457
6458 // And mark it for deletion.
6459 DeadInsts.push_back(Elt: DeadUser);
6460 Changed = true;
6461 }
6462 for (Use *DeadOp : AS.getDeadOperands()) {
6463 clobberUse(U&: *DeadOp);
6464 Changed = true;
6465 }
6466
6467 // No slices to split. Leave the dead alloca for a later pass to clean up.
6468 if (AS.begin() == AS.end())
6469 return {Changed, CFGChanged};
6470
6471 Changed |= splitAlloca(AI, AS);
6472
6473 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6474 while (!SpeculatablePHIs.empty())
6475 speculatePHINodeLoads(IRB, PN&: *SpeculatablePHIs.pop_back_val());
6476
6477 LLVM_DEBUG(dbgs() << " Rewriting stores through PHIs\n");
6478 auto RemainingPHIsWithStoreToRewrite = PHIsWithStoreToRewrite.takeVector();
6479 while (!RemainingPHIsWithStoreToRewrite.empty()) {
6480 PHINode *PN = RemainingPHIsWithStoreToRewrite.pop_back_val();
6481 auto *SI = cast<StoreInst>(Val: PN->user_back());
6482 CFGChanged |= rewritePHINodeStore(PN&: *PN, SI&: *SI, DTU&: *DTU, Worklist);
6483 }
6484
6485 LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6486 auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6487 while (!RemainingSelectsToRewrite.empty()) {
6488 const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6489 CFGChanged |=
6490 rewriteSelectInstMemOps(SI&: *K, Ops: V, IRB, DTU: PreserveCFG ? nullptr : DTU);
6491 }
6492
6493 return {Changed, CFGChanged};
6494}
6495
6496/// Delete the dead instructions accumulated in this run.
6497///
6498/// Recursively deletes the dead instructions we've accumulated. This is done
6499/// at the very end to maximize locality of the recursive delete and to
6500/// minimize the problems of invalidated instruction pointers as such pointers
6501/// are used heavily in the intermediate stages of the algorithm.
6502///
6503/// We also record the alloca instructions deleted here so that they aren't
6504/// subsequently handed to mem2reg to promote.
6505bool SROA::deleteDeadInstructions(
6506 AllocaInst *AI, SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6507 bool Changed = false;
6508 while (!DeadInsts.empty()) {
6509 Instruction *I = dyn_cast_or_null<Instruction>(Val: DeadInsts.pop_back_val());
6510 if (!I)
6511 continue;
6512 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6513
6514 // If the instruction is an alloca, find the possible dbg.declare connected
6515 // to it, and remove it too. We must do this before calling RAUW or we will
6516 // not be able to find it.
6517 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: I)) {
6518 DeletedAllocas.insert(Ptr: AI);
6519 for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6520 OldDII->eraseFromParent();
6521 }
6522
6523 at::deleteAssignmentMarkers(Inst: I);
6524 I->replaceAllUsesWith(V: UndefValue::get(T: I->getType()));
6525
6526 for (Use &Operand : I->operands())
6527 if (Instruction *U = dyn_cast<Instruction>(Val&: Operand)) {
6528 // Zero out the operand and see if it becomes trivially dead.
6529 Operand = nullptr;
6530 if (isInstructionTriviallyDead(I: U))
6531 DeadInsts.push_back(Elt: U);
6532 }
6533
6534 ++NumDeleted;
6535 I->eraseFromParent();
6536 Changed = true;
6537 }
6538
6539 if (!DeletedAllocas.contains(Ptr: AI) && AI->hasNUndroppableUses(N: 0)) {
6540 DeletedAllocas.insert(Ptr: AI);
6541 AI->dropDroppableUses();
6542 for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6543 OldDII->eraseFromParent();
6544 ++NumDeleted;
6545 AI->eraseFromParent();
6546 Changed = true;
6547 }
6548
6549 return Changed;
6550}
6551/// Promote the allocas, using the best available technique.
6552///
6553/// This attempts to promote whatever allocas have been identified as viable in
6554/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6555/// This function returns whether any promotion occurred.
6556bool SROA::promoteAllocas() {
6557 if (PromotableAllocas.empty())
6558 return false;
6559
6560 if (getScalarOptions(C: *C).sroa_skip_mem2reg) {
6561 LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6562 } else {
6563 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6564 NumPromoted += PromotableAllocas.size();
6565 PromoteMemToReg(Allocas: PromotableAllocas.getArrayRef(), DT&: DTU->getDomTree(), AC);
6566 }
6567
6568 PromotableAllocas.clear();
6569 return true;
6570}
6571
6572std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
6573 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6574
6575 const DataLayout &DL = F.getDataLayout();
6576 BasicBlock &EntryBB = F.getEntryBlock();
6577 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(x: EntryBB.end());
6578 I != E; ++I) {
6579 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val&: I)) {
6580 std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6581 if (Size && Size->isScalable() && isAllocaPromotable(AI))
6582 PromotableAllocas.insert(X: AI);
6583 else
6584 Worklist.insert(X: AI);
6585 }
6586 }
6587
6588 bool Changed = false;
6589 bool CFGChanged = false;
6590 // A set of deleted alloca instruction pointers which should be removed from
6591 // the list of promotable allocas.
6592 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
6593
6594 do {
6595 while (!Worklist.empty()) {
6596 AllocaInst *AI = Worklist.pop_back_val();
6597 auto [IterationChanged, IterationCFGChanged] = runOnAlloca(AI&: *AI);
6598 Changed |= IterationChanged;
6599 CFGChanged |= IterationCFGChanged;
6600
6601 Changed |= deleteDeadInstructions(AI, DeletedAllocas);
6602
6603 // Remove the deleted allocas from various lists so that we don't try to
6604 // continue processing them.
6605 if (!DeletedAllocas.empty()) {
6606 Worklist.set_subtract(DeletedAllocas);
6607 PostPromotionWorklist.set_subtract(DeletedAllocas);
6608 PromotableAllocas.set_subtract(DeletedAllocas);
6609 DeletedAllocas.clear();
6610 }
6611 }
6612
6613 Changed |= promoteAllocas();
6614
6615 Worklist = PostPromotionWorklist;
6616 PostPromotionWorklist.clear();
6617 } while (!Worklist.empty());
6618
6619 assert((!CFGChanged || Changed) && "Can not only modify the CFG.");
6620 assert((!CFGChanged || !PreserveCFG) &&
6621 "Should not have modified the CFG when told to preserve it.");
6622
6623 if (Changed && isAssignmentTrackingEnabled(M: *F.getParent())) {
6624 for (auto &BB : F) {
6625 RemoveRedundantDbgInstrs(BB: &BB);
6626 }
6627 }
6628
6629 return {Changed, CFGChanged};
6630}
6631
6632PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
6633 DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
6634 AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
6635 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6636 auto [Changed, CFGChanged] =
6637 SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6638 if (!Changed)
6639 return PreservedAnalyses::all();
6640 PreservedAnalyses PA;
6641 if (!CFGChanged)
6642 PA.preserveSet<CFGAnalyses>();
6643 PA.preserve<DominatorTreeAnalysis>();
6644 return PA;
6645}
6646
6647void SROAPass::printPipeline(
6648 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6649 static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
6650 OS, MapClassName2PassName);
6651 OS << '<'
6652 << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6653 : "modify-cfg");
6654 if (Options.AggregateToVector)
6655 OS << ";aggregate-to-vector";
6656 OS << '>';
6657}
6658
6659SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
6660
6661namespace {
6662
6663/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6664class SROALegacyPass : public FunctionPass {
6665 SROAOptions Options;
6666
6667public:
6668 static char ID;
6669
6670 SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
6671 : FunctionPass(ID), Options(Options) {
6672 initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
6673 }
6674
6675 bool runOnFunction(Function &F) override {
6676 if (skipFunction(F))
6677 return false;
6678
6679 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6680 AssumptionCache &AC =
6681 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6682 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6683 auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6684 return Changed;
6685 }
6686
6687 void getAnalysisUsage(AnalysisUsage &AU) const override {
6688 AU.addRequired<AssumptionCacheTracker>();
6689 AU.addRequired<DominatorTreeWrapperPass>();
6690 AU.addPreserved<GlobalsAAWrapperPass>();
6691 AU.addPreserved<DominatorTreeWrapperPass>();
6692 }
6693
6694 StringRef getPassName() const override { return "SROA"; }
6695};
6696
6697} // end anonymous namespace
6698
6699char SROALegacyPass::ID = 0;
6700
6701FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
6702 return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
6703 : SROAOptions::ModifyCFG,
6704 AggregateToVector));
6705}
6706
6707INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6708 "Scalar Replacement Of Aggregates", false, false)
6709INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6710INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
6711INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
6712 false, false)
6713