1//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This transformation implements the well known scalar replacement of
10/// aggregates transformation. It tries to identify promotable elements of an
11/// aggregate alloca, and promote them to registers. It will also try to
12/// convert uses of an element (or set of elements) of an alloca into a vector
13/// or bitfield-style integer scalar if appropriate.
14///
15/// It works to do this with minimal slicing of the alloca so that regions
16/// which are merely transferred in and out of external memory remain unchanged
17/// and are not decomposed to scalar code.
18///
19/// Because this also performs alloca promotion, it can be thought of as also
20/// serving the purpose of SSA formation. The algorithm iterates on the
21/// function until all opportunities for promotion have been realized.
22///
23//===----------------------------------------------------------------------===//
24
25#include "llvm/Transforms/Scalar/SROA.h"
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/MapVector.h"
30#include "llvm/ADT/PointerIntPair.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SetVector.h"
33#include "llvm/ADT/SmallPtrSet.h"
34#include "llvm/ADT/SmallVector.h"
35#include "llvm/ADT/SparseBitVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/ADT/Twine.h"
39#include "llvm/ADT/iterator.h"
40#include "llvm/ADT/iterator_range.h"
41#include "llvm/Analysis/AssumptionCache.h"
42#include "llvm/Analysis/DomTreeUpdater.h"
43#include "llvm/Analysis/GlobalsModRef.h"
44#include "llvm/Analysis/Loads.h"
45#include "llvm/Analysis/PtrUseVisitor.h"
46#include "llvm/Analysis/ValueTracking.h"
47#include "llvm/Analysis/VectorUtils.h"
48#include "llvm/IR/BasicBlock.h"
49#include "llvm/IR/Constant.h"
50#include "llvm/IR/ConstantFolder.h"
51#include "llvm/IR/Constants.h"
52#include "llvm/IR/DIBuilder.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfo.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalAlias.h"
60#include "llvm/IR/IRBuilder.h"
61#include "llvm/IR/InstVisitor.h"
62#include "llvm/IR/Instruction.h"
63#include "llvm/IR/Instructions.h"
64#include "llvm/IR/IntrinsicInst.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/Module.h"
68#include "llvm/IR/Operator.h"
69#include "llvm/IR/PassManager.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/InitializePasses.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/Debug.h"
81#include "llvm/Support/ErrorHandling.h"
82#include "llvm/Support/raw_ostream.h"
83#include "llvm/Transforms/Scalar.h"
84#include "llvm/Transforms/Utils/BasicBlockUtils.h"
85#include "llvm/Transforms/Utils/Local.h"
86#include "llvm/Transforms/Utils/PromoteMemToReg.h"
87#include "llvm/Transforms/Utils/SSAUpdater.h"
88#include <algorithm>
89#include <cassert>
90#include <cstddef>
91#include <cstdint>
92#include <cstring>
93#include <iterator>
94#include <string>
95#include <tuple>
96#include <utility>
97#include <variant>
98#include <vector>
99
100using namespace llvm;
101
102#define DEBUG_TYPE "sroa"
103
104STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112STATISTIC(NumLoadsPredicated,
113 "Number of loads rewritten into predicated loads to allow promotion");
114STATISTIC(
115 NumStoresPredicated,
116 "Number of stores rewritten into predicated loads to allow promotion");
117STATISTIC(NumDeleted, "Number of instructions deleted");
118STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120namespace llvm {
121/// Disable running mem2reg during SROA in order to test or debug SROA.
122static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(Val: false),
123 cl::Hidden);
124} // namespace llvm
125
126namespace {
127
128class AllocaSliceRewriter;
129class AllocaSlices;
130class Partition;
131
132class SelectHandSpeculativity {
133 unsigned char Storage = 0; // None are speculatable by default.
134 using TrueVal = Bitfield::Element<bool, 0, 1>; // Low 0'th bit.
135 using FalseVal = Bitfield::Element<bool, 1, 1>; // Low 1'th bit.
136public:
137 SelectHandSpeculativity() = default;
138 SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
139 bool isSpeculatable(bool isTrueVal) const;
140 bool areAllSpeculatable() const;
141 bool areAnySpeculatable() const;
142 bool areNoneSpeculatable() const;
143 // For interop as int half of PointerIntPair.
144 explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
145 explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
146};
147static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
148
149using PossiblySpeculatableLoad =
150 PointerIntPair<LoadInst *, 2, SelectHandSpeculativity>;
151using UnspeculatableStore = StoreInst *;
152using RewriteableMemOp =
153 std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
154using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
155
156/// An optimization pass providing Scalar Replacement of Aggregates.
157///
158/// This pass takes allocations which can be completely analyzed (that is, they
159/// don't escape) and tries to turn them into scalar SSA values. There are
160/// a few steps to this process.
161///
162/// 1) It takes allocations of aggregates and analyzes the ways in which they
163/// are used to try to split them into smaller allocations, ideally of
164/// a single scalar data type. It will split up memcpy and memset accesses
165/// as necessary and try to isolate individual scalar accesses.
166/// 2) It will transform accesses into forms which are suitable for SSA value
167/// promotion. This can be replacing a memset with a scalar store of an
168/// integer value, or it can involve speculating operations on a PHI or
169/// select to be a PHI or select of the results.
170/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
171/// onto insert and extract operations on a vector value, and convert them to
172/// this form. By doing so, it will enable promotion of vector aggregates to
173/// SSA vector values.
174class SROA {
175 LLVMContext *const C;
176 DomTreeUpdater *const DTU;
177 AssumptionCache *const AC;
178 const bool PreserveCFG;
179 const bool AggregateToVector;
180
181 /// Worklist of alloca instructions to simplify.
182 ///
183 /// Each alloca in the function is added to this. Each new alloca formed gets
184 /// added to it as well to recursively simplify unless that alloca can be
185 /// directly promoted. Finally, each time we rewrite a use of an alloca other
186 /// the one being actively rewritten, we add it back onto the list if not
187 /// already present to ensure it is re-visited.
188 SmallSetVector<AllocaInst *, 16> Worklist;
189
190 /// A collection of instructions to delete.
191 /// We try to batch deletions to simplify code and make things a bit more
192 /// efficient. We also make sure there is no dangling pointers.
193 SmallVector<WeakVH, 8> DeadInsts;
194
195 /// Post-promotion worklist.
196 ///
197 /// Sometimes we discover an alloca which has a high probability of becoming
198 /// viable for SROA after a round of promotion takes place. In those cases,
199 /// the alloca is enqueued here for re-processing.
200 ///
201 /// Note that we have to be very careful to clear allocas out of this list in
202 /// the event they are deleted.
203 SmallSetVector<AllocaInst *, 16> PostPromotionWorklist;
204
205 /// A collection of alloca instructions we can directly promote.
206 SetVector<AllocaInst *, SmallVector<AllocaInst *>,
207 SmallPtrSet<AllocaInst *, 16>, 16>
208 PromotableAllocas;
209
210 /// A worklist of PHIs to speculate prior to promoting allocas.
211 ///
212 /// All of these PHIs have been checked for the safety of speculation and by
213 /// being speculated will allow promoting allocas currently in the promotable
214 /// queue.
215 SmallSetVector<PHINode *, 8> SpeculatablePHIs;
216
217 /// A worklist of select instructions to rewrite prior to promoting
218 /// allocas.
219 SmallMapVector<SelectInst *, RewriteableMemOps, 8> SelectsToRewrite;
220
221 /// Select instructions that use an alloca and are subsequently loaded can be
222 /// rewritten to load both input pointers and then select between the result,
223 /// allowing the load of the alloca to be promoted.
224 /// From this:
225 /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other
226 /// %V = load <type>, ptr %P2
227 /// to:
228 /// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
229 /// %V2 = load <type>, ptr %Other
230 /// %V = select i1 %cond, <type> %V1, <type> %V2
231 ///
232 /// We can do this to a select if its only uses are loads
233 /// and if either the operand to the select can be loaded unconditionally,
234 /// or if we are allowed to perform CFG modifications.
235 static std::optional<RewriteableMemOps>
236 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
237
238public:
239 SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
240 SROAOptions Options)
241 : C(C), DTU(DTU), AC(AC),
242 PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
243 AggregateToVector(Options.AggregateToVector) {}
244
245 /// Main run method used by both the SROAPass and by the legacy pass.
246 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
247
248private:
249 friend class AllocaSliceRewriter;
250
251 bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
252 std::pair<AllocaInst *, uint64_t>
253 rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
254 bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
255 bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
256 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runOnAlloca(AllocaInst &AI);
257 void clobberUse(Use &U);
258 bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
259 bool promoteAllocas();
260};
261
262} // end anonymous namespace
263
264/// Calculate the fragment of a variable to use when slicing a store
265/// based on the slice dimensions, existing fragment, and base storage
266/// fragment.
267/// Results:
268/// UseFrag - Use Target as the new fragment.
269/// UseNoFrag - The new slice already covers the whole variable.
270/// Skip - The new alloca slice doesn't include this variable.
271/// FIXME: Can we use calculateFragmentIntersect instead?
272namespace {
273enum FragCalcResult { UseFrag, UseNoFrag, Skip };
274}
275static FragCalcResult
276calculateFragment(DILocalVariable *Variable,
277 uint64_t NewStorageSliceOffsetInBits,
278 uint64_t NewStorageSliceSizeInBits,
279 std::optional<DIExpression::FragmentInfo> StorageFragment,
280 std::optional<DIExpression::FragmentInfo> CurrentFragment,
281 DIExpression::FragmentInfo &Target) {
282 // If the base storage describes part of the variable apply the offset and
283 // the size constraint.
284 if (StorageFragment) {
285 Target.SizeInBits =
286 std::min(a: NewStorageSliceSizeInBits, b: StorageFragment->SizeInBits);
287 Target.OffsetInBits =
288 NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits;
289 } else {
290 Target.SizeInBits = NewStorageSliceSizeInBits;
291 Target.OffsetInBits = NewStorageSliceOffsetInBits;
292 }
293
294 // If this slice extracts the entirety of an independent variable from a
295 // larger alloca, do not produce a fragment expression, as the variable is
296 // not fragmented.
297 if (!CurrentFragment) {
298 if (auto Size = Variable->getSizeInBits()) {
299 // Treat the current fragment as covering the whole variable.
300 CurrentFragment = DIExpression::FragmentInfo(*Size, 0);
301 if (Target == CurrentFragment)
302 return UseNoFrag;
303 }
304 }
305
306 // No additional work to do if there isn't a fragment already, or there is
307 // but it already exactly describes the new assignment.
308 if (!CurrentFragment || *CurrentFragment == Target)
309 return UseFrag;
310
311 // Reject the target fragment if it doesn't fit wholly within the current
312 // fragment. TODO: We could instead chop up the target to fit in the case of
313 // a partial overlap.
314 if (Target.startInBits() < CurrentFragment->startInBits() ||
315 Target.endInBits() > CurrentFragment->endInBits())
316 return Skip;
317
318 // Target fits within the current fragment, return it.
319 return UseFrag;
320}
321
322static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
323 return DebugVariable(DVR->getVariable(), std::nullopt,
324 DVR->getDebugLoc().getInlinedAt());
325}
326
327/// Find linked dbg.assign and generate a new one with the correct
328/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
329/// value component is copied from the old dbg.assign to the new.
330/// \param OldAlloca Alloca for the variable before splitting.
331/// \param IsSplit True if the store (not necessarily alloca)
332/// is being split.
333/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
334/// \param SliceSizeInBits New number of bits being written to.
335/// \param OldInst Instruction that is being split.
336/// \param Inst New instruction performing this part of the
337/// split store.
338/// \param Dest Store destination.
339/// \param Value Stored value.
340/// \param DL Datalayout.
341static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
342 uint64_t OldAllocaOffsetInBits,
343 uint64_t SliceSizeInBits, Instruction *OldInst,
344 Instruction *Inst, Value *Dest, Value *Value,
345 const DataLayout &DL) {
346 // If we want allocas to be migrated using this helper then we need to ensure
347 // that the BaseFragments map code still works. A simple solution would be
348 // to choose to always clone alloca dbg_assigns (rather than sometimes
349 // "stealing" them).
350 assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
351
352 auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(Inst: OldInst);
353 // Nothing to do if OldInst has no linked dbg.assign intrinsics.
354 if (DVRAssignMarkerRange.empty())
355 return;
356
357 LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
358 LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
359 LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
360 LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
361 << "\n");
362 LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
363 LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
364 LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
365 LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
366 if (Value)
367 LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
368
369 /// Map of aggregate variables to their fragment associated with OldAlloca.
370 DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>>
371 BaseFragments;
372 for (auto *DVR : at::getDVRAssignmentMarkers(Inst: OldAlloca))
373 BaseFragments[getAggregateVariable(DVR)] =
374 DVR->getExpression()->getFragmentInfo();
375
376 // The new inst needs a DIAssignID unique metadata tag (if OldInst has
377 // one). It shouldn't already have one: assert this assumption.
378 assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
379 DIAssignID *NewID = nullptr;
380 auto &Ctx = Inst->getContext();
381 DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
382 assert(OldAlloca->isStaticAlloca());
383
384 auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
385 LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
386 << "\n");
387 auto *Expr = DbgAssign->getExpression();
388 bool SetKillLocation = false;
389
390 if (IsSplit) {
391 std::optional<DIExpression::FragmentInfo> BaseFragment;
392 {
393 auto R = BaseFragments.find(Val: getAggregateVariable(DVR: DbgAssign));
394 if (R == BaseFragments.end())
395 return;
396 BaseFragment = R->second;
397 }
398 std::optional<DIExpression::FragmentInfo> CurrentFragment =
399 Expr->getFragmentInfo();
400 DIExpression::FragmentInfo NewFragment;
401 FragCalcResult Result = calculateFragment(
402 Variable: DbgAssign->getVariable(), NewStorageSliceOffsetInBits: OldAllocaOffsetInBits, NewStorageSliceSizeInBits: SliceSizeInBits,
403 StorageFragment: BaseFragment, CurrentFragment, Target&: NewFragment);
404
405 if (Result == Skip)
406 return;
407 if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
408 if (CurrentFragment) {
409 // Rewrite NewFragment to be relative to the existing one (this is
410 // what createFragmentExpression wants). CalculateFragment has
411 // already resolved the size for us. FIXME: Should it return the
412 // relative fragment too?
413 NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits;
414 }
415 // Add the new fragment info to the existing expression if possible.
416 if (auto E = DIExpression::createFragmentExpression(
417 Expr, OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits)) {
418 Expr = *E;
419 } else {
420 // Otherwise, add the new fragment info to an empty expression and
421 // discard the value component of this dbg.assign as the value cannot
422 // be computed with the new fragment.
423 Expr = *DIExpression::createFragmentExpression(
424 Expr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
425 OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits);
426 SetKillLocation = true;
427 }
428 }
429 }
430
431 // If we haven't created a DIAssignID ID do that now and attach it to Inst.
432 if (!NewID) {
433 NewID = DIAssignID::getDistinct(Context&: Ctx);
434 Inst->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: NewID);
435 }
436
437 DbgVariableRecord *NewAssign;
438 if (IsSplit) {
439 ::Value *NewValue = Value ? Value : DbgAssign->getValue();
440 NewAssign = cast<DbgVariableRecord>(Val: DIB.insertDbgAssign(
441 LinkedInstr: Inst, Val: NewValue, SrcVar: DbgAssign->getVariable(), ValExpr: Expr, Addr: Dest,
442 AddrExpr: DIExpression::get(Context&: Expr->getContext(), Elements: {}), DL: DbgAssign->getDebugLoc()));
443 } else {
444 // The store is not split, simply steal the existing dbg_assign.
445 NewAssign = DbgAssign;
446 NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
447 NewAssign->setAddress(Dest);
448 if (Value)
449 NewAssign->replaceVariableLocationOp(OpIdx: 0u, NewValue: Value);
450 assert(Expr == NewAssign->getExpression());
451 }
452
453 // If we've updated the value but the original dbg.assign has an arglist
454 // then kill it now - we can't use the requested new value.
455 // We can't replace the DIArgList with the new value as it'd leave
456 // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
457 // an arglist). And we can't keep the DIArgList in case the linked store
458 // is being split - in which case the DIArgList + expression may no longer
459 // be computing the correct value.
460 // This should be a very rare situation as it requires the value being
461 // stored to differ from the dbg.assign (i.e., the value has been
462 // represented differently in the debug intrinsic for some reason).
463 SetKillLocation |=
464 Value && (DbgAssign->hasArgList() ||
465 !DbgAssign->getExpression()->isSingleLocationExpression());
466 if (SetKillLocation)
467 NewAssign->setKillLocation();
468
469 // We could use more precision here at the cost of some additional (code)
470 // complexity - if the original dbg.assign was adjacent to its store, we
471 // could position this new dbg.assign adjacent to its store rather than the
472 // old dbg.assgn. That would result in interleaved dbg.assigns rather than
473 // what we get now:
474 // split store !1
475 // split store !2
476 // dbg.assign !1
477 // dbg.assign !2
478 // This (current behaviour) results results in debug assignments being
479 // noted as slightly offset (in code) from the store. In practice this
480 // should have little effect on the debugging experience due to the fact
481 // that all the split stores should get the same line number.
482 if (NewAssign != DbgAssign) {
483 NewAssign->moveBefore(MoveBefore: DbgAssign->getIterator());
484 NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
485 }
486 LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
487 };
488
489 for_each(Range&: DVRAssignMarkerRange, F: MigrateDbgAssign);
490}
491
492namespace {
493
494/// A custom IRBuilder inserter which prefixes all names, but only in
495/// Assert builds.
496class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
497 std::string Prefix;
498
499 Twine getNameWithPrefix(const Twine &Name) const {
500 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
501 }
502
503public:
504 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
505
506 void InsertHelper(Instruction *I, const Twine &Name,
507 BasicBlock::iterator InsertPt) const override {
508 IRBuilderDefaultInserter::InsertHelper(I, Name: getNameWithPrefix(Name),
509 InsertPt);
510 }
511};
512
513/// Provide a type for IRBuilder that drops names in release builds.
514using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
515
516/// A used slice of an alloca.
517///
518/// This structure represents a slice of an alloca used by some instruction. It
519/// stores both the begin and end offsets of this use, a pointer to the use
520/// itself, and a flag indicating whether we can classify the use as splittable
521/// or not when forming partitions of the alloca.
522class Slice {
523 /// The beginning offset of the range.
524 uint64_t BeginOffset = 0;
525
526 /// The ending offset, not included in the range.
527 uint64_t EndOffset = 0;
528
529 /// Storage for both the use of this slice and whether it can be
530 /// split.
531 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
532
533public:
534 Slice() = default;
535
536 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
537 : BeginOffset(BeginOffset), EndOffset(EndOffset),
538 UseAndIsSplittable(U, IsSplittable) {}
539
540 uint64_t beginOffset() const { return BeginOffset; }
541 uint64_t endOffset() const { return EndOffset; }
542
543 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
544 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
545
546 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
547
548 bool isDead() const { return getUse() == nullptr; }
549 void kill() { UseAndIsSplittable.setPointer(nullptr); }
550
551 /// Support for ordering ranges.
552 ///
553 /// This provides an ordering over ranges such that start offsets are
554 /// always increasing, and within equal start offsets, the end offsets are
555 /// decreasing. Thus the spanning range comes first in a cluster with the
556 /// same start position.
557 bool operator<(const Slice &RHS) const {
558 if (beginOffset() < RHS.beginOffset())
559 return true;
560 if (beginOffset() > RHS.beginOffset())
561 return false;
562 if (isSplittable() != RHS.isSplittable())
563 return !isSplittable();
564 if (endOffset() > RHS.endOffset())
565 return true;
566 return false;
567 }
568
569 /// Support comparison with a single offset to allow binary searches.
570 [[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
571 return LHS.beginOffset() < RHSOffset;
572 }
573 [[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
574 return LHSOffset < RHS.beginOffset();
575 }
576
577 bool operator==(const Slice &RHS) const {
578 return isSplittable() == RHS.isSplittable() &&
579 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
580 }
581 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
582};
583
584/// Representation of the alloca slices.
585///
586/// This class represents the slices of an alloca which are formed by its
587/// various uses. If a pointer escapes, we can't fully build a representation
588/// for the slices used and we reflect that in this structure. The uses are
589/// stored, sorted by increasing beginning offset and with unsplittable slices
590/// starting at a particular offset before splittable slices.
591class AllocaSlices {
592public:
593 /// Construct the slices of a particular alloca.
594 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
595
596 /// Test whether a pointer to the allocation escapes our analysis.
597 ///
598 /// If this is true, the slices are never fully built and should be
599 /// ignored.
600 bool isEscaped() const { return PointerEscapingInstr; }
601 bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
602
603 /// Support for iterating over the slices.
604 /// @{
605 using iterator = SmallVectorImpl<Slice>::iterator;
606 using range = iterator_range<iterator>;
607
608 iterator begin() { return Slices.begin(); }
609 iterator end() { return Slices.end(); }
610
611 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
612 using const_range = iterator_range<const_iterator>;
613
614 const_iterator begin() const { return Slices.begin(); }
615 const_iterator end() const { return Slices.end(); }
616 /// @}
617
618 /// Erase a range of slices.
619 void erase(iterator Start, iterator Stop) { Slices.erase(CS: Start, CE: Stop); }
620
621 /// Insert new slices for this alloca.
622 ///
623 /// This moves the slices into the alloca's slices collection, and re-sorts
624 /// everything so that the usual ordering properties of the alloca's slices
625 /// hold.
626 void insert(ArrayRef<Slice> NewSlices) {
627 int OldSize = Slices.size();
628 Slices.append(in_start: NewSlices.begin(), in_end: NewSlices.end());
629 auto SliceI = Slices.begin() + OldSize;
630 std::stable_sort(first: SliceI, last: Slices.end());
631 std::inplace_merge(first: Slices.begin(), middle: SliceI, last: Slices.end());
632 }
633
634 // Forward declare the iterator and range accessor for walking the
635 // partitions.
636 class partition_iterator;
637 iterator_range<partition_iterator> partitions();
638
639 /// Access the dead users for this alloca.
640 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
641
642 /// Access Uses that should be dropped if the alloca is promotable.
643 ArrayRef<Use *> getDeadUsesIfPromotable() const {
644 return DeadUseIfPromotable;
645 }
646
647 /// Access the dead operands referring to this alloca.
648 ///
649 /// These are operands which have cannot actually be used to refer to the
650 /// alloca as they are outside its range and the user doesn't correct for
651 /// that. These mostly consist of PHI node inputs and the like which we just
652 /// need to replace with undef.
653 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
654
655#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
656 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
657 void printSlice(raw_ostream &OS, const_iterator I,
658 StringRef Indent = " ") const;
659 void printUse(raw_ostream &OS, const_iterator I,
660 StringRef Indent = " ") const;
661 void print(raw_ostream &OS) const;
662 void dump(const_iterator I) const;
663 void dump() const;
664#endif
665
666private:
667 template <typename DerivedT, typename RetT = void> class BuilderBase;
668 class SliceBuilder;
669
670 friend class AllocaSlices::SliceBuilder;
671
672#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
673 /// Handle to alloca instruction to simplify method interfaces.
674 AllocaInst &AI;
675#endif
676
677 /// The instruction responsible for this alloca not having a known set
678 /// of slices.
679 ///
680 /// When an instruction (potentially) escapes the pointer to the alloca, we
681 /// store a pointer to that here and abort trying to form slices of the
682 /// alloca. This will be null if the alloca slices are analyzed successfully.
683 Instruction *PointerEscapingInstr;
684 Instruction *PointerEscapingInstrReadOnly;
685
686 /// The slices of the alloca.
687 ///
688 /// We store a vector of the slices formed by uses of the alloca here. This
689 /// vector is sorted by increasing begin offset, and then the unsplittable
690 /// slices before the splittable ones. See the Slice inner class for more
691 /// details.
692 SmallVector<Slice, 8> Slices;
693
694 /// Instructions which will become dead if we rewrite the alloca.
695 ///
696 /// Note that these are not separated by slice. This is because we expect an
697 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
698 /// all these instructions can simply be removed and replaced with poison as
699 /// they come from outside of the allocated space.
700 SmallVector<Instruction *, 8> DeadUsers;
701
702 /// Uses which will become dead if can promote the alloca.
703 SmallVector<Use *, 8> DeadUseIfPromotable;
704
705 /// Operands which will become dead if we rewrite the alloca.
706 ///
707 /// These are operands that in their particular use can be replaced with
708 /// poison when we rewrite the alloca. These show up in out-of-bounds inputs
709 /// to PHI nodes and the like. They aren't entirely dead (there might be
710 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
711 /// want to swap this particular input for poison to simplify the use lists of
712 /// the alloca.
713 SmallVector<Use *, 8> DeadOperands;
714};
715
716/// A partition of the slices.
717///
718/// An ephemeral representation for a range of slices which can be viewed as
719/// a partition of the alloca. This range represents a span of the alloca's
720/// memory which cannot be split, and provides access to all of the slices
721/// overlapping some part of the partition.
722///
723/// Objects of this type are produced by traversing the alloca's slices, but
724/// are only ephemeral and not persistent.
725class Partition {
726private:
727 friend class AllocaSlices;
728 friend class AllocaSlices::partition_iterator;
729
730 using iterator = AllocaSlices::iterator;
731
732 /// The beginning and ending offsets of the alloca for this
733 /// partition.
734 uint64_t BeginOffset = 0, EndOffset = 0;
735
736 /// The start and end iterators of this partition.
737 iterator SI, SJ;
738
739 /// A collection of split slice tails overlapping the partition.
740 SmallVector<Slice *, 4> SplitTails;
741
742 /// Raw constructor builds an empty partition starting and ending at
743 /// the given iterator.
744 Partition(iterator SI) : SI(SI), SJ(SI) {}
745
746public:
747 /// The start offset of this partition.
748 ///
749 /// All of the contained slices start at or after this offset.
750 uint64_t beginOffset() const { return BeginOffset; }
751
752 /// The end offset of this partition.
753 ///
754 /// All of the contained slices end at or before this offset.
755 uint64_t endOffset() const { return EndOffset; }
756
757 /// The size of the partition.
758 ///
759 /// Note that this can never be zero.
760 uint64_t size() const {
761 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
762 return EndOffset - BeginOffset;
763 }
764
765 /// Test whether this partition contains no slices, and merely spans
766 /// a region occupied by split slices.
767 bool empty() const { return SI == SJ; }
768
769 /// \name Iterate slices that start within the partition.
770 /// These may be splittable or unsplittable. They have a begin offset >= the
771 /// partition begin offset.
772 /// @{
773 // FIXME: We should probably define a "concat_iterator" helper and use that
774 // to stitch together pointee_iterators over the split tails and the
775 // contiguous iterators of the partition. That would give a much nicer
776 // interface here. We could then additionally expose filtered iterators for
777 // split, unsplit, and unsplittable splices based on the usage patterns.
778 iterator begin() const { return SI; }
779 iterator end() const { return SJ; }
780 /// @}
781
782 /// Get the sequence of split slice tails.
783 ///
784 /// These tails are of slices which start before this partition but are
785 /// split and overlap into the partition. We accumulate these while forming
786 /// partitions.
787 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
788};
789
790} // end anonymous namespace
791
792/// An iterator over partitions of the alloca's slices.
793///
794/// This iterator implements the core algorithm for partitioning the alloca's
795/// slices. It is a forward iterator as we don't support backtracking for
796/// efficiency reasons, and re-use a single storage area to maintain the
797/// current set of split slices.
798///
799/// It is templated on the slice iterator type to use so that it can operate
800/// with either const or non-const slice iterators.
801class AllocaSlices::partition_iterator
802 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
803 Partition> {
804 friend class AllocaSlices;
805
806 /// Most of the state for walking the partitions is held in a class
807 /// with a nice interface for examining them.
808 Partition P;
809
810 /// We need to keep the end of the slices to know when to stop.
811 AllocaSlices::iterator SE;
812
813 /// We also need to keep track of the maximum split end offset seen.
814 /// FIXME: Do we really?
815 uint64_t MaxSplitSliceEndOffset = 0;
816
817 /// Sets the partition to be empty at given iterator, and sets the
818 /// end iterator.
819 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
820 : P(SI), SE(SE) {
821 // If not already at the end, advance our state to form the initial
822 // partition.
823 if (SI != SE)
824 advance();
825 }
826
827 /// Advance the iterator to the next partition.
828 ///
829 /// Requires that the iterator not be at the end of the slices.
830 void advance() {
831 assert((P.SI != SE || !P.SplitTails.empty()) &&
832 "Cannot advance past the end of the slices!");
833
834 // Clear out any split uses which have ended.
835 if (!P.SplitTails.empty()) {
836 if (P.EndOffset >= MaxSplitSliceEndOffset) {
837 // If we've finished all splits, this is easy.
838 P.SplitTails.clear();
839 MaxSplitSliceEndOffset = 0;
840 } else {
841 // Remove the uses which have ended in the prior partition. This
842 // cannot change the max split slice end because we just checked that
843 // the prior partition ended prior to that max.
844 llvm::erase_if(C&: P.SplitTails,
845 P: [&](Slice *S) { return S->endOffset() <= P.EndOffset; });
846 assert(llvm::any_of(P.SplitTails,
847 [&](Slice *S) {
848 return S->endOffset() == MaxSplitSliceEndOffset;
849 }) &&
850 "Could not find the current max split slice offset!");
851 assert(llvm::all_of(P.SplitTails,
852 [&](Slice *S) {
853 return S->endOffset() <= MaxSplitSliceEndOffset;
854 }) &&
855 "Max split slice end offset is not actually the max!");
856 }
857 }
858
859 // If P.SI is already at the end, then we've cleared the split tail and
860 // now have an end iterator.
861 if (P.SI == SE) {
862 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
863 return;
864 }
865
866 // If we had a non-empty partition previously, set up the state for
867 // subsequent partitions.
868 if (P.SI != P.SJ) {
869 // Accumulate all the splittable slices which started in the old
870 // partition into the split list.
871 for (Slice &S : P)
872 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
873 P.SplitTails.push_back(Elt: &S);
874 MaxSplitSliceEndOffset =
875 std::max(a: S.endOffset(), b: MaxSplitSliceEndOffset);
876 }
877
878 // Start from the end of the previous partition.
879 P.SI = P.SJ;
880
881 // If P.SI is now at the end, we at most have a tail of split slices.
882 if (P.SI == SE) {
883 P.BeginOffset = P.EndOffset;
884 P.EndOffset = MaxSplitSliceEndOffset;
885 return;
886 }
887
888 // If the we have split slices and the next slice is after a gap and is
889 // not splittable immediately form an empty partition for the split
890 // slices up until the next slice begins.
891 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
892 !P.SI->isSplittable()) {
893 P.BeginOffset = P.EndOffset;
894 P.EndOffset = P.SI->beginOffset();
895 return;
896 }
897 }
898
899 // OK, we need to consume new slices. Set the end offset based on the
900 // current slice, and step SJ past it. The beginning offset of the
901 // partition is the beginning offset of the next slice unless we have
902 // pre-existing split slices that are continuing, in which case we begin
903 // at the prior end offset.
904 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
905 P.EndOffset = P.SI->endOffset();
906 ++P.SJ;
907
908 // There are two strategies to form a partition based on whether the
909 // partition starts with an unsplittable slice or a splittable slice.
910 if (!P.SI->isSplittable()) {
911 // When we're forming an unsplittable region, it must always start at
912 // the first slice and will extend through its end.
913 assert(P.BeginOffset == P.SI->beginOffset());
914
915 // Form a partition including all of the overlapping slices with this
916 // unsplittable slice.
917 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
918 if (!P.SJ->isSplittable())
919 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
920 ++P.SJ;
921 }
922
923 // We have a partition across a set of overlapping unsplittable
924 // partitions.
925 return;
926 }
927
928 // If we're starting with a splittable slice, then we need to form
929 // a synthetic partition spanning it and any other overlapping splittable
930 // splices.
931 assert(P.SI->isSplittable() && "Forming a splittable partition!");
932
933 // Collect all of the overlapping splittable slices.
934 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
935 P.SJ->isSplittable()) {
936 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
937 ++P.SJ;
938 }
939
940 // Back upiP.EndOffset if we ended the span early when encountering an
941 // unsplittable slice. This synthesizes the early end offset of
942 // a partition spanning only splittable slices.
943 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
944 assert(!P.SJ->isSplittable());
945 P.EndOffset = P.SJ->beginOffset();
946 }
947 }
948
949public:
950 bool operator==(const partition_iterator &RHS) const {
951 assert(SE == RHS.SE &&
952 "End iterators don't match between compared partition iterators!");
953
954 // The observed positions of partitions is marked by the P.SI iterator and
955 // the emptiness of the split slices. The latter is only relevant when
956 // P.SI == SE, as the end iterator will additionally have an empty split
957 // slices list, but the prior may have the same P.SI and a tail of split
958 // slices.
959 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
960 assert(P.SJ == RHS.P.SJ &&
961 "Same set of slices formed two different sized partitions!");
962 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
963 "Same slice position with differently sized non-empty split "
964 "slice tails!");
965 return true;
966 }
967 return false;
968 }
969
970 partition_iterator &operator++() {
971 advance();
972 return *this;
973 }
974
975 Partition &operator*() { return P; }
976};
977
978/// A forward range over the partitions of the alloca's slices.
979///
980/// This accesses an iterator range over the partitions of the alloca's
981/// slices. It computes these partitions on the fly based on the overlapping
982/// offsets of the slices and the ability to split them. It will visit "empty"
983/// partitions to cover regions of the alloca only accessed via split
984/// slices.
985iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
986 return make_range(x: partition_iterator(begin(), end()),
987 y: partition_iterator(end(), end()));
988}
989
990static Value *foldSelectInst(SelectInst &SI) {
991 // If the condition being selected on is a constant or the same value is
992 // being selected between, fold the select. Yes this does (rarely) happen
993 // early on.
994 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: SI.getCondition()))
995 return SI.getOperand(i_nocapture: 1 + CI->isZero());
996 if (SI.getOperand(i_nocapture: 1) == SI.getOperand(i_nocapture: 2))
997 return SI.getOperand(i_nocapture: 1);
998
999 return nullptr;
1000}
1001
1002/// A helper that folds a PHI node or a select.
1003static Value *foldPHINodeOrSelectInst(Instruction &I) {
1004 if (PHINode *PN = dyn_cast<PHINode>(Val: &I)) {
1005 // If PN merges together the same value, return that value.
1006 return PN->hasConstantValue();
1007 }
1008 return foldSelectInst(SI&: cast<SelectInst>(Val&: I));
1009}
1010
1011/// Builder for the alloca slices.
1012///
1013/// This class builds a set of alloca slices by recursively visiting the uses
1014/// of an alloca and making a slice for each load and store at each offset.
1015class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1016 friend class PtrUseVisitor<SliceBuilder>;
1017 friend class InstVisitor<SliceBuilder>;
1018
1019 using Base = PtrUseVisitor<SliceBuilder>;
1020
1021 const uint64_t AllocSize;
1022 AllocaSlices &AS;
1023
1024 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
1025 SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
1026
1027 /// Set to de-duplicate dead instructions found in the use walk.
1028 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
1029
1030public:
1031 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1032 : PtrUseVisitor<SliceBuilder>(DL),
1033 AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1034
1035private:
1036 void markAsDead(Instruction &I) {
1037 if (VisitedDeadInsts.insert(Ptr: &I).second)
1038 AS.DeadUsers.push_back(Elt: &I);
1039 }
1040
1041 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1042 bool IsSplittable = false) {
1043 // Completely skip uses which have a zero size or start either before or
1044 // past the end of the allocation.
1045 if (Size == 0 || Offset.uge(RHS: AllocSize)) {
1046 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1047 << Offset
1048 << " which has zero size or starts outside of the "
1049 << AllocSize << " byte alloca:\n"
1050 << " alloca: " << AS.AI << "\n"
1051 << " use: " << I << "\n");
1052 return markAsDead(I);
1053 }
1054
1055 uint64_t BeginOffset = Offset.getZExtValue();
1056 uint64_t EndOffset = BeginOffset + Size;
1057
1058 // Clamp the end offset to the end of the allocation. Note that this is
1059 // formulated to handle even the case where "BeginOffset + Size" overflows.
1060 // This may appear superficially to be something we could ignore entirely,
1061 // but that is not so! There may be widened loads or PHI-node uses where
1062 // some instructions are dead but not others. We can't completely ignore
1063 // them, and so have to record at least the information here.
1064 assert(AllocSize >= BeginOffset); // Established above.
1065 if (Size > AllocSize - BeginOffset) {
1066 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1067 << Offset << " to remain within the " << AllocSize
1068 << " byte alloca:\n"
1069 << " alloca: " << AS.AI << "\n"
1070 << " use: " << I << "\n");
1071 EndOffset = AllocSize;
1072 }
1073
1074 AS.Slices.push_back(Elt: Slice(BeginOffset, EndOffset, U, IsSplittable));
1075 }
1076
1077 void visitBitCastInst(BitCastInst &BC) {
1078 if (BC.use_empty())
1079 return markAsDead(I&: BC);
1080
1081 return Base::visitBitCastInst(BC);
1082 }
1083
1084 void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1085 if (ASC.use_empty())
1086 return markAsDead(I&: ASC);
1087
1088 return Base::visitAddrSpaceCastInst(ASC);
1089 }
1090
1091 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1092 if (GEPI.use_empty())
1093 return markAsDead(I&: GEPI);
1094
1095 return Base::visitGetElementPtrInst(GEPI);
1096 }
1097
1098 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
1099 uint64_t Size, bool IsVolatile) {
1100 // We allow splitting of non-volatile loads and stores where the type is an
1101 // integer type. These may be used to implement 'memcpy' or other "transfer
1102 // of bits" patterns.
1103 bool IsSplittable =
1104 Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1105
1106 insertUse(I, Offset, Size, IsSplittable);
1107 }
1108
1109 void visitLoadInst(LoadInst &LI) {
1110 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
1111 "All simple FCA loads should have been pre-split");
1112
1113 // If there is a load with an unknown offset, we can still perform store
1114 // to load forwarding for other known-offset loads.
1115 if (!IsOffsetKnown)
1116 return PI.setEscapedReadOnly(&LI);
1117
1118 TypeSize Size = DL.getTypeStoreSize(Ty: LI.getType());
1119 if (Size.isScalable()) {
1120 unsigned VScale = LI.getFunction()->getVScaleValue();
1121 if (!VScale)
1122 return PI.setAborted(&LI);
1123
1124 Size = TypeSize::getFixed(ExactSize: Size.getKnownMinValue() * VScale);
1125 }
1126
1127 return handleLoadOrStore(Ty: LI.getType(), I&: LI, Offset, Size: Size.getFixedValue(),
1128 IsVolatile: LI.isVolatile());
1129 }
1130
1131 void visitStoreInst(StoreInst &SI) {
1132 Value *ValOp = SI.getValueOperand();
1133 if (ValOp == *U)
1134 return PI.setEscapedAndAborted(&SI);
1135 if (!IsOffsetKnown)
1136 return PI.setAborted(&SI);
1137
1138 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValOp->getType());
1139 if (StoreSize.isScalable()) {
1140 unsigned VScale = SI.getFunction()->getVScaleValue();
1141 if (!VScale)
1142 return PI.setAborted(&SI);
1143
1144 StoreSize = TypeSize::getFixed(ExactSize: StoreSize.getKnownMinValue() * VScale);
1145 }
1146
1147 uint64_t Size = StoreSize.getFixedValue();
1148
1149 // If this memory access can be shown to *statically* extend outside the
1150 // bounds of the allocation, it's behavior is undefined, so simply
1151 // ignore it. Note that this is more strict than the generic clamping
1152 // behavior of insertUse. We also try to handle cases which might run the
1153 // risk of overflow.
1154 // FIXME: We should instead consider the pointer to have escaped if this
1155 // function is being instrumented for addressing bugs or race conditions.
1156 if (Size > AllocSize || Offset.ugt(RHS: AllocSize - Size)) {
1157 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1158 << Offset << " which extends past the end of the "
1159 << AllocSize << " byte alloca:\n"
1160 << " alloca: " << AS.AI << "\n"
1161 << " use: " << SI << "\n");
1162 return markAsDead(I&: SI);
1163 }
1164
1165 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1166 "All simple FCA stores should have been pre-split");
1167 handleLoadOrStore(Ty: ValOp->getType(), I&: SI, Offset, Size, IsVolatile: SI.isVolatile());
1168 }
1169
1170 void visitMemSetInst(MemSetInst &II) {
1171 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1172 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1173 if ((Length && Length->getValue() == 0) ||
1174 (IsOffsetKnown && Offset.uge(RHS: AllocSize)))
1175 // Zero-length mem transfer intrinsics can be ignored entirely.
1176 return markAsDead(I&: II);
1177
1178 if (!IsOffsetKnown)
1179 return PI.setAborted(&II);
1180
1181 insertUse(I&: II, Offset,
1182 Size: Length ? Length->getLimitedValue()
1183 : AllocSize - Offset.getLimitedValue(),
1184 IsSplittable: (bool)Length);
1185 }
1186
1187 void visitMemTransferInst(MemTransferInst &II) {
1188 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1189 if (Length && Length->getValue() == 0)
1190 // Zero-length mem transfer intrinsics can be ignored entirely.
1191 return markAsDead(I&: II);
1192
1193 // Because we can visit these intrinsics twice, also check to see if the
1194 // first time marked this instruction as dead. If so, skip it.
1195 if (VisitedDeadInsts.count(Ptr: &II))
1196 return;
1197
1198 if (!IsOffsetKnown)
1199 return PI.setAborted(&II);
1200
1201 // This side of the transfer is completely out-of-bounds, and so we can
1202 // nuke the entire transfer. However, we also need to nuke the other side
1203 // if already added to our partitions.
1204 // FIXME: Yet another place we really should bypass this when
1205 // instrumenting for ASan.
1206 if (Offset.uge(RHS: AllocSize)) {
1207 auto MTPI = MemTransferSliceMap.find(Val: &II);
1208 if (MTPI != MemTransferSliceMap.end())
1209 AS.Slices[MTPI->second].kill();
1210 return markAsDead(I&: II);
1211 }
1212
1213 uint64_t RawOffset = Offset.getLimitedValue();
1214 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1215
1216 // Check for the special case where the same exact value is used for both
1217 // source and dest.
1218 if (*U == II.getRawDest() && *U == II.getRawSource()) {
1219 // For non-volatile transfers this is a no-op.
1220 if (!II.isVolatile())
1221 return markAsDead(I&: II);
1222
1223 return insertUse(I&: II, Offset, Size, /*IsSplittable=*/false);
1224 }
1225
1226 // If we have seen both source and destination for a mem transfer, then
1227 // they both point to the same alloca.
1228 bool Inserted;
1229 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
1230 std::tie(args&: MTPI, args&: Inserted) =
1231 MemTransferSliceMap.insert(KV: std::make_pair(x: &II, y: AS.Slices.size()));
1232 unsigned PrevIdx = MTPI->second;
1233 if (!Inserted) {
1234 Slice &PrevP = AS.Slices[PrevIdx];
1235
1236 // Check if the begin offsets match and this is a non-volatile transfer.
1237 // In that case, we can completely elide the transfer.
1238 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1239 PrevP.kill();
1240 return markAsDead(I&: II);
1241 }
1242
1243 // Otherwise we have an offset transfer within the same alloca. We can't
1244 // split those.
1245 PrevP.makeUnsplittable();
1246 }
1247
1248 // Insert the use now that we've fixed up the splittable nature.
1249 insertUse(I&: II, Offset, Size, /*IsSplittable=*/Inserted && Length);
1250
1251 // Check that we ended up with a valid index in the map.
1252 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1253 "Map index doesn't point back to a slice with this user.");
1254 }
1255
1256 // Disable SRoA for any intrinsics except for lifetime invariants.
1257 // FIXME: What about debug intrinsics? This matches old behavior, but
1258 // doesn't make sense.
1259 void visitIntrinsicInst(IntrinsicInst &II) {
1260 if (II.isDroppable()) {
1261 AS.DeadUseIfPromotable.push_back(Elt: U);
1262 return;
1263 }
1264
1265 if (!IsOffsetKnown)
1266 return PI.setAborted(&II);
1267
1268 if (II.isLifetimeStartOrEnd()) {
1269 insertUse(I&: II, Offset, Size: AllocSize, IsSplittable: true);
1270 return;
1271 }
1272
1273 Base::visitIntrinsicInst(II);
1274 }
1275
1276 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
1277 // We consider any PHI or select that results in a direct load or store of
1278 // the same offset to be a viable use for slicing purposes. These uses
1279 // are considered unsplittable and the size is the maximum loaded or stored
1280 // size.
1281 SmallPtrSet<Instruction *, 4> Visited;
1282 SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses;
1283 Visited.insert(Ptr: Root);
1284 Uses.push_back(Elt: std::make_pair(x: cast<Instruction>(Val&: *U), y&: Root));
1285 const DataLayout &DL = Root->getDataLayout();
1286 // If there are no loads or stores, the access is dead. We mark that as
1287 // a size zero access.
1288 Size = 0;
1289 do {
1290 Instruction *I, *UsedI;
1291 std::tie(args&: UsedI, args&: I) = Uses.pop_back_val();
1292
1293 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1294 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
1295 if (LoadSize.isScalable()) {
1296 PI.setAborted(LI);
1297 return nullptr;
1298 }
1299 Size = std::max(a: Size, b: LoadSize.getFixedValue());
1300 continue;
1301 }
1302 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1303 Value *Op = SI->getOperand(i_nocapture: 0);
1304 if (Op == UsedI)
1305 return SI;
1306 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Op->getType());
1307 if (StoreSize.isScalable()) {
1308 PI.setAborted(SI);
1309 return nullptr;
1310 }
1311 Size = std::max(a: Size, b: StoreSize.getFixedValue());
1312 continue;
1313 }
1314
1315 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1316 if (!GEP->hasAllZeroIndices())
1317 return GEP;
1318 } else if (!isa<BitCastInst>(Val: I) && !isa<PHINode>(Val: I) &&
1319 !isa<SelectInst>(Val: I) && !isa<AddrSpaceCastInst>(Val: I)) {
1320 return I;
1321 }
1322
1323 for (User *U : I->users())
1324 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
1325 Uses.push_back(Elt: std::make_pair(x&: I, y: cast<Instruction>(Val: U)));
1326 } while (!Uses.empty());
1327
1328 return nullptr;
1329 }
1330
1331 void visitPHINodeOrSelectInst(Instruction &I) {
1332 assert(isa<PHINode>(I) || isa<SelectInst>(I));
1333 if (I.use_empty())
1334 return markAsDead(I);
1335
1336 // If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1337 // instructions in this BB, which may be required during rewriting. Bail out
1338 // on these cases.
1339 if (isa<PHINode>(Val: I) && !I.getParent()->hasInsertionPt())
1340 return PI.setAborted(&I);
1341
1342 // TODO: We could use simplifyInstruction here to fold PHINodes and
1343 // SelectInsts. However, doing so requires to change the current
1344 // dead-operand-tracking mechanism. For instance, suppose neither loading
1345 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1346 // trap either. However, if we simply replace %U with undef using the
1347 // current dead-operand-tracking mechanism, "load (select undef, undef,
1348 // %other)" may trap because the select may return the first operand
1349 // "undef".
1350 if (Value *Result = foldPHINodeOrSelectInst(I)) {
1351 if (Result == *U)
1352 // If the result of the constant fold will be the pointer, recurse
1353 // through the PHI/select as if we had RAUW'ed it.
1354 enqueueUsers(I);
1355 else
1356 // Otherwise the operand to the PHI/select is dead, and we can replace
1357 // it with poison.
1358 AS.DeadOperands.push_back(Elt: U);
1359
1360 return;
1361 }
1362
1363 if (!IsOffsetKnown)
1364 return PI.setAborted(&I);
1365
1366 // See if we already have computed info on this node.
1367 uint64_t &Size = PHIOrSelectSizes[&I];
1368 if (!Size) {
1369 // This is a new PHI/Select, check for an unsafe use of it.
1370 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(Root: &I, Size))
1371 return PI.setAborted(UnsafeI);
1372 }
1373
1374 // For PHI and select operands outside the alloca, we can't nuke the entire
1375 // phi or select -- the other side might still be relevant, so we special
1376 // case them here and use a separate structure to track the operands
1377 // themselves which should be replaced with poison.
1378 // FIXME: This should instead be escaped in the event we're instrumenting
1379 // for address sanitization.
1380 if (Offset.uge(RHS: AllocSize)) {
1381 AS.DeadOperands.push_back(Elt: U);
1382 return;
1383 }
1384
1385 insertUse(I, Offset, Size);
1386 }
1387
1388 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(I&: PN); }
1389
1390 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(I&: SI); }
1391
1392 /// Disable SROA entirely if there are unhandled users of the alloca.
1393 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1394
1395 void visitCallBase(CallBase &CB) {
1396 // If the call operand is read-only and only does a read-only or address
1397 // capture, then we mark it as EscapedReadOnly.
1398 if (CB.isDataOperand(U) &&
1399 !capturesFullProvenance(CC: CB.getCaptureInfo(OpNo: U->getOperandNo())) &&
1400 CB.onlyReadsMemory(OpNo: U->getOperandNo())) {
1401 PI.setEscapedReadOnly(&CB);
1402 return;
1403 }
1404
1405 Base::visitCallBase(CB);
1406 }
1407};
1408
1409AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1410 :
1411#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1412 AI(AI),
1413#endif
1414 PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1415 SliceBuilder PB(DL, AI, *this);
1416 SliceBuilder::PtrInfo PtrI = PB.visitPtr(I&: AI);
1417 if (PtrI.isEscaped() || PtrI.isAborted()) {
1418 // FIXME: We should sink the escape vs. abort info into the caller nicely,
1419 // possibly by just storing the PtrInfo in the AllocaSlices.
1420 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1421 : PtrI.getAbortingInst();
1422 assert(PointerEscapingInstr && "Did not track a bad instruction");
1423 return;
1424 }
1425 PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1426
1427 llvm::erase_if(C&: Slices, P: [](const Slice &S) { return S.isDead(); });
1428
1429 // Sort the uses. This arranges for the offsets to be in ascending order,
1430 // and the sizes to be in descending order.
1431 llvm::stable_sort(Range&: Slices);
1432}
1433
1434#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1435
1436void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1437 StringRef Indent) const {
1438 printSlice(OS, I, Indent);
1439 OS << "\n";
1440 printUse(OS, I, Indent);
1441}
1442
1443void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1444 StringRef Indent) const {
1445 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1446 << " slice #" << (I - begin())
1447 << (I->isSplittable() ? " (splittable)" : "");
1448}
1449
1450void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1451 StringRef Indent) const {
1452 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1453}
1454
1455void AllocaSlices::print(raw_ostream &OS) const {
1456 if (PointerEscapingInstr) {
1457 OS << "Can't analyze slices for alloca: " << AI << "\n"
1458 << " A pointer to this alloca escaped by:\n"
1459 << " " << *PointerEscapingInstr << "\n";
1460 return;
1461 }
1462
1463 if (PointerEscapingInstrReadOnly)
1464 OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1465
1466 OS << "Slices of alloca: " << AI << "\n";
1467 for (const_iterator I = begin(), E = end(); I != E; ++I)
1468 print(OS, I);
1469}
1470
1471LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1472 print(dbgs(), I);
1473}
1474LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1475
1476#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1477
1478/// Find a common load/store type used through a pointer PHI or select.
1479///
1480/// Look through a PHI or select to see if all of its users are loads or stores
1481/// of one common type. Whether those accesses can be speculated does not affect
1482/// the type they use and is checked separately when attempting promotion.
1483static Type *findCommonTypeThroughPHIOrSelect(Instruction &I) {
1484 assert((isa<PHINode, SelectInst>(I)) && "expected a PHI or select");
1485 Type *Ty = nullptr;
1486
1487 for (User *U : I.users()) {
1488 Type *UserTy = nullptr;
1489 if (auto *LI = dyn_cast<LoadInst>(Val: U))
1490 UserTy = LI->getType();
1491 else if (auto *Store = dyn_cast<StoreInst>(Val: U))
1492 // Slice building rejects stores of the PHI-or-select-derived pointer, so
1493 // it must be the store's pointer operand here.
1494 UserTy = Store->getValueOperand()->getType();
1495
1496 if (!UserTy || (Ty && Ty != UserTy))
1497 return nullptr;
1498 Ty = UserTy;
1499 }
1500
1501 return Ty;
1502}
1503
1504/// Walk the range of a partitioning looking for a common type to cover this
1505/// sequence of slices.
1506static std::pair<Type *, IntegerType *>
1507findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1508 uint64_t EndOffset) {
1509 Type *Ty = nullptr;
1510 bool TyIsCommon = true;
1511 IntegerType *ITy = nullptr;
1512
1513 // Note that we need to look at *every* alloca slice's Use to ensure we
1514 // always get consistent results regardless of the order of slices.
1515 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1516 Use *U = I->getUse();
1517 if (isa<IntrinsicInst>(Val: *U->getUser()))
1518 continue;
1519 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1520 continue;
1521
1522 Type *UserTy = nullptr;
1523 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
1524 UserTy = LI->getType();
1525 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
1526 UserTy = SI->getValueOperand()->getType();
1527 } else if (isa<PHINode, SelectInst>(Val: U->getUser())) {
1528 UserTy =
1529 findCommonTypeThroughPHIOrSelect(I&: *cast<Instruction>(Val: U->getUser()));
1530 }
1531
1532 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(Val: UserTy)) {
1533 // If the type is larger than the partition, skip it. We only encounter
1534 // this for split integer operations where we want to use the type of the
1535 // entity causing the split. Also skip if the type is not a byte width
1536 // multiple.
1537 if (UserITy->getBitWidth() % 8 != 0 ||
1538 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
1539 continue;
1540
1541 // Track the largest bitwidth integer type used in this way in case there
1542 // is no common type.
1543 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1544 ITy = UserITy;
1545 }
1546
1547 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1548 // depend on types skipped above.
1549 if (!UserTy || (Ty && Ty != UserTy))
1550 TyIsCommon = false; // Give up on anything but an iN type.
1551 else
1552 Ty = UserTy;
1553 }
1554
1555 return {TyIsCommon ? Ty : nullptr, ITy};
1556}
1557
1558/// PHI instructions that use an alloca and are subsequently loaded can be
1559/// rewritten to load both input pointers in the pred blocks and then PHI the
1560/// results, allowing the load of the alloca to be promoted.
1561/// From this:
1562/// %P2 = phi [i32* %Alloca, i32* %Other]
1563/// %V = load i32* %P2
1564/// to:
1565/// %V1 = load i32* %Alloca -> will be mem2reg'd
1566/// ...
1567/// %V2 = load i32* %Other
1568/// ...
1569/// %V = phi [i32 %V1, i32 %V2]
1570///
1571/// We can do this to a select if its only uses are loads and if the operands
1572/// to the select can be loaded unconditionally.
1573///
1574/// FIXME: This should be hoisted into a generic utility, likely in
1575/// Transforms/Util/Local.h
1576static bool isSafePHIToSpeculate(PHINode &PN) {
1577 const DataLayout &DL = PN.getDataLayout();
1578
1579 // For now, we can only do this promotion if the load is in the same block
1580 // as the PHI, and if there are no stores between the phi and load.
1581 // TODO: Allow recursive phi users.
1582 // TODO: Allow stores.
1583 BasicBlock *BB = PN.getParent();
1584 Align MaxAlign;
1585 uint64_t APWidth = DL.getIndexTypeSizeInBits(Ty: PN.getType());
1586 Type *LoadType = nullptr;
1587 for (User *U : PN.users()) {
1588 LoadInst *LI = dyn_cast<LoadInst>(Val: U);
1589 if (!LI || !LI->isSimple())
1590 return false;
1591
1592 // For now we only allow loads in the same block as the PHI. This is
1593 // a common case that happens when instcombine merges two loads through
1594 // a PHI.
1595 if (LI->getParent() != BB)
1596 return false;
1597
1598 if (LoadType) {
1599 if (LoadType != LI->getType())
1600 return false;
1601 } else {
1602 LoadType = LI->getType();
1603 }
1604
1605 // Ensure that there are no instructions between the PHI and the load that
1606 // could store.
1607 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1608 if (BBI->mayWriteToMemory())
1609 return false;
1610
1611 MaxAlign = std::max(a: MaxAlign, b: LI->getAlign());
1612 }
1613
1614 if (!LoadType)
1615 return false;
1616
1617 APInt LoadSize =
1618 APInt(APWidth, DL.getTypeStoreSize(Ty: LoadType).getFixedValue());
1619
1620 // We can only transform this if it is safe to push the loads into the
1621 // predecessor blocks. The only thing to watch out for is that we can't put
1622 // a possibly trapping load in the predecessor if it is a critical edge.
1623 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1624 Instruction *TI = PN.getIncomingBlock(i: Idx)->getTerminator();
1625 Value *InVal = PN.getIncomingValue(i: Idx);
1626
1627 // If the value is produced by the terminator of the predecessor (an
1628 // invoke) or it has side-effects, there is no valid place to put a load
1629 // in the predecessor.
1630 if (TI == InVal || TI->mayHaveSideEffects())
1631 return false;
1632
1633 // If the predecessor has a single successor, then the edge isn't
1634 // critical.
1635 if (TI->getNumSuccessors() == 1)
1636 continue;
1637
1638 // If this pointer is always safe to load, or if we can prove that there
1639 // is already a load in the block, then we can move the load to the pred
1640 // block.
1641 if (isSafeToLoadUnconditionally(V: InVal, Alignment: MaxAlign, Size: LoadSize,
1642 SQ: SimplifyQuery(DL, TI)))
1643 continue;
1644
1645 return false;
1646 }
1647
1648 return true;
1649}
1650
1651static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1652 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1653
1654 LoadInst *SomeLoad = cast<LoadInst>(Val: PN.user_back());
1655 Type *LoadTy = SomeLoad->getType();
1656 IRB.SetInsertPoint(&PN);
1657 PHINode *NewPN = IRB.CreatePHI(Ty: LoadTy, NumReservedValues: PN.getNumIncomingValues(),
1658 Name: PN.getName() + ".sroa.speculated");
1659
1660 // Get the AA tags and alignment to use from one of the loads. It does not
1661 // matter which one we get and if any differ.
1662 AAMDNodes AATags = SomeLoad->getAAMetadata();
1663 Align Alignment = SomeLoad->getAlign();
1664
1665 // Rewrite all loads of the PN to use the new PHI.
1666 while (!PN.use_empty()) {
1667 LoadInst *LI = cast<LoadInst>(Val: PN.user_back());
1668 LI->replaceAllUsesWith(V: NewPN);
1669 LI->eraseFromParent();
1670 }
1671
1672 // Inject loads into all of the pred blocks.
1673 DenseMap<BasicBlock *, Value *> InjectedLoads;
1674 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1675 BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1676 Value *InVal = PN.getIncomingValue(i: Idx);
1677
1678 // A PHI node is allowed to have multiple (duplicated) entries for the same
1679 // basic block, as long as the value is the same. So if we already injected
1680 // a load in the predecessor, then we should reuse the same load for all
1681 // duplicated entries.
1682 if (Value *V = InjectedLoads.lookup(Val: Pred)) {
1683 NewPN->addIncoming(V, BB: Pred);
1684 continue;
1685 }
1686
1687 Instruction *TI = Pred->getTerminator();
1688 IRB.SetInsertPoint(TI);
1689
1690 LoadInst *Load = IRB.CreateAlignedLoad(
1691 Ty: LoadTy, Ptr: InVal, Align: Alignment,
1692 Name: (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1693 ++NumLoadsSpeculated;
1694 if (AATags)
1695 Load->setAAMetadata(AATags);
1696 NewPN->addIncoming(V: Load, BB: Pred);
1697 InjectedLoads[Pred] = Load;
1698 }
1699
1700 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1701 PN.eraseFromParent();
1702}
1703
1704SelectHandSpeculativity &
1705SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1706 if (isTrueVal)
1707 Bitfield::set<SelectHandSpeculativity::TrueVal>(Packed&: Storage, Value: true);
1708 else
1709 Bitfield::set<SelectHandSpeculativity::FalseVal>(Packed&: Storage, Value: true);
1710 return *this;
1711}
1712
1713bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1714 return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Packed: Storage)
1715 : Bitfield::get<SelectHandSpeculativity::FalseVal>(Packed: Storage);
1716}
1717
1718bool SelectHandSpeculativity::areAllSpeculatable() const {
1719 return isSpeculatable(/*isTrueVal=*/true) &&
1720 isSpeculatable(/*isTrueVal=*/false);
1721}
1722
1723bool SelectHandSpeculativity::areAnySpeculatable() const {
1724 return isSpeculatable(/*isTrueVal=*/true) ||
1725 isSpeculatable(/*isTrueVal=*/false);
1726}
1727bool SelectHandSpeculativity::areNoneSpeculatable() const {
1728 return !areAnySpeculatable();
1729}
1730
1731static SelectHandSpeculativity
1732isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1733 assert(LI.isSimple() && "Only for simple loads");
1734 SelectHandSpeculativity Spec;
1735
1736 const DataLayout &DL = SI.getDataLayout();
1737 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1738 if (isSafeToLoadUnconditionally(V: Value, Ty: LI.getType(), Alignment: LI.getAlign(),
1739 SQ: SimplifyQuery(DL, &LI)))
1740 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1741 else if (PreserveCFG)
1742 return Spec;
1743
1744 return Spec;
1745}
1746
1747std::optional<RewriteableMemOps>
1748SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1749 RewriteableMemOps Ops;
1750
1751 for (User *U : SI.users()) {
1752 if (auto *Store = dyn_cast<StoreInst>(Val: U)) {
1753 // Note that atomic stores can be transformed; atomic semantics do not
1754 // have any meaning for a local alloca. Stores are not speculatable,
1755 // however, so if we can't turn it into a predicated store, we are done.
1756 if (Store->isVolatile() || PreserveCFG)
1757 return {}; // Give up on this `select`.
1758 Ops.emplace_back(Args&: Store);
1759 continue;
1760 }
1761
1762 auto *LI = dyn_cast<LoadInst>(Val: U);
1763
1764 // Note that atomic loads can be transformed;
1765 // atomic semantics do not have any meaning for a local alloca.
1766 if (!LI || LI->isVolatile())
1767 return {}; // Give up on this `select`.
1768
1769 PossiblySpeculatableLoad Load(LI);
1770 if (!LI->isSimple()) {
1771 // If the `load` is not simple, we can't speculatively execute it,
1772 // but we could handle this via a CFG modification. But can we?
1773 if (PreserveCFG)
1774 return {}; // Give up on this `select`.
1775 Ops.emplace_back(Args&: Load);
1776 continue;
1777 }
1778
1779 SelectHandSpeculativity Spec =
1780 isSafeLoadOfSelectToSpeculate(LI&: *LI, SI, PreserveCFG);
1781 if (PreserveCFG && !Spec.areAllSpeculatable())
1782 return {}; // Give up on this `select`.
1783
1784 Load.setInt(Spec);
1785 Ops.emplace_back(Args&: Load);
1786 }
1787
1788 return Ops;
1789}
1790
1791static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1792 IRBuilderTy &IRB) {
1793 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1794
1795 Value *TV = SI.getTrueValue();
1796 Value *FV = SI.getFalseValue();
1797 // Replace the given load of the select with a select of two loads.
1798
1799 assert(LI.isSimple() && "We only speculate simple loads");
1800
1801 IRB.SetInsertPoint(&LI);
1802
1803 LoadInst *TL =
1804 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: TV, Align: LI.getAlign(),
1805 Name: LI.getName() + ".sroa.speculate.load.true");
1806 LoadInst *FL =
1807 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: FV, Align: LI.getAlign(),
1808 Name: LI.getName() + ".sroa.speculate.load.false");
1809 NumLoadsSpeculated += 2;
1810
1811 // Transfer alignment and AA info if present.
1812 TL->setAlignment(LI.getAlign());
1813 FL->setAlignment(LI.getAlign());
1814
1815 AAMDNodes Tags = LI.getAAMetadata();
1816 if (Tags) {
1817 TL->setAAMetadata(Tags);
1818 FL->setAAMetadata(Tags);
1819 }
1820
1821 Value *V = IRB.CreateSelect(C: SI.getCondition(), True: TL, False: FL,
1822 Name: LI.getName() + ".sroa.speculated", MDFrom: &SI);
1823
1824 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1825 LI.replaceAllUsesWith(V);
1826}
1827
1828template <typename T>
1829static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1830 SelectHandSpeculativity Spec,
1831 DomTreeUpdater &DTU) {
1832 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!");
1833 LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1834 BasicBlock *Head = I.getParent();
1835 Instruction *ThenTerm = nullptr;
1836 Instruction *ElseTerm = nullptr;
1837 if (Spec.areNoneSpeculatable())
1838 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1839 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU);
1840 else {
1841 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1842 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU,
1843 /*LI=*/nullptr, /*ThenBlock=*/nullptr);
1844 if (Spec.isSpeculatable(/*isTrueVal=*/true))
1845 cast<CondBrInst>(Val: Head->getTerminator())->swapSuccessors();
1846 }
1847 auto *HeadBI = cast<CondBrInst>(Val: Head->getTerminator());
1848 Spec = {}; // Do not use `Spec` beyond this point.
1849 BasicBlock *Tail = I.getParent();
1850 Tail->setName(Head->getName() + ".cont");
1851 PHINode *PN;
1852 if (isa<LoadInst>(I))
1853 PN = PHINode::Create(Ty: I.getType(), NumReservedValues: 2, NameStr: "", InsertBefore: I.getIterator());
1854 for (BasicBlock *SuccBB : successors(BB: Head)) {
1855 bool IsThen = SuccBB == HeadBI->getSuccessor(i: 0);
1856 int SuccIdx = IsThen ? 0 : 1;
1857 auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1858 auto &CondMemOp = cast<T>(*I.clone());
1859 if (NewMemOpBB != Head) {
1860 NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1861 if (isa<LoadInst>(I))
1862 ++NumLoadsPredicated;
1863 else
1864 ++NumStoresPredicated;
1865 } else {
1866 CondMemOp.dropUBImplyingAttrsAndMetadata();
1867 ++NumLoadsSpeculated;
1868 }
1869 CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
1870 Value *Ptr = SI.getOperand(i_nocapture: 1 + SuccIdx);
1871 CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
1872 if (isa<LoadInst>(I)) {
1873 CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
1874 PN->addIncoming(V: &CondMemOp, BB: NewMemOpBB);
1875 } else
1876 LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
1877 }
1878 if (isa<LoadInst>(I)) {
1879 PN->takeName(V: &I);
1880 LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
1881 I.replaceAllUsesWith(PN);
1882 }
1883}
1884
1885static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I,
1886 SelectHandSpeculativity Spec,
1887 DomTreeUpdater &DTU) {
1888 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1889 rewriteMemOpOfSelect(SI&: SelInst, I&: *LI, Spec, DTU);
1890 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1891 rewriteMemOpOfSelect(SI&: SelInst, I&: *SI, Spec, DTU);
1892 else
1893 llvm_unreachable_internal(msg: "Only for load and store.");
1894}
1895
1896static bool rewriteSelectInstMemOps(SelectInst &SI,
1897 const RewriteableMemOps &Ops,
1898 IRBuilderTy &IRB, DomTreeUpdater *DTU) {
1899 bool CFGChanged = false;
1900 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
1901
1902 for (const RewriteableMemOp &Op : Ops) {
1903 SelectHandSpeculativity Spec;
1904 Instruction *I;
1905 if (auto *const *US = std::get_if<UnspeculatableStore>(ptr: &Op)) {
1906 I = *US;
1907 } else {
1908 auto PSL = std::get<PossiblySpeculatableLoad>(v: Op);
1909 I = PSL.getPointer();
1910 Spec = PSL.getInt();
1911 }
1912 if (Spec.areAllSpeculatable()) {
1913 speculateSelectInstLoads(SI, LI&: cast<LoadInst>(Val&: *I), IRB);
1914 } else {
1915 assert(DTU && "Should not get here when not allowed to modify the CFG!");
1916 rewriteMemOpOfSelect(SelInst&: SI, I&: *I, Spec, DTU&: *DTU);
1917 CFGChanged = true;
1918 }
1919 I->eraseFromParent();
1920 }
1921
1922 for (User *U : make_early_inc_range(Range: SI.users()))
1923 cast<BitCastInst>(Val: U)->eraseFromParent();
1924 SI.eraseFromParent();
1925 return CFGChanged;
1926}
1927
1928/// Compute an adjusted pointer from Ptr by Offset bytes where the
1929/// resulting pointer has PointerTy.
1930static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
1931 APInt Offset, Type *PointerTy,
1932 const Twine &NamePrefix) {
1933 if (Offset != 0)
1934 Ptr = IRB.CreateInBoundsPtrAdd(Ptr, Offset: IRB.getInt(AI: Offset),
1935 Name: NamePrefix + "sroa_idx");
1936 return IRB.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: PointerTy,
1937 Name: NamePrefix + "sroa_cast");
1938}
1939
1940/// Compute the adjusted alignment for a load or store from an offset.
1941static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
1942 return commonAlignment(A: getLoadStoreAlignment(I), Offset);
1943}
1944
1945/// Test whether we can convert a value from the old to the new type.
1946///
1947/// This predicate should be used to guard calls to convertValue in order to
1948/// ensure that we only try to convert viable values. The strategy is that we
1949/// will peel off single element struct and array wrappings to get to an
1950/// underlying value, and convert that value.
1951static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
1952 unsigned VScale = 0) {
1953 if (OldTy == NewTy)
1954 return true;
1955
1956 // For integer types, we can't handle any bit-width differences. This would
1957 // break both vector conversions with extension and introduce endianness
1958 // issues when in conjunction with loads and stores.
1959 if (isa<IntegerType>(Val: OldTy) && isa<IntegerType>(Val: NewTy)) {
1960 assert(cast<IntegerType>(OldTy)->getBitWidth() !=
1961 cast<IntegerType>(NewTy)->getBitWidth() &&
1962 "We can't have the same bitwidth for different int types");
1963 return false;
1964 }
1965
1966 TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
1967 TypeSize OldSize = DL.getTypeSizeInBits(Ty: OldTy);
1968
1969 if ((isa<ScalableVectorType>(Val: NewTy) && isa<FixedVectorType>(Val: OldTy)) ||
1970 (isa<ScalableVectorType>(Val: OldTy) && isa<FixedVectorType>(Val: NewTy))) {
1971 // Conversion is only possible when the size of scalable vectors is known.
1972 if (!VScale)
1973 return false;
1974
1975 // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
1976 // a single domain (either fixed or scalable). Any additional conversion
1977 // between fixed and scalable types is handled through integer types.
1978 auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
1979 auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
1980
1981 if (isa<ScalableVectorType>(Val: NewTy)) {
1982 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: NewVTy), EltTy: OldVTy))
1983 return false;
1984
1985 NewSize = TypeSize::getFixed(ExactSize: NewSize.getKnownMinValue() * VScale);
1986 } else {
1987 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: OldVTy), EltTy: NewVTy))
1988 return false;
1989
1990 OldSize = TypeSize::getFixed(ExactSize: OldSize.getKnownMinValue() * VScale);
1991 }
1992 }
1993
1994 if (NewSize != OldSize)
1995 return false;
1996 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
1997 return false;
1998
1999 // We can convert pointers to integers and vice-versa. Same for vectors
2000 // of pointers and integers.
2001 OldTy = OldTy->getScalarType();
2002 NewTy = NewTy->getScalarType();
2003 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
2004 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
2005 unsigned OldAS = OldTy->getPointerAddressSpace();
2006 unsigned NewAS = NewTy->getPointerAddressSpace();
2007 // Convert pointers if they are pointers from the same address space or
2008 // different integral (not non-integral) address spaces with the same
2009 // pointer size.
2010 return OldAS == NewAS ||
2011 (!DL.isNonIntegralAddressSpace(AddrSpace: OldAS) &&
2012 !DL.isNonIntegralAddressSpace(AddrSpace: NewAS) &&
2013 DL.getPointerSize(AS: OldAS) == DL.getPointerSize(AS: NewAS));
2014 }
2015
2016 // We can convert integers to integral pointers, but not to non-integral
2017 // pointers.
2018 if (OldTy->isIntegerTy())
2019 return !DL.isNonIntegralPointerType(Ty: NewTy);
2020
2021 // We can convert integral pointers to integers, but non-integral pointers
2022 // need to remain pointers.
2023 if (!DL.isNonIntegralPointerType(Ty: OldTy))
2024 return NewTy->isIntegerTy();
2025
2026 return false;
2027 }
2028
2029 if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy())
2030 return false;
2031
2032 return true;
2033}
2034
2035/// Test whether the given slice use can be promoted to a vector.
2036///
2037/// This function is called to test each entry in a partition which is slated
2038/// for a single slice.
2039static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2040 VectorType *Ty,
2041 uint64_t ElementSize,
2042 const DataLayout &DL,
2043 unsigned VScale) {
2044 // First validate the slice offsets.
2045 uint64_t BeginOffset =
2046 std::max(a: S.beginOffset(), b: P.beginOffset()) - P.beginOffset();
2047 uint64_t BeginIndex = BeginOffset / ElementSize;
2048 if (BeginIndex * ElementSize != BeginOffset ||
2049 BeginIndex >= cast<FixedVectorType>(Val: Ty)->getNumElements())
2050 return false;
2051 uint64_t EndOffset = std::min(a: S.endOffset(), b: P.endOffset()) - P.beginOffset();
2052 uint64_t EndIndex = EndOffset / ElementSize;
2053 if (EndIndex * ElementSize != EndOffset ||
2054 EndIndex > cast<FixedVectorType>(Val: Ty)->getNumElements())
2055 return false;
2056
2057 assert(EndIndex > BeginIndex && "Empty vector!");
2058 uint64_t NumElements = EndIndex - BeginIndex;
2059 Type *SliceTy = (NumElements == 1)
2060 ? Ty->getElementType()
2061 : FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElements);
2062
2063 Type *SplitIntTy =
2064 Type::getIntNTy(C&: Ty->getContext(), N: NumElements * ElementSize * 8);
2065
2066 Use *U = S.getUse();
2067
2068 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2069 if (MI->isVolatile())
2070 return false;
2071 if (!S.isSplittable())
2072 return false; // Skip any unsplittable intrinsics.
2073 if (isa<MemSetInst>(Val: MI)) {
2074 Type *SplatTy = Type::getIntNTy(C&: Ty->getContext(), N: ElementSize * 8);
2075 if (!canConvertValue(DL, OldTy: SplatTy, NewTy: Ty->getElementType(), VScale))
2076 return false;
2077 }
2078 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2079 if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2080 return false;
2081 } else if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2082 if (LI->isVolatile())
2083 return false;
2084 Type *LTy = LI->getType();
2085 // Disable vector promotion when there are loads or stores of an FCA.
2086 if (LTy->isStructTy())
2087 return false;
2088 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2089 assert(LTy->isIntegerTy());
2090 LTy = SplitIntTy;
2091 }
2092 if (!canConvertValue(DL, OldTy: SliceTy, NewTy: LTy, VScale))
2093 return false;
2094 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2095 if (SI->isVolatile())
2096 return false;
2097 Type *STy = SI->getValueOperand()->getType();
2098 // Disable vector promotion when there are loads or stores of an FCA.
2099 if (STy->isStructTy())
2100 return false;
2101 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2102 assert(STy->isIntegerTy());
2103 STy = SplitIntTy;
2104 }
2105 if (!canConvertValue(DL, OldTy: STy, NewTy: SliceTy, VScale))
2106 return false;
2107 } else {
2108 return false;
2109 }
2110
2111 return true;
2112}
2113
2114/// Test whether any vector type in \p CandidateTys is viable for promotion.
2115///
2116/// This implements the necessary checking for \c isVectorPromotionViable over
2117/// all slices of the alloca for the given VectorType.
2118static VectorType *
2119checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
2120 SmallVectorImpl<VectorType *> &CandidateTys,
2121 bool HaveCommonEltTy, Type *CommonEltTy,
2122 bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2123 VectorType *CommonVecPtrTy, unsigned VScale) {
2124 // If we didn't find a vector type, nothing to do here.
2125 if (CandidateTys.empty())
2126 return nullptr;
2127
2128 // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2129 // then we should choose it, not some other alternative.
2130 // But, we can't perform a no-op pointer address space change via bitcast,
2131 // so if we didn't have a common pointer element type, bail.
2132 if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2133 return nullptr;
2134
2135 // Try to pick the "best" element type out of the choices.
2136 if (!HaveCommonEltTy && HaveVecPtrTy) {
2137 // If there was a pointer element type, there's really only one choice.
2138 CandidateTys.clear();
2139 CandidateTys.push_back(Elt: CommonVecPtrTy);
2140 } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2141 // Integer-ify vector types.
2142 for (VectorType *&VTy : CandidateTys) {
2143 if (!VTy->getElementType()->isIntegerTy())
2144 VTy = cast<VectorType>(Val: VTy->getWithNewType(EltTy: IntegerType::getIntNTy(
2145 C&: VTy->getContext(), N: VTy->getScalarSizeInBits())));
2146 }
2147
2148 // Rank the remaining candidate vector types. This is easy because we know
2149 // they're all integer vectors. We sort by ascending number of elements.
2150 auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2151 (void)DL;
2152 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2153 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2154 "Cannot have vector types of different sizes!");
2155 assert(RHSTy->getElementType()->isIntegerTy() &&
2156 "All non-integer types eliminated!");
2157 assert(LHSTy->getElementType()->isIntegerTy() &&
2158 "All non-integer types eliminated!");
2159 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() <
2160 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2161 };
2162 auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2163 (void)DL;
2164 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2165 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2166 "Cannot have vector types of different sizes!");
2167 assert(RHSTy->getElementType()->isIntegerTy() &&
2168 "All non-integer types eliminated!");
2169 assert(LHSTy->getElementType()->isIntegerTy() &&
2170 "All non-integer types eliminated!");
2171 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() ==
2172 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2173 };
2174 llvm::sort(C&: CandidateTys, Comp: RankVectorTypesComp);
2175 CandidateTys.erase(CS: llvm::unique(R&: CandidateTys, P: RankVectorTypesEq),
2176 CE: CandidateTys.end());
2177 } else {
2178// The only way to have the same element type in every vector type is to
2179// have the same vector type. Check that and remove all but one.
2180#ifndef NDEBUG
2181 for (VectorType *VTy : CandidateTys) {
2182 assert(VTy->getElementType() == CommonEltTy &&
2183 "Unaccounted for element type!");
2184 assert(VTy == CandidateTys[0] &&
2185 "Different vector types with the same element type!");
2186 }
2187#endif
2188 CandidateTys.resize(N: 1);
2189 }
2190
2191 // FIXME: hack. Do we have a named constant for this?
2192 // SDAG SDNode can't have more than 65535 operands.
2193 llvm::erase_if(C&: CandidateTys, P: [](VectorType *VTy) {
2194 return cast<FixedVectorType>(Val: VTy)->getNumElements() >
2195 std::numeric_limits<unsigned short>::max();
2196 });
2197
2198 // Find a vector type viable for promotion by iterating over all slices.
2199 auto *VTy = llvm::find_if(Range&: CandidateTys, P: [&](VectorType *VTy) -> bool {
2200 uint64_t ElementSize =
2201 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2202
2203 // While the definition of LLVM vectors is bitpacked, we don't support sizes
2204 // that aren't byte sized.
2205 if (ElementSize % 8)
2206 return false;
2207 assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&
2208 "vector size not a multiple of element size?");
2209 ElementSize /= 8;
2210
2211 for (const Slice &S : P)
2212 if (!isVectorPromotionViableForSlice(P, S, Ty: VTy, ElementSize, DL, VScale))
2213 return false;
2214
2215 for (const Slice *S : P.splitSliceTails())
2216 if (!isVectorPromotionViableForSlice(P, S: *S, Ty: VTy, ElementSize, DL, VScale))
2217 return false;
2218
2219 return true;
2220 });
2221 return VTy != CandidateTys.end() ? *VTy : nullptr;
2222}
2223
2224static VectorType *createAndCheckVectorTypesForPromotion(
2225 SetVector<Type *> &OtherTys, ArrayRef<VectorType *> CandidateTysCopy,
2226 function_ref<void(Type *)> CheckCandidateType, Partition &P,
2227 const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2228 bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
2229 bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
2230 [[maybe_unused]] VectorType *OriginalElt =
2231 CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
2232 // Consider additional vector types where the element type size is a
2233 // multiple of load/store element size.
2234 for (Type *Ty : OtherTys) {
2235 if (!VectorType::isValidElementType(ElemTy: Ty))
2236 continue;
2237 unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2238 // Make a copy of CandidateTys and iterate through it, because we
2239 // might append to CandidateTys in the loop.
2240 for (VectorType *const VTy : CandidateTysCopy) {
2241 // The elements in the copy should remain invariant throughout the loop
2242 assert(CandidateTysCopy[0] == OriginalElt && "Different Element");
2243 unsigned VectorSize = DL.getTypeSizeInBits(Ty: VTy).getFixedValue();
2244 unsigned ElementSize =
2245 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2246 if (TypeSize != VectorSize && TypeSize != ElementSize &&
2247 VectorSize % TypeSize == 0) {
2248 VectorType *NewVTy = VectorType::get(ElementType: Ty, NumElements: VectorSize / TypeSize, Scalable: false);
2249 CheckCandidateType(NewVTy);
2250 }
2251 }
2252 }
2253
2254 return checkVectorTypesForPromotion(
2255 P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2256 HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2257}
2258
2259/// Test whether the given alloca partitioning and range of slices can be
2260/// promoted to a vector.
2261///
2262/// This is a quick test to check whether we can rewrite a particular alloca
2263/// partition (and its newly formed alloca) into a vector alloca with only
2264/// whole-vector loads and stores such that it could be promoted to a vector
2265/// SSA value. We only can ensure this for a limited set of operations, and we
2266/// don't want to do the rewrites unless we are confident that the result will
2267/// be promotable, so we have an early test here.
2268static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL,
2269 unsigned VScale) {
2270 // Collect the candidate types for vector-based promotion. Also track whether
2271 // we have different element types.
2272 SmallVector<VectorType *, 4> CandidateTys;
2273 SetVector<Type *> LoadStoreTys;
2274 SetVector<Type *> DeferredTys;
2275 Type *CommonEltTy = nullptr;
2276 VectorType *CommonVecPtrTy = nullptr;
2277 bool HaveVecPtrTy = false;
2278 bool HaveCommonEltTy = true;
2279 bool HaveCommonVecPtrTy = true;
2280 auto CheckCandidateType = [&](Type *Ty) {
2281 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2282 // Return if bitcast to vectors is different for total size in bits.
2283 if (!CandidateTys.empty()) {
2284 VectorType *V = CandidateTys[0];
2285 if (DL.getTypeSizeInBits(Ty: VTy).getFixedValue() !=
2286 DL.getTypeSizeInBits(Ty: V).getFixedValue()) {
2287 CandidateTys.clear();
2288 return;
2289 }
2290 }
2291 CandidateTys.push_back(Elt: VTy);
2292 Type *EltTy = VTy->getElementType();
2293
2294 if (!CommonEltTy)
2295 CommonEltTy = EltTy;
2296 else if (CommonEltTy != EltTy)
2297 HaveCommonEltTy = false;
2298
2299 if (EltTy->isPointerTy()) {
2300 HaveVecPtrTy = true;
2301 if (!CommonVecPtrTy)
2302 CommonVecPtrTy = VTy;
2303 else if (CommonVecPtrTy != VTy)
2304 HaveCommonVecPtrTy = false;
2305 }
2306 }
2307 };
2308
2309 // Put load and store types into a set for de-duplication.
2310 for (const Slice &S : P) {
2311 Type *Ty;
2312 if (auto *LI = dyn_cast<LoadInst>(Val: S.getUse()->getUser()))
2313 Ty = LI->getType();
2314 else if (auto *SI = dyn_cast<StoreInst>(Val: S.getUse()->getUser()))
2315 Ty = SI->getValueOperand()->getType();
2316 else
2317 continue;
2318
2319 auto CandTy = Ty->getScalarType();
2320 if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() ||
2321 S.endOffset() != P.endOffset())) {
2322 DeferredTys.insert(X: Ty);
2323 continue;
2324 }
2325
2326 LoadStoreTys.insert(X: Ty);
2327 // Consider any loads or stores that are the exact size of the slice.
2328 if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2329 CheckCandidateType(Ty);
2330 }
2331
2332 SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
2333 if (auto *VTy = createAndCheckVectorTypesForPromotion(
2334 OtherTys&: LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2335 CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2336 HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2337 return VTy;
2338
2339 CandidateTys.clear();
2340 return createAndCheckVectorTypesForPromotion(
2341 OtherTys&: DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2342 HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2343 CommonVecPtrTy, VScale);
2344}
2345
2346/// Test whether a slice of an alloca is valid for integer widening.
2347///
2348/// This implements the necessary checking for the \c isIntegerWideningViable
2349/// test below on a single slice of the alloca.
2350static bool isIntegerWideningViableForSlice(const Slice &S,
2351 uint64_t AllocBeginOffset,
2352 Type *AllocaTy,
2353 const DataLayout &DL,
2354 bool &WholeAllocaOp) {
2355 uint64_t Size = DL.getTypeStoreSize(Ty: AllocaTy).getFixedValue();
2356
2357 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2358 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2359
2360 Use *U = S.getUse();
2361
2362 // Lifetime intrinsics operate over the whole alloca whose sizes are usually
2363 // larger than other load/store slices (RelEnd > Size). But lifetime are
2364 // always promotable and should not impact other slices' promotability of the
2365 // partition.
2366 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2367 if (II->isLifetimeStartOrEnd() || II->isDroppable())
2368 return true;
2369 }
2370
2371 // We can't reasonably handle cases where the load or store extends past
2372 // the end of the alloca's type and into its padding.
2373 if (RelEnd > Size)
2374 return false;
2375
2376 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2377 if (LI->isVolatile())
2378 return false;
2379 // We can't handle loads that extend past the allocated memory.
2380 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
2381 if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
2382 return false;
2383 // So far, AllocaSliceRewriter does not support widening split slice tails
2384 // in rewriteIntegerLoad.
2385 if (S.beginOffset() < AllocBeginOffset)
2386 return false;
2387 // Note that we don't count vector loads or stores as whole-alloca
2388 // operations which enable integer widening because we would prefer to use
2389 // vector widening instead.
2390 if (!isa<VectorType>(Val: LI->getType()) && RelBegin == 0 && RelEnd == Size)
2391 WholeAllocaOp = true;
2392 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: LI->getType())) {
2393 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2394 return false;
2395 } else if (RelBegin != 0 || RelEnd != Size ||
2396 !canConvertValue(DL, OldTy: AllocaTy, NewTy: LI->getType())) {
2397 // Non-integer loads need to be convertible from the alloca type so that
2398 // they are promotable.
2399 return false;
2400 }
2401 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2402 Type *ValueTy = SI->getValueOperand()->getType();
2403 if (SI->isVolatile())
2404 return false;
2405 // We can't handle stores that extend past the allocated memory.
2406 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValueTy);
2407 if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
2408 return false;
2409 // So far, AllocaSliceRewriter does not support widening split slice tails
2410 // in rewriteIntegerStore.
2411 if (S.beginOffset() < AllocBeginOffset)
2412 return false;
2413 // Note that we don't count vector loads or stores as whole-alloca
2414 // operations which enable integer widening because we would prefer to use
2415 // vector widening instead.
2416 if (!isa<VectorType>(Val: ValueTy) && RelBegin == 0 && RelEnd == Size)
2417 WholeAllocaOp = true;
2418 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: ValueTy)) {
2419 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2420 return false;
2421 } else if (RelBegin != 0 || RelEnd != Size ||
2422 !canConvertValue(DL, OldTy: ValueTy, NewTy: AllocaTy)) {
2423 // Non-integer stores need to be convertible to the alloca type so that
2424 // they are promotable.
2425 return false;
2426 }
2427 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2428 if (MI->isVolatile() || !isa<Constant>(Val: MI->getLength()))
2429 return false;
2430 if (!S.isSplittable())
2431 return false; // Skip any unsplittable intrinsics.
2432 } else {
2433 return false;
2434 }
2435
2436 return true;
2437}
2438
2439/// Test whether the given alloca partition's integer operations can be
2440/// widened to promotable ones.
2441///
2442/// This is a quick test to check whether we can rewrite the integer loads and
2443/// stores to a particular alloca into wider loads and stores and be able to
2444/// promote the resulting alloca.
2445static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2446 const DataLayout &DL) {
2447 uint64_t SizeInBits = DL.getTypeSizeInBits(Ty: AllocaTy).getFixedValue();
2448 // Don't create integer types larger than the maximum bitwidth.
2449 if (SizeInBits > IntegerType::MAX_INT_BITS)
2450 return false;
2451
2452 // Don't try to handle allocas with bit-padding.
2453 if (SizeInBits != DL.getTypeStoreSizeInBits(Ty: AllocaTy).getFixedValue())
2454 return false;
2455
2456 // We need to ensure that an integer type with the appropriate bitwidth can
2457 // be converted to the alloca type, whatever that is. We don't want to force
2458 // the alloca itself to have an integer type if there is a more suitable one.
2459 Type *IntTy = Type::getIntNTy(C&: AllocaTy->getContext(), N: SizeInBits);
2460 if (!canConvertValue(DL, OldTy: AllocaTy, NewTy: IntTy) ||
2461 !canConvertValue(DL, OldTy: IntTy, NewTy: AllocaTy))
2462 return false;
2463
2464 // While examining uses, we ensure that the alloca has a covering load or
2465 // store. We don't want to widen the integer operations only to fail to
2466 // promote due to some other unsplittable entry (which we may make splittable
2467 // later). However, if there are only splittable uses, go ahead and assume
2468 // that we cover the alloca.
2469 // FIXME: We shouldn't consider split slices that happen to start in the
2470 // partition here...
2471 bool WholeAllocaOp = P.empty() && DL.isLegalInteger(Width: SizeInBits);
2472
2473 for (const Slice &S : P)
2474 if (!isIntegerWideningViableForSlice(S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2475 WholeAllocaOp))
2476 return false;
2477
2478 for (const Slice *S : P.splitSliceTails())
2479 if (!isIntegerWideningViableForSlice(S: *S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2480 WholeAllocaOp))
2481 return false;
2482
2483 return WholeAllocaOp;
2484}
2485
2486static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
2487 IntegerType *Ty, uint64_t Offset,
2488 const Twine &Name) {
2489 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2490 IntegerType *IntTy = cast<IntegerType>(Val: V->getType());
2491 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2492 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2493 "Element extends past full value");
2494 uint64_t ShAmt = 8 * Offset;
2495 if (DL.isBigEndian())
2496 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2497 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2498 if (ShAmt) {
2499 V = IRB.CreateLShr(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2500 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2501 }
2502 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2503 "Cannot extract to a larger integer!");
2504 if (Ty != IntTy) {
2505 V = IRB.CreateTrunc(V, DestTy: Ty, Name: Name + ".trunc");
2506 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2507 }
2508 return V;
2509}
2510
2511static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2512 Value *V, uint64_t Offset, const Twine &Name) {
2513 IntegerType *IntTy = cast<IntegerType>(Val: Old->getType());
2514 IntegerType *Ty = cast<IntegerType>(Val: V->getType());
2515 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2516 "Cannot insert a larger integer!");
2517 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2518 if (Ty != IntTy) {
2519 V = IRB.CreateZExt(V, DestTy: IntTy, Name: Name + ".ext");
2520 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2521 }
2522 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2523 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2524 "Element store outside of alloca store");
2525 uint64_t ShAmt = 8 * Offset;
2526 if (DL.isBigEndian())
2527 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2528 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2529 if (ShAmt) {
2530 V = IRB.CreateShl(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2531 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2532 }
2533
2534 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2535 APInt Mask = ~Ty->getMask().zext(width: IntTy->getBitWidth()).shl(shiftAmt: ShAmt);
2536 Old = IRB.CreateAnd(LHS: Old, RHS: Mask, Name: Name + ".mask");
2537 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2538 V = IRB.CreateOr(LHS: Old, RHS: V, Name: Name + ".insert");
2539 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2540 }
2541 return V;
2542}
2543
2544static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2545 unsigned EndIndex, const Twine &Name) {
2546 auto *VecTy = cast<FixedVectorType>(Val: V->getType());
2547 unsigned NumElements = EndIndex - BeginIndex;
2548 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2549
2550 if (NumElements == VecTy->getNumElements())
2551 return V;
2552
2553 if (NumElements == 1) {
2554 V = IRB.CreateExtractElement(Vec: V, Idx: BeginIndex, Name: Name + ".extract");
2555 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2556 return V;
2557 }
2558
2559 auto Mask = llvm::to_vector<8>(Range: llvm::seq<int>(Begin: BeginIndex, End: EndIndex));
2560 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".extract");
2561 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2562 return V;
2563}
2564
2565static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
2566 unsigned BeginIndex, const Twine &Name) {
2567 VectorType *VecTy = cast<VectorType>(Val: Old->getType());
2568 assert(VecTy && "Can only insert a vector into a vector");
2569
2570 VectorType *Ty = dyn_cast<VectorType>(Val: V->getType());
2571 if (!Ty) {
2572 // Single element to insert.
2573 V = IRB.CreateInsertElement(Vec: Old, NewElt: V, Idx: BeginIndex, Name: Name + ".insert");
2574 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2575 return V;
2576 }
2577
2578 unsigned NumSubElements = cast<FixedVectorType>(Val: Ty)->getNumElements();
2579 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2580
2581 assert(NumSubElements <= NumElements && "Too many elements!");
2582 if (NumSubElements == NumElements) {
2583 assert(V->getType() == VecTy && "Vector type mismatch");
2584 return V;
2585 }
2586 unsigned EndIndex = BeginIndex + NumSubElements;
2587
2588 // When inserting a smaller vector into the larger to store, we first
2589 // use a shuffle vector to widen it with undef elements, and then
2590 // a second shuffle vector to select between the loaded vector and the
2591 // incoming vector.
2592 SmallVector<int, 8> Mask;
2593 Mask.reserve(N: NumElements);
2594 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2595 if (Idx >= BeginIndex && Idx < EndIndex)
2596 Mask.push_back(Elt: Idx - BeginIndex);
2597 else
2598 Mask.push_back(Elt: -1);
2599 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".expand");
2600 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2601
2602 Mask.clear();
2603 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2604 if (Idx >= BeginIndex && Idx < EndIndex)
2605 Mask.push_back(Elt: Idx);
2606 else
2607 Mask.push_back(Elt: Idx + NumElements);
2608 V = IRB.CreateShuffleVector(V1: V, V2: Old, Mask, Name: Name + "blend");
2609 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2610 return V;
2611}
2612
2613/// This function takes two vector values and combines them into a single vector
2614/// by concatenating their elements. The function handles:
2615///
2616/// 1. Element type mismatch: If either vector's element type differs from
2617/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2618/// preserving the total bit width (adjusting the number of elements
2619/// accordingly).
2620///
2621/// 2. Size mismatch: After transforming the vectors to have the desired element
2622/// type, if the two vectors have different numbers of elements, the smaller
2623/// vector is extended with poison values to match the size of the larger
2624/// vector before concatenation.
2625///
2626/// 3. Concatenation: The vectors are merged using a shuffle operation that
2627/// places all elements of V0 first, followed by all elements of V1.
2628///
2629/// \param V0 The first vector to merge (must be a vector type)
2630/// \param V1 The second vector to merge (must be a vector type)
2631/// \param DL The data layout for size calculations
2632/// \param NewAIEltTy The desired element type for the result vector
2633/// \param Builder IRBuilder for creating new instructions
2634/// \return A new vector containing all elements from V0 followed by all
2635/// elements from V1
2636static Value *mergeTwoVectors(Value *V0, Value *V1, const DataLayout &DL,
2637 Type *NewAIEltTy, IRBuilder<> &Builder) {
2638 // V0 and V1 are vectors
2639 // Create a new vector type with combined elements
2640 // Use ShuffleVector to concatenate the vectors
2641 auto *VecType0 = cast<FixedVectorType>(Val: V0->getType());
2642 auto *VecType1 = cast<FixedVectorType>(Val: V1->getType());
2643
2644 // If V0/V1 element types are different from NewAllocaElementType,
2645 // we need to introduce bitcasts before merging them
2646 auto BitcastIfNeeded = [&](Value *&V, FixedVectorType *&VecType,
2647 const char *DebugName) {
2648 Type *EltType = VecType->getElementType();
2649 if (EltType != NewAIEltTy) {
2650 // Calculate new number of elements to maintain same bit width
2651 unsigned TotalBits =
2652 VecType->getNumElements() * DL.getTypeSizeInBits(Ty: EltType);
2653 unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(Ty: NewAIEltTy);
2654
2655 auto *NewVecType = FixedVectorType::get(ElementType: NewAIEltTy, NumElts: NewNumElts);
2656 V = Builder.CreateBitCast(V, DestTy: NewVecType);
2657 VecType = NewVecType;
2658 LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2659 }
2660 };
2661
2662 BitcastIfNeeded(V0, VecType0, "V0");
2663 BitcastIfNeeded(V1, VecType1, "V1");
2664
2665 unsigned NumElts0 = VecType0->getNumElements();
2666 unsigned NumElts1 = VecType1->getNumElements();
2667
2668 SmallVector<int, 16> ShuffleMask;
2669
2670 if (NumElts0 == NumElts1) {
2671 for (unsigned i = 0; i < NumElts0 + NumElts1; ++i)
2672 ShuffleMask.push_back(Elt: i);
2673 } else {
2674 // If two vectors have different sizes, we need to extend
2675 // the smaller vector to the size of the larger vector.
2676 unsigned SmallSize = std::min(a: NumElts0, b: NumElts1);
2677 unsigned LargeSize = std::max(a: NumElts0, b: NumElts1);
2678 bool IsV0Smaller = NumElts0 < NumElts1;
2679 Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2680 SmallVector<int, 16> ExtendMask;
2681 for (unsigned i = 0; i < SmallSize; ++i)
2682 ExtendMask.push_back(Elt: i);
2683 for (unsigned i = SmallSize; i < LargeSize; ++i)
2684 ExtendMask.push_back(Elt: PoisonMaskElem);
2685 ExtendedVec = Builder.CreateShuffleVector(
2686 V1: ExtendedVec, V2: PoisonValue::get(T: ExtendedVec->getType()), Mask: ExtendMask);
2687 LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2688 for (unsigned i = 0; i < NumElts0; ++i)
2689 ShuffleMask.push_back(Elt: i);
2690 for (unsigned i = 0; i < NumElts1; ++i)
2691 ShuffleMask.push_back(Elt: LargeSize + i);
2692 }
2693
2694 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
2695}
2696
2697namespace {
2698
2699/// Visitor to rewrite instructions using p particular slice of an alloca
2700/// to use a new alloca.
2701///
2702/// Also implements the rewriting to vector-based accesses when the partition
2703/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2704/// lives here.
2705class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2706 // Befriend the base class so it can delegate to private visit methods.
2707 friend class InstVisitor<AllocaSliceRewriter, bool>;
2708
2709 using Base = InstVisitor<AllocaSliceRewriter, bool>;
2710
2711 const DataLayout &DL;
2712 AllocaSlices &AS;
2713 SROA &Pass;
2714 AllocaInst &OldAI, &NewAI;
2715 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2716 Type *NewAllocaTy;
2717
2718 // This is a convenience and flag variable that will be null unless the new
2719 // alloca's integer operations should be widened to this integer type due to
2720 // passing isIntegerWideningViable above. If it is non-null, the desired
2721 // integer type will be stored here for easy access during rewriting.
2722 IntegerType *IntTy;
2723
2724 // If we are rewriting an alloca partition which can be written as pure
2725 // vector operations, we stash extra information here. When VecTy is
2726 // non-null, we have some strict guarantees about the rewritten alloca:
2727 // - The new alloca is exactly the size of the vector type here.
2728 // - The accesses all either map to the entire vector or to a single
2729 // element.
2730 // - The set of accessing instructions is only one of those handled above
2731 // in isVectorPromotionViable. Generally these are the same access kinds
2732 // which are promotable via mem2reg.
2733 VectorType *VecTy;
2734 Type *ElementTy;
2735 uint64_t ElementSize;
2736
2737 // The original offset of the slice currently being rewritten relative to
2738 // the original alloca.
2739 uint64_t BeginOffset = 0;
2740 uint64_t EndOffset = 0;
2741
2742 // The new offsets of the slice currently being rewritten relative to the
2743 // original alloca.
2744 uint64_t NewBeginOffset = 0, NewEndOffset = 0;
2745
2746 uint64_t SliceSize = 0;
2747 bool IsSplittable = false;
2748 bool IsSplit = false;
2749 Use *OldUse = nullptr;
2750 Instruction *OldPtr = nullptr;
2751
2752 // Track post-rewrite users which are PHI nodes and Selects.
2753 SmallSetVector<PHINode *, 8> &PHIUsers;
2754 SmallSetVector<SelectInst *, 8> &SelectUsers;
2755
2756 // Utility IR builder, whose name prefix is setup for each visited use, and
2757 // the insertion point is set to point to the user.
2758 IRBuilderTy IRB;
2759
2760 // Return the new alloca, addrspacecasted if required to avoid changing the
2761 // addrspace of a volatile access.
2762 Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) {
2763 if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace())
2764 return &NewAI;
2765
2766 Type *AccessTy = IRB.getPtrTy(AddrSpace);
2767 return IRB.CreateAddrSpaceCast(V: &NewAI, DestTy: AccessTy);
2768 }
2769
2770public:
2771 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2772 AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2773 uint64_t NewAllocaBeginOffset,
2774 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2775 VectorType *PromotableVecTy,
2776 SmallSetVector<PHINode *, 8> &PHIUsers,
2777 SmallSetVector<SelectInst *, 8> &SelectUsers)
2778 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2779 NewAllocaBeginOffset(NewAllocaBeginOffset),
2780 NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2781 IntTy(IsIntegerPromotable
2782 ? Type::getIntNTy(
2783 C&: NewAI.getContext(),
2784 N: DL.getTypeSizeInBits(Ty: NewAllocaTy).getFixedValue())
2785 : nullptr),
2786 VecTy(PromotableVecTy),
2787 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2788 ElementSize(VecTy ? DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8
2789 : 0),
2790 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2791 IRB(NewAI.getContext(), ConstantFolder()) {
2792 if (VecTy) {
2793 assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
2794 "Only multiple-of-8 sized vector elements are viable");
2795 ++NumVectorized;
2796 }
2797 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
2798 }
2799
2800 bool visit(AllocaSlices::const_iterator I) {
2801 bool CanSROA = true;
2802 BeginOffset = I->beginOffset();
2803 EndOffset = I->endOffset();
2804 IsSplittable = I->isSplittable();
2805 IsSplit =
2806 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
2807 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2808 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2809 LLVM_DEBUG(dbgs() << "\n");
2810
2811 // Compute the intersecting offset range.
2812 assert(BeginOffset < NewAllocaEndOffset);
2813 assert(EndOffset > NewAllocaBeginOffset);
2814 NewBeginOffset = std::max(a: BeginOffset, b: NewAllocaBeginOffset);
2815 NewEndOffset = std::min(a: EndOffset, b: NewAllocaEndOffset);
2816
2817 SliceSize = NewEndOffset - NewBeginOffset;
2818 LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2819 << ") NewBegin:(" << NewBeginOffset << ", "
2820 << NewEndOffset << ") NewAllocaBegin:("
2821 << NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2822 << ")\n");
2823 assert(IsSplit || NewBeginOffset == BeginOffset);
2824 OldUse = I->getUse();
2825 OldPtr = cast<Instruction>(Val: OldUse->get());
2826
2827 Instruction *OldUserI = cast<Instruction>(Val: OldUse->getUser());
2828 IRB.SetInsertPoint(OldUserI);
2829 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2830 // Avoid materializing the name prefix when it is discarded anyway.
2831 if (!IRB.getContext().shouldDiscardValueNames())
2832 IRB.getInserter().SetNamePrefix(Twine(NewAI.getName()) + "." +
2833 Twine(BeginOffset) + ".");
2834
2835 CanSROA &= visit(I: cast<Instruction>(Val: OldUse->getUser()));
2836 if (VecTy || IntTy)
2837 assert(CanSROA);
2838 return CanSROA;
2839 }
2840
2841 /// Attempts to rewrite a partition using tree-structured merge optimization.
2842 ///
2843 /// This function handles two patterns. Both produce an O(log n) tree of
2844 /// shufflevectors in place of the linear expand+blend chain that SROA would
2845 /// otherwise emit for each partial store.
2846 ///
2847 /// Pattern 1 (stores-only):
2848 /// Multiple non-overlapping partial stores completely fill the alloca
2849 /// and there is exactly one full-width load coming after the stores.
2850 /// The stores are tree-merged into a single vector and stored once.
2851 ///
2852 /// Example transformation:
2853 /// Before: (stores do not have to be in order)
2854 /// %alloca = alloca <8 x float>
2855 /// store <2 x float> %val0, ptr %alloca ; offset 0-1
2856 /// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2857 /// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2858 /// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2859 /// %r = load <8 x float>, ptr %alloca
2860 ///
2861 /// After: tree of shufflevectors producing <8 x float> directly.
2862 ///
2863 /// Pattern 2 (init + RMW, possibly multi-round):
2864 /// A single full-width init store, followed by partial loads and
2865 /// partial stores that read-modify-write the alloca one or more
2866 /// times, optionally followed by a full-width load. The only
2867 /// structural requirement is that the distinct [begin, end) ranges
2868 /// touched by the partial loads and stores, taken together, tile
2869 /// the alloca disjointly.
2870 ///
2871 /// We keep a map from each slice range to the SSA value that
2872 /// currently lives there, `SliceValues[r] -> Value*`:
2873 /// - initialize each entry to the corresponding piece of the
2874 /// init store's value (via a shufflevector picking the
2875 /// range's elements out of the init value),
2876 /// - walk partial loads and stores in block order,
2877 /// - for a partial load at range r: RAUW with `SliceValues[r]`,
2878 /// - for a partial store at range r: update `SliceValues[r]` to
2879 /// the stored value and drop the store.
2880 /// At the end, the final `SliceValues[r]` entries are tree-merged
2881 /// (in range order) into a single store to the alloca, and the
2882 /// optional full-width load is replaced by a load of the alloca.
2883 ///
2884 /// Because the ranges are disjoint by construction, a store at one
2885 /// range cannot affect another range's tracked value, so a single
2886 /// block-order walk correctly tracks the memory state at each
2887 /// range. The algorithm handles multi-round RMW, partial loads
2888 /// and stores interleaved in any order, read-only slices (the
2889 /// tracked value stays at the init extract), and write-only
2890 /// slices (the tracked value never flows into a load).
2891 ///
2892 /// \param P The partition to analyze and potentially rewrite
2893 /// \return An optional vector of values that were deleted during the
2894 /// rewrite, or std::nullopt if the partition cannot be optimized.
2895 std::optional<SmallVector<Value *, 4>>
2896 rewriteTreeStructuredMerge(Partition &P) {
2897 // No tail slices that overlap with the partition
2898 if (P.splitSliceTails().size() > 0)
2899 return std::nullopt;
2900
2901 // Structure to hold store information
2902 struct StoreInfo {
2903 StoreInst *Store;
2904 uint64_t BeginOffset;
2905 uint64_t EndOffset;
2906 Value *StoredValue;
2907 StoreInfo(StoreInst *SI, uint64_t Begin, uint64_t End, Value *Val)
2908 : Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
2909 };
2910 struct LoadInfo {
2911 LoadInst *Load;
2912 uint64_t BeginOffset;
2913 uint64_t EndOffset;
2914 };
2915
2916 SmallVector<StoreInfo, 4> StoreInfos; // partial stores only
2917 SmallVector<LoadInfo, 4> LoadInfos; // partial loads only
2918 LoadInst *FullLoad = nullptr; // optional full-width load
2919 StoreInst *InitStore = nullptr; // optional full-width init store
2920
2921 // If the new alloca is a fixed vector type, we use its element type as the
2922 // allocated element type, otherwise we use i8 as the allocated element
2923 Type *AllocatedEltTy =
2924 isa<FixedVectorType>(Val: NewAllocaTy)
2925 ? cast<FixedVectorType>(Val: NewAllocaTy)->getElementType()
2926 : Type::getInt8Ty(C&: NewAI.getContext());
2927 unsigned AllocatedEltTySize = DL.getTypeSizeInBits(Ty: AllocatedEltTy);
2928
2929 // Helper to check if a type is
2930 // 1. A fixed vector type
2931 // 2. The element type is not a pointer
2932 // 3. The element type size is byte-aligned
2933 // We only handle the cases that the ld/st meet these conditions
2934 auto IsTypeValidForTreeStructuredMerge = [&](Type *Ty) -> bool {
2935 auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: Ty);
2936 return FixedVecTy &&
2937 DL.getTypeSizeInBits(Ty: FixedVecTy->getElementType()) % 8 == 0 &&
2938 !FixedVecTy->getElementType()->isPointerTy();
2939 };
2940
2941 for (Slice &S : P) {
2942 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
2943 // A "full-width" slice spans the entire alloca; it's either the single
2944 // init store (Pattern 2) or the single final load (both patterns).
2945 bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
2946 S.endOffset() == NewAllocaEndOffset);
2947 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
2948 // Only handle simple (non-volatile, non-atomic) loads.
2949 if (!LI->isSimple() ||
2950 !IsTypeValidForTreeStructuredMerge(LI->getType()))
2951 return std::nullopt;
2952 if (IsFullWidth) {
2953 // We accept at most one full-width load (the "final" load, after
2954 // all the partial stores).
2955 if (FullLoad)
2956 return std::nullopt;
2957 FullLoad = LI;
2958 } else {
2959 // Partial load (RMW pattern only).
2960 LoadInfos.push_back(Elt: {.Load: LI, .BeginOffset: S.beginOffset(), .EndOffset: S.endOffset()});
2961 }
2962 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
2963 // Do not handle the case if
2964 // 1. The store does not meet the conditions in the helper function
2965 // 2. The store is not simple — we drop stores as part of the
2966 // rewrite, so volatile stores (which must be kept) and atomic
2967 // stores (which carry memory-ordering semantics) are unsound
2968 // to replace with SSA bookkeeping.
2969 // 3. The total store size is not a multiple of the allocated
2970 // element type size (required so the tree merge can produce a
2971 // vector whose element type matches the alloca).
2972 if (!SI->isSimple() || !IsTypeValidForTreeStructuredMerge(
2973 SI->getValueOperand()->getType()))
2974 return std::nullopt;
2975 auto *StVecTy = cast<FixedVectorType>(Val: SI->getValueOperand()->getType());
2976 unsigned NumElts = StVecTy->getNumElements();
2977 unsigned EltSize = DL.getTypeSizeInBits(Ty: StVecTy->getElementType());
2978 if (NumElts * EltSize % AllocatedEltTySize != 0)
2979 return std::nullopt;
2980 if (IsFullWidth) {
2981 // At most one full-width store is allowed — it's the init store
2982 // for the RMW pattern.
2983 if (InitStore)
2984 return std::nullopt;
2985 InitStore = SI;
2986 } else {
2987 StoreInfos.emplace_back(Args&: SI, Args: S.beginOffset(), Args: S.endOffset(),
2988 Args: SI->getValueOperand());
2989 }
2990 } else {
2991 // If we have instructions other than load and store, we cannot do
2992 // the tree structured merge.
2993 return std::nullopt;
2994 }
2995 }
2996
2997 // Need at least two partial stores to benefit from tree-merging; a
2998 // single store is already optimal as-is. This applies to both patterns
2999 // below, so check it before classifying.
3000 if (StoreInfos.size() < 2)
3001 return std::nullopt;
3002
3003 // Classify the pattern by looking at what we collected:
3004 // Pattern 1 (stores-only): only partial stores + exactly one full load.
3005 // Pattern 2 (RMW): one full init store + partial loads + partial stores
3006 // (+ optional full final load). RMW also needs VecTy to be set
3007 // because we use getIndex() to convert byte offsets to element
3008 // indices, which requires a promoted vector alloca.
3009 bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
3010 bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
3011 if (!IsRMWPattern && !IsStoresOnlyPattern)
3012 return std::nullopt;
3013
3014 // All partial stores must live in the same basic block — the tree merge
3015 // is built in a single BB using block-order ordering (comesBefore).
3016 BasicBlock *StoreBB = StoreInfos[0].Store->getParent();
3017 for (auto &Info : StoreInfos)
3018 if (Info.Store->getParent() != StoreBB)
3019 return std::nullopt;
3020
3021 SmallVector<Value *, 4> DeletedValues;
3022
3023 // Helper: pairwise tree-merge a list of vectors into a single vector.
3024 // At each iteration we merge each adjacent pair via mergeTwoVectors,
3025 // collect the merged values into Next, and (if Vals had odd length)
3026 // carry the trailing element through unchanged. Loop until one value
3027 // remains — the fully-merged vector.
3028 auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3029 IRBuilder<> &B) -> Value * {
3030 LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3031 while (Vals.size() > 1) {
3032 SmallVector<Value *, 8> Next;
3033 for (unsigned I = 0, E = Vals.size(); I + 1 < E; I += 2) {
3034 Value *M =
3035 mergeTwoVectors(V0: Vals[I], V1: Vals[I + 1], DL, NewAIEltTy: AllocatedEltTy, Builder&: B);
3036 LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3037 Next.push_back(Elt: M);
3038 }
3039 if (Vals.size() % 2 == 1)
3040 Next.push_back(Elt: Vals.back());
3041 Vals = std::move(Next);
3042 }
3043 return Vals[0];
3044 };
3045
3046 // Replace a full-width load with a load of the freshly-merged alloca.
3047 // The merge stored a value of type Merged->getType() into NewAI; we load
3048 // that same type back so every access to NewAI stays consistently typed
3049 // (otherwise the alloca is no longer promotable).
3050 auto ReplaceFullLoad = [&](LoadInst *LoadToReplace, Value *Merged) {
3051 IRBuilder<> LoadBuilder(LoadToReplace);
3052 Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3053 Ty: Merged->getType(), Ptr: &NewAI, Align: getSliceAlign(),
3054 isVolatile: LoadToReplace->isVolatile(),
3055 Name: LoadToReplace->getName() + ".sroa.new.load");
3056 if (NewLoad->getType() != LoadToReplace->getType())
3057 NewLoad = LoadBuilder.CreateBitCast(V: NewLoad, DestTy: LoadToReplace->getType());
3058 LoadToReplace->replaceAllUsesWith(V: NewLoad);
3059 DeletedValues.push_back(Elt: LoadToReplace);
3060 };
3061
3062 if (IsStoresOnlyPattern) {
3063 // Stores should not overlap and should cover the whole alloca.
3064 // Sort by begin offset to verify this with a single linear scan.
3065 llvm::sort(C&: StoreInfos, Comp: [](const StoreInfo &A, const StoreInfo &B) {
3066 return A.BeginOffset < B.BeginOffset;
3067 });
3068 // Check for gap or overlap: each begin offset must equal the previous
3069 // end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3070 // NewAllocaEndOffset) exactly.
3071 uint64_t Expected = NewAllocaBeginOffset;
3072 for (auto &Info : StoreInfos) {
3073 if (Info.BeginOffset != Expected)
3074 return std::nullopt;
3075 Expected = Info.EndOffset;
3076 }
3077 // Stores cover the entire alloca (no trailing gap either).
3078 if (Expected != NewAllocaEndOffset)
3079 return std::nullopt;
3080
3081 // The load should not be in the middle of the stores.
3082 // Note:
3083 // If the load is in a different basic block from the stores, we can
3084 // still do the tree-structured merge. We don't have store->load
3085 // forwarding here — the merged vector is stored back to NewAI and
3086 // the new load loads from NewAI. The forwarding will be handled
3087 // later when NewAI is promoted.
3088 BasicBlock *LoadBB = FullLoad->getParent();
3089 if (LoadBB == StoreBB) {
3090 for (auto &Info : StoreInfos)
3091 if (!Info.Store->comesBefore(Other: FullLoad))
3092 return std::nullopt;
3093 }
3094
3095 LLVM_DEBUG({
3096 dbgs() << "Tree structured merge rewrite (stores-only):\n";
3097 dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3098 for (auto [I, Info] : enumerate(StoreInfos)) {
3099 dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3100 << Info.EndOffset << ") \tStore: " << *Info.Store
3101 << "\tValue: " << *Info.StoredValue << "\n";
3102 }
3103 });
3104
3105 // StoreInfos is sorted by offset, not by block order. Anchoring to
3106 // StoreInfos.back().Store (last by offset) can place shuffles before
3107 // operands that appear later in the block (invalid SSA). Insert before
3108 // FullLoad when it shares the store block (after all stores, before
3109 // any later IR in that block). Otherwise insert before the store
3110 // block's terminator so the merge runs after every store and any
3111 // trailing instructions in that block.
3112 IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(Val: FullLoad)
3113 : StoreBB->getTerminator());
3114 SmallVector<Value *, 8> Vals;
3115 for (const auto &Info : StoreInfos) {
3116 DeletedValues.push_back(Elt: Info.Store);
3117 Vals.push_back(Elt: Info.StoredValue);
3118 }
3119 // Merge all stored values and store the merged value into the alloca.
3120 Value *Merged = TreeMerge(Vals, Builder);
3121 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3122
3123 // Replace the original load with a load of the newly-merged alloca.
3124 ReplaceFullLoad(FullLoad, Merged);
3125 return DeletedValues;
3126 }
3127
3128 // RMW pattern handling starts from here.
3129 // Like StoreBB above: keep the init store, all partial loads and all
3130 // partial stores in one basic block so we can reason about ordering
3131 // with comesBefore and build SSA without PHIs.
3132 if (InitStore->getParent() != StoreBB)
3133 return std::nullopt;
3134 if (any_of(Range&: LoadInfos, P: [&](const LoadInfo &I) {
3135 return I.Load->getParent() != StoreBB;
3136 }))
3137 return std::nullopt;
3138 // FullLoad (if any) is allowed to live in a different basic block. See
3139 // the note on the stores-only path: we don't do store->load forwarding
3140 // directly — the merged vector is stored to NewAI and the new load
3141 // loads from NewAI, so cross-BB ordering is resolved later when NewAI
3142 // is promoted.
3143
3144 // Collect the combined partial-load/partial-store accesses sorted
3145 // by block order. Used both for ordering checks and for the rewrite
3146 // walk below.
3147 struct Access {
3148 Instruction *Inst;
3149 uint64_t BeginOffset, EndOffset;
3150 bool IsStore;
3151 };
3152 SmallVector<Access, 16> Accesses;
3153 Accesses.reserve(N: LoadInfos.size() + StoreInfos.size());
3154 for (const auto &L : LoadInfos)
3155 Accesses.push_back(Elt: {.Inst: L.Load, .BeginOffset: L.BeginOffset, .EndOffset: L.EndOffset, .IsStore: false});
3156 for (const auto &S : StoreInfos)
3157 Accesses.push_back(Elt: {.Inst: S.Store, .BeginOffset: S.BeginOffset, .EndOffset: S.EndOffset, .IsStore: true});
3158 llvm::sort(C&: Accesses, Comp: [](const Access &A, const Access &B) {
3159 return A.Inst->comesBefore(Other: B.Inst);
3160 });
3161
3162 // Ordering constraint 1: InitStore must come before every partial
3163 // access — they read/write the RMW state initialised by InitStore.
3164 // Accesses is sorted by block order, so the first element is the
3165 // earliest; checking it is enough.
3166 if (!InitStore->comesBefore(Other: Accesses.front().Inst))
3167 return std::nullopt;
3168 // Ordering constraint 2: when FullLoad shares the block with the
3169 // partial accesses, it must come after every one of them — otherwise
3170 // it could read a stale value. Accesses is sorted, so the last
3171 // element is the latest; checking it is enough. If FullLoad is in
3172 // another block, mem2reg forwards the merged store to it.
3173 if (FullLoad && FullLoad->getParent() == StoreBB &&
3174 !Accesses.back().Inst->comesBefore(Other: FullLoad))
3175 return std::nullopt;
3176
3177 // Coverage check: the distinct [begin, end) ranges touched by the
3178 // partial loads and stores must tile the alloca disjointly. That is
3179 // the only precondition the per-range SliceValues tracking below
3180 // needs — a disjoint tile guarantees the entries don't alias each
3181 // other. We don't check per-range load/store counts: a range with
3182 // only loads ends with SliceValues[r] = the init extract
3183 // (contributed to the final tree-merge), and a range with only
3184 // stores ends with SliceValues[r] = its last stored value. Both are
3185 // correct.
3186 using SliceRange = std::pair<uint64_t, uint64_t>;
3187 SmallVector<SliceRange, 8> SortedRanges;
3188 SortedRanges.reserve(N: Accesses.size());
3189 for (auto &Acc : Accesses)
3190 SortedRanges.emplace_back(Args&: Acc.BeginOffset, Args&: Acc.EndOffset);
3191 llvm::sort(C&: SortedRanges);
3192 SortedRanges.erase(CS: llvm::unique(R&: SortedRanges), CE: SortedRanges.end());
3193 // Disjoint + contiguous tile of the whole alloca.
3194 uint64_t Expected = NewAllocaBeginOffset;
3195 for (auto &Range : SortedRanges) {
3196 if (Range.first != Expected)
3197 return std::nullopt;
3198 Expected = Range.second;
3199 }
3200 if (Expected != NewAllocaEndOffset)
3201 return std::nullopt;
3202
3203 LLVM_DEBUG({
3204 dbgs() << "Tree structured merge rewrite (RMW):\n";
3205 dbgs() << " Init store: " << *InitStore << "\n";
3206 if (FullLoad)
3207 dbgs() << " Final load: " << *FullLoad << "\n";
3208 dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3209 for (auto &Range : SortedRanges)
3210 dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3211 });
3212
3213 // Initialize SliceValues: one SSA value per slice range, tracking
3214 // the value the alloca currently holds at that range. Each entry
3215 // starts at the corresponding piece of the init store, obtained by
3216 // bitcasting the init value to the alloca's vector type (if needed)
3217 // and extracting the slice's sub-range.
3218 IRB.SetInsertPoint(InitStore->getNextNode());
3219 Value *InitVec = InitStore->getValueOperand();
3220 if (InitVec->getType() != NewAllocaTy)
3221 InitVec = IRB.CreateBitCast(V: InitVec, DestTy: NewAllocaTy, Name: "init.cast");
3222 DenseMap<SliceRange, Value *> SliceValues;
3223 for (auto &Range : SortedRanges) {
3224 unsigned BeginIdx = getIndex(Offset: Range.first);
3225 unsigned EndIdx = getIndex(Offset: Range.second);
3226 SliceValues[Range] = IRB.CreateShuffleVector(
3227 V: InitVec, Mask: createSequentialMask(Start: BeginIdx, NumInts: EndIdx - BeginIdx, NumUndefs: 0),
3228 Name: "init.extract");
3229 }
3230 // The init store itself becomes dead — its value is consumed via the
3231 // extracts above.
3232 DeletedValues.push_back(Elt: InitStore);
3233
3234 // Walk accesses in block order:
3235 // - partial load at range r: replace with SliceValues[r] (bitcast
3236 // if the load's type differs from the current tracked value's
3237 // type, e.g. because a previous store wrote a vector with a
3238 // different element type);
3239 // - partial store at range r: update SliceValues[r] to the stored
3240 // value and drop the store.
3241 for (auto &Acc : Accesses) {
3242 SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3243 if (!Acc.IsStore) {
3244 Value *V = SliceValues[Range];
3245 if (V->getType() != Acc.Inst->getType()) {
3246 IRB.SetInsertPoint(cast<LoadInst>(Val: Acc.Inst));
3247 V = IRB.CreateBitCast(V, DestTy: Acc.Inst->getType());
3248 }
3249 Acc.Inst->replaceAllUsesWith(V);
3250 } else {
3251 SliceValues[Range] = cast<StoreInst>(Val: Acc.Inst)->getValueOperand();
3252 }
3253 DeletedValues.push_back(Elt: Acc.Inst);
3254 }
3255
3256 // Tree-merge the final per-range values (in range order) into the
3257 // alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3258 // shares the partial-access block) or otherwise to the block's
3259 // terminator — never to a partial access, since those are queued for
3260 // deletion. Both anchors are guaranteed to dominate every SliceValues
3261 // entry: each one is either an init extract (before any access) or a
3262 // stored value defined before its (now-deleted) store.
3263 IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3264 ? cast<Instruction>(Val: FullLoad)
3265 : StoreBB->getTerminator());
3266 SmallVector<Value *, 8> Vals;
3267 for (auto &Range : SortedRanges)
3268 Vals.push_back(Elt: SliceValues[Range]);
3269 Value *Merged = TreeMerge(Vals, Builder);
3270 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3271
3272 // Replace the optional final full-width load with a load of the newly
3273 // merged alloca. Later promotion will forward the store above to it.
3274 if (FullLoad)
3275 ReplaceFullLoad(FullLoad, Merged);
3276
3277 return DeletedValues;
3278 }
3279
3280private:
3281 // Make sure the other visit overloads are visible.
3282 using Base::visit;
3283
3284 // Every instruction which can end up as a user must have a rewrite rule.
3285 bool visitInstruction(Instruction &I) {
3286 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3287 llvm_unreachable("No rewrite rule for this instruction!");
3288 }
3289
3290 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
3291 // Note that the offset computation can use BeginOffset or NewBeginOffset
3292 // interchangeably for unsplit slices.
3293 assert(IsSplit || BeginOffset == NewBeginOffset);
3294 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3295
3296 StringRef OldName = OldPtr->getName();
3297 // Skip through the last '.sroa.' component of the name.
3298 size_t LastSROAPrefix = OldName.rfind(Str: ".sroa.");
3299 if (LastSROAPrefix != StringRef::npos) {
3300 OldName = OldName.substr(Start: LastSROAPrefix + strlen(s: ".sroa."));
3301 // Look for an SROA slice index.
3302 size_t IndexEnd = OldName.find_first_not_of(Chars: "0123456789");
3303 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
3304 // Strip the index and look for the offset.
3305 OldName = OldName.substr(Start: IndexEnd + 1);
3306 size_t OffsetEnd = OldName.find_first_not_of(Chars: "0123456789");
3307 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
3308 // Strip the offset.
3309 OldName = OldName.substr(Start: OffsetEnd + 1);
3310 }
3311 }
3312 // Strip any SROA suffixes as well.
3313 OldName = OldName.substr(Start: 0, N: OldName.find(Str: ".sroa_"));
3314
3315 return getAdjustedPtr(IRB, DL, Ptr: &NewAI,
3316 Offset: APInt(DL.getIndexTypeSizeInBits(Ty: PointerTy), Offset),
3317 PointerTy, NamePrefix: Twine(OldName) + ".");
3318 }
3319
3320 /// Compute suitable alignment to access this slice of the *new*
3321 /// alloca.
3322 ///
3323 /// You can optionally pass a type to this routine and if that type's ABI
3324 /// alignment is itself suitable, this will return zero.
3325 Align getSliceAlign() {
3326 return commonAlignment(A: NewAI.getAlign(),
3327 Offset: NewBeginOffset - NewAllocaBeginOffset);
3328 }
3329
3330 unsigned getIndex(uint64_t Offset) {
3331 assert(VecTy && "Can only call getIndex when rewriting a vector");
3332 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3333 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3334 uint32_t Index = RelOffset / ElementSize;
3335 assert(Index * ElementSize == RelOffset);
3336 return Index;
3337 }
3338
3339 void deleteIfTriviallyDead(Value *V) {
3340 Instruction *I = cast<Instruction>(Val: V);
3341 if (isInstructionTriviallyDead(I))
3342 Pass.DeadInsts.push_back(Elt: I);
3343 }
3344
3345 Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3346 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3347 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3348 assert(EndIndex > BeginIndex && "Empty vector!");
3349
3350 LoadInst *Load =
3351 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3352
3353 Load->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3354 LLVMContext::MD_access_group});
3355 return extractVector(IRB, V: Load, BeginIndex, EndIndex, Name: "vec");
3356 }
3357
3358 Value *rewriteIntegerLoad(LoadInst &LI) {
3359 assert(IntTy && "We cannot insert an integer to the alloca");
3360 assert(!LI.isVolatile());
3361 Value *V =
3362 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3363 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: IntTy);
3364 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3365 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3366 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
3367 IntegerType *ExtractTy = Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8);
3368 V = extractInteger(DL, IRB, V, Ty: ExtractTy, Offset, Name: "extract");
3369 }
3370 // It is possible that the extracted type is not the load type. This
3371 // happens if there is a load past the end of the alloca, and as
3372 // a consequence the slice is narrower but still a candidate for integer
3373 // lowering. To handle this case, we just zero extend the extracted
3374 // integer.
3375 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
3376 "Can only handle an extract for an overly wide load");
3377 if (cast<IntegerType>(Val: LI.getType())->getBitWidth() > SliceSize * 8)
3378 V = IRB.CreateZExt(V, DestTy: LI.getType());
3379 return V;
3380 }
3381
3382 bool visitLoadInst(LoadInst &LI) {
3383 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3384 Value *OldOp = LI.getOperand(i_nocapture: 0);
3385 assert(OldOp == OldPtr);
3386
3387 AAMDNodes AATags = LI.getAAMetadata();
3388
3389 unsigned AS = LI.getPointerAddressSpace();
3390
3391 Type *TargetTy = IsSplit ? Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8)
3392 : LI.getType();
3393 bool IsPtrAdjusted = false;
3394 Value *V;
3395 if (VecTy) {
3396 V = rewriteVectorizedLoadInst(LI);
3397 } else if (IntTy && LI.getType()->isIntegerTy()) {
3398 V = rewriteIntegerLoad(LI);
3399 } else if (NewBeginOffset == NewAllocaBeginOffset &&
3400 NewEndOffset == NewAllocaEndOffset &&
3401 (canConvertValue(DL, OldTy: NewAllocaTy, NewTy: TargetTy) ||
3402 (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3403 DL.getTypeStoreSize(Ty: TargetTy).getFixedValue() > SliceSize &&
3404 !LI.isVolatile()))) {
3405 Value *NewPtr =
3406 getPtrToNewAI(AddrSpace: LI.getPointerAddressSpace(), IsVolatile: LI.isVolatile());
3407 LoadInst *NewLI = IRB.CreateAlignedLoad(
3408 Ty: NewAllocaTy, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3409 if (LI.isVolatile())
3410 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3411 if (NewLI->isAtomic())
3412 NewLI->setAlignment(LI.getAlign());
3413
3414 // Copy any metadata that is valid for the new load. This may require
3415 // conversion to a different kind of metadata, e.g. !nonnull might change
3416 // to !range or vice versa.
3417 copyMetadataForLoad(Dest&: *NewLI, Source: LI);
3418
3419 // Do this after copyMetadataForLoad() to preserve the TBAA shift.
3420 if (AATags)
3421 NewLI->setAAMetadata(AATags.adjustForAccess(
3422 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3423
3424 // Try to preserve nonnull metadata
3425 V = NewLI;
3426
3427 // If this is an integer load past the end of the slice (which means the
3428 // bytes outside the slice are undef or this load is dead) just forcibly
3429 // fix the integer size with correct handling of endianness.
3430 if (auto *AITy = dyn_cast<IntegerType>(Val: NewAllocaTy))
3431 if (auto *TITy = dyn_cast<IntegerType>(Val: TargetTy))
3432 if (AITy->getBitWidth() < TITy->getBitWidth()) {
3433 V = IRB.CreateZExt(V, DestTy: TITy, Name: "load.ext");
3434 if (DL.isBigEndian())
3435 V = IRB.CreateShl(LHS: V, RHS: TITy->getBitWidth() - AITy->getBitWidth(),
3436 Name: "endian_shift");
3437 }
3438 } else {
3439 Type *LTy = IRB.getPtrTy(AddrSpace: AS);
3440 LoadInst *NewLI =
3441 IRB.CreateAlignedLoad(Ty: TargetTy, Ptr: getNewAllocaSlicePtr(IRB, PointerTy: LTy),
3442 Align: getSliceAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3443
3444 if (AATags)
3445 NewLI->setAAMetadata(AATags.adjustForAccess(
3446 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3447
3448 if (LI.isVolatile())
3449 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3450 NewLI->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3451 LLVMContext::MD_access_group});
3452
3453 V = NewLI;
3454 IsPtrAdjusted = true;
3455 }
3456 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: TargetTy);
3457
3458 if (IsSplit) {
3459 assert(!LI.isVolatile());
3460 assert(LI.getType()->isIntegerTy() &&
3461 "Only integer type loads and stores are split");
3462 assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3463 "Split load isn't smaller than original load");
3464 assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3465 "Non-byte-multiple bit width");
3466 // Move the insertion point just past the load so that we can refer to it.
3467 BasicBlock::iterator LIIt = std::next(x: LI.getIterator());
3468 // Ensure the insertion point comes before any debug-info immediately
3469 // after the load, so that variable values referring to the load are
3470 // dominated by it.
3471 LIIt.setHeadBit(true);
3472 IRB.SetInsertPoint(TheBB: LI.getParent(), IP: LIIt);
3473 // Create a placeholder value with the same type as LI to use as the
3474 // basis for the new value. This allows us to replace the uses of LI with
3475 // the computed value, and then replace the placeholder with LI, leaving
3476 // LI only used for this computation.
3477 Value *Placeholder =
3478 new LoadInst(LI.getType(), PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AS)), "",
3479 false, Align(1));
3480 V = insertInteger(DL, IRB, Old: Placeholder, V, Offset: NewBeginOffset - BeginOffset,
3481 Name: "insert");
3482 LI.replaceAllUsesWith(V);
3483 Placeholder->replaceAllUsesWith(V: &LI);
3484 Placeholder->deleteValue();
3485 } else {
3486 LI.replaceAllUsesWith(V);
3487 }
3488
3489 Pass.DeadInsts.push_back(Elt: &LI);
3490 deleteIfTriviallyDead(V: OldOp);
3491 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3492 return !LI.isVolatile() && !IsPtrAdjusted;
3493 }
3494
3495 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
3496 AAMDNodes AATags) {
3497 // Capture V for the purpose of debug-info accounting once it's converted
3498 // to a vector store.
3499 Value *OrigV = V;
3500 if (V->getType() != VecTy) {
3501 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3502 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3503 assert(EndIndex > BeginIndex && "Empty vector!");
3504 unsigned NumElements = EndIndex - BeginIndex;
3505 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3506 "Too many elements!");
3507 Type *SliceTy = (NumElements == 1)
3508 ? ElementTy
3509 : FixedVectorType::get(ElementType: ElementTy, NumElts: NumElements);
3510 if (V->getType() != SliceTy)
3511 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: SliceTy);
3512
3513 // Mix in the existing elements.
3514 Value *Old =
3515 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3516 V = insertVector(IRB, Old, V, BeginIndex, Name: "vec");
3517 }
3518 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3519 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3520 LLVMContext::MD_access_group});
3521 if (AATags)
3522 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3523 AccessTy: V->getType(), DL));
3524 Pass.DeadInsts.push_back(Elt: &SI);
3525
3526 // NOTE: Careful to use OrigV rather than V.
3527 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3528 Inst: Store, Dest: Store->getPointerOperand(), Value: OrigV, DL);
3529 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3530 return true;
3531 }
3532
3533 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3534 assert(IntTy && "We cannot extract an integer from the alloca");
3535 assert(!SI.isVolatile());
3536 if (DL.getTypeSizeInBits(Ty: V->getType()).getFixedValue() !=
3537 IntTy->getBitWidth()) {
3538 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3539 Name: "oldload");
3540 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3541 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3542 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3543 V = insertInteger(DL, IRB, Old, V: SI.getValueOperand(), Offset, Name: "insert");
3544 }
3545 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3546 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3547 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3548 LLVMContext::MD_access_group});
3549 if (AATags)
3550 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3551 AccessTy: V->getType(), DL));
3552
3553 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3554 Inst: Store, Dest: Store->getPointerOperand(),
3555 Value: Store->getValueOperand(), DL);
3556
3557 Pass.DeadInsts.push_back(Elt: &SI);
3558 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3559 return true;
3560 }
3561
3562 bool visitStoreInst(StoreInst &SI) {
3563 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3564 Value *OldOp = SI.getOperand(i_nocapture: 1);
3565 assert(OldOp == OldPtr);
3566
3567 AAMDNodes AATags = SI.getAAMetadata();
3568 Value *V = SI.getValueOperand();
3569
3570 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3571 // alloca that should be re-examined after promoting this alloca.
3572 if (V->getType()->isPointerTy())
3573 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V->stripInBoundsOffsets()))
3574 Pass.PostPromotionWorklist.insert(X: AI);
3575
3576 TypeSize StoreSize = DL.getTypeStoreSize(Ty: V->getType());
3577 if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3578 assert(!SI.isVolatile());
3579 assert(V->getType()->isIntegerTy() &&
3580 "Only integer type loads and stores are split");
3581 assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3582 "Non-byte-multiple bit width");
3583 IntegerType *NarrowTy = Type::getIntNTy(C&: SI.getContext(), N: SliceSize * 8);
3584 V = extractInteger(DL, IRB, V, Ty: NarrowTy, Offset: NewBeginOffset - BeginOffset,
3585 Name: "extract");
3586 }
3587
3588 if (VecTy)
3589 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3590 if (IntTy && V->getType()->isIntegerTy())
3591 return rewriteIntegerStore(V, SI, AATags);
3592
3593 StoreInst *NewSI;
3594 if (NewBeginOffset == NewAllocaBeginOffset &&
3595 NewEndOffset == NewAllocaEndOffset &&
3596 canConvertValue(DL, OldTy: V->getType(), NewTy: NewAllocaTy)) {
3597 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3598 Value *NewPtr =
3599 getPtrToNewAI(AddrSpace: SI.getPointerAddressSpace(), IsVolatile: SI.isVolatile());
3600
3601 NewSI =
3602 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: SI.isVolatile());
3603 } else {
3604 unsigned AS = SI.getPointerAddressSpace();
3605 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: IRB.getPtrTy(AddrSpace: AS));
3606 NewSI =
3607 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: getSliceAlign(), isVolatile: SI.isVolatile());
3608 }
3609 NewSI->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3610 LLVMContext::MD_access_group});
3611 if (AATags)
3612 NewSI->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3613 AccessTy: V->getType(), DL));
3614 if (SI.isVolatile())
3615 NewSI->setAtomic(Ordering: SI.getOrdering(), SSID: SI.getSyncScopeID());
3616 if (NewSI->isAtomic())
3617 NewSI->setAlignment(SI.getAlign());
3618
3619 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3620 Inst: NewSI, Dest: NewSI->getPointerOperand(),
3621 Value: NewSI->getValueOperand(), DL);
3622
3623 Pass.DeadInsts.push_back(Elt: &SI);
3624 deleteIfTriviallyDead(V: OldOp);
3625
3626 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3627 return NewSI->getPointerOperand() == &NewAI &&
3628 NewSI->getValueOperand()->getType() == NewAllocaTy &&
3629 !SI.isVolatile();
3630 }
3631
3632 /// Compute an integer value from splatting an i8 across the given
3633 /// number of bytes.
3634 ///
3635 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3636 /// call this routine.
3637 /// FIXME: Heed the advice above.
3638 ///
3639 /// \param V The i8 value to splat.
3640 /// \param Size The number of bytes in the output (assuming i8 is one byte)
3641 Value *getIntegerSplat(Value *V, unsigned Size) {
3642 assert(Size > 0 && "Expected a positive number of bytes.");
3643 IntegerType *VTy = cast<IntegerType>(Val: V->getType());
3644 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
3645 if (Size == 1)
3646 return V;
3647
3648 Type *SplatIntTy = Type::getIntNTy(C&: VTy->getContext(), N: Size * 8);
3649 V = IRB.CreateMul(
3650 LHS: IRB.CreateZExt(V, DestTy: SplatIntTy, Name: "zext"),
3651 RHS: IRB.CreateUDiv(LHS: Constant::getAllOnesValue(Ty: SplatIntTy),
3652 RHS: IRB.CreateZExt(V: Constant::getAllOnesValue(Ty: V->getType()),
3653 DestTy: SplatIntTy)),
3654 Name: "isplat");
3655 return V;
3656 }
3657
3658 /// Compute a vector splat for a given element value.
3659 Value *getVectorSplat(Value *V, unsigned NumElements) {
3660 V = IRB.CreateVectorSplat(NumElts: NumElements, V, Name: "vsplat");
3661 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3662 return V;
3663 }
3664
3665 bool visitMemSetInst(MemSetInst &II) {
3666 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3667 assert(II.getRawDest() == OldPtr);
3668
3669 AAMDNodes AATags = II.getAAMetadata();
3670
3671 // If the memset has a variable size, it cannot be split, just adjust the
3672 // pointer to the new alloca.
3673 if (!isa<ConstantInt>(Val: II.getLength())) {
3674 assert(!IsSplit);
3675 assert(NewBeginOffset == BeginOffset);
3676 II.setDest(getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()));
3677 II.setDestAlignment(getSliceAlign());
3678 // In theory we should call migrateDebugInfo here. However, we do not
3679 // emit dbg.assign intrinsics for mem intrinsics storing through non-
3680 // constant geps, or storing a variable number of bytes.
3681 assert(at::getDVRAssignmentMarkers(&II).empty() &&
3682 "AT: Unexpected link to non-const GEP");
3683 deleteIfTriviallyDead(V: OldPtr);
3684 return false;
3685 }
3686
3687 // Record this instruction for deletion.
3688 Pass.DeadInsts.push_back(Elt: &II);
3689
3690 Type *ScalarTy = NewAllocaTy->getScalarType();
3691
3692 const bool CanContinue = [&]() {
3693 if (VecTy || IntTy)
3694 return true;
3695 if (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset)
3696 return false;
3697 // Length must be in range for FixedVectorType.
3698 auto *C = cast<ConstantInt>(Val: II.getLength());
3699 const uint64_t Len = C->getLimitedValue();
3700 if (Len > std::numeric_limits<unsigned>::max())
3701 return false;
3702 auto *Int8Ty = IntegerType::getInt8Ty(C&: NewAI.getContext());
3703 auto *SrcTy = FixedVectorType::get(ElementType: Int8Ty, NumElts: Len);
3704 return canConvertValue(DL, OldTy: SrcTy, NewTy: NewAllocaTy) &&
3705 DL.isLegalInteger(Width: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue());
3706 }();
3707
3708 // If this doesn't map cleanly onto the alloca type, and that type isn't
3709 // a single value type, just emit a memset.
3710 if (!CanContinue) {
3711 Type *SizeTy = II.getLength()->getType();
3712 unsigned Sz = NewEndOffset - NewBeginOffset;
3713 Constant *Size = ConstantInt::get(Ty: SizeTy, V: Sz);
3714 MemIntrinsic *New = cast<MemIntrinsic>(Val: IRB.CreateMemSet(
3715 Ptr: getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()), Val: II.getValue(), Size,
3716 Align: MaybeAlign(getSliceAlign()), isVolatile: II.isVolatile()));
3717 if (AATags)
3718 New->setAAMetadata(
3719 AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset, AccessSize: Sz));
3720
3721 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3722 Inst: New, Dest: New->getRawDest(), Value: nullptr, DL);
3723
3724 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3725 return false;
3726 }
3727
3728 // If we can represent this as a simple value, we have to build the actual
3729 // value to store, which requires expanding the byte present in memset to
3730 // a sensible representation for the alloca type. This is essentially
3731 // splatting the byte to a sufficiently wide integer, splatting it across
3732 // any desired vector width, and bitcasting to the final type.
3733 Value *V;
3734
3735 if (VecTy) {
3736 // If this is a memset of a vectorized alloca, insert it.
3737 assert(ElementTy == ScalarTy);
3738
3739 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3740 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3741 assert(EndIndex > BeginIndex && "Empty vector!");
3742 unsigned NumElements = EndIndex - BeginIndex;
3743 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3744 "Too many elements!");
3745
3746 Value *Splat = getIntegerSplat(
3747 V: II.getValue(), Size: DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8);
3748 Splat = IRB.CreateBitPreservingCastChain(DL, V: Splat, NewTy: ElementTy);
3749 if (NumElements > 1)
3750 Splat = getVectorSplat(V: Splat, NumElements);
3751
3752 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3753 Name: "oldload");
3754 V = insertVector(IRB, Old, V: Splat, BeginIndex, Name: "vec");
3755 } else if (IntTy) {
3756 // If this is a memset on an alloca where we can widen stores, insert the
3757 // set integer.
3758 assert(!II.isVolatile());
3759
3760 uint64_t Size = NewEndOffset - NewBeginOffset;
3761 V = getIntegerSplat(V: II.getValue(), Size);
3762
3763 if (IntTy && (NewBeginOffset != NewAllocaBeginOffset ||
3764 NewEndOffset != NewAllocaEndOffset)) {
3765 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI,
3766 Align: NewAI.getAlign(), Name: "oldload");
3767 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3768 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3769 V = insertInteger(DL, IRB, Old, V, Offset, Name: "insert");
3770 } else {
3771 assert(V->getType() == IntTy &&
3772 "Wrong type for an alloca wide integer!");
3773 }
3774 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3775 } else {
3776 // Established these invariants above.
3777 assert(NewBeginOffset == NewAllocaBeginOffset);
3778 assert(NewEndOffset == NewAllocaEndOffset);
3779
3780 V = getIntegerSplat(V: II.getValue(),
3781 Size: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue() / 8);
3782 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(Val: NewAllocaTy))
3783 V = getVectorSplat(
3784 V, NumElements: cast<FixedVectorType>(Val: AllocaVecTy)->getNumElements());
3785
3786 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3787 }
3788
3789 Value *NewPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3790 StoreInst *New =
3791 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: II.isVolatile());
3792 New->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3793 LLVMContext::MD_access_group});
3794 if (AATags)
3795 New->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3796 AccessTy: V->getType(), DL));
3797
3798 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3799 Inst: New, Dest: New->getPointerOperand(), Value: V, DL);
3800
3801 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3802 return !II.isVolatile();
3803 }
3804
3805 bool visitMemTransferInst(MemTransferInst &II) {
3806 // Rewriting of memory transfer instructions can be a bit tricky. We break
3807 // them into two categories: split intrinsics and unsplit intrinsics.
3808
3809 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3810
3811 AAMDNodes AATags = II.getAAMetadata();
3812
3813 bool IsDest = &II.getRawDestUse() == OldUse;
3814 assert((IsDest && II.getRawDest() == OldPtr) ||
3815 (!IsDest && II.getRawSource() == OldPtr));
3816
3817 Align SliceAlign = getSliceAlign();
3818 // For unsplit intrinsics, we simply modify the source and destination
3819 // pointers in place. This isn't just an optimization, it is a matter of
3820 // correctness. With unsplit intrinsics we may be dealing with transfers
3821 // within a single alloca before SROA ran, or with transfers that have
3822 // a variable length. We may also be dealing with memmove instead of
3823 // memcpy, and so simply updating the pointers is the necessary for us to
3824 // update both source and dest of a single call.
3825 if (!IsSplittable) {
3826 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3827 if (IsDest) {
3828 // Update the address component of linked dbg.assigns.
3829 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(Inst: &II)) {
3830 if (llvm::is_contained(Range: DbgAssign->location_ops(), Element: II.getDest()) ||
3831 DbgAssign->getAddress() == II.getDest())
3832 DbgAssign->replaceVariableLocationOp(OldValue: II.getDest(), NewValue: AdjustedPtr);
3833 }
3834 II.setDest(AdjustedPtr);
3835 II.setDestAlignment(SliceAlign);
3836 } else {
3837 II.setSource(AdjustedPtr);
3838 II.setSourceAlignment(SliceAlign);
3839 }
3840
3841 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3842 deleteIfTriviallyDead(V: OldPtr);
3843 return false;
3844 }
3845 // For split transfer intrinsics we have an incredibly useful assurance:
3846 // the source and destination do not reside within the same alloca, and at
3847 // least one of them does not escape. This means that we can replace
3848 // memmove with memcpy, and we don't need to worry about all manner of
3849 // downsides to splitting and transforming the operations.
3850
3851 // If this doesn't map cleanly onto the alloca type, and that type isn't
3852 // a single value type, just emit a memcpy.
3853 bool EmitMemCpy =
3854 !VecTy && !IntTy &&
3855 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
3856 SliceSize != DL.getTypeStoreSize(Ty: NewAllocaTy).getFixedValue() ||
3857 !DL.typeSizeEqualsStoreSize(Ty: NewAllocaTy) ||
3858 !NewAllocaTy->isSingleValueType());
3859
3860 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
3861 // size hasn't been shrunk based on analysis of the viable range, this is
3862 // a no-op.
3863 if (EmitMemCpy && &OldAI == &NewAI) {
3864 // Ensure the start lines up.
3865 assert(NewBeginOffset == BeginOffset);
3866
3867 // Rewrite the size as needed.
3868 if (NewEndOffset != EndOffset)
3869 II.setLength(NewEndOffset - NewBeginOffset);
3870 return false;
3871 }
3872 // Record this instruction for deletion.
3873 Pass.DeadInsts.push_back(Elt: &II);
3874
3875 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3876 // alloca that should be re-examined after rewriting this instruction.
3877 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
3878 if (AllocaInst *AI =
3879 dyn_cast<AllocaInst>(Val: OtherPtr->stripInBoundsOffsets())) {
3880 assert(AI != &OldAI && AI != &NewAI &&
3881 "Splittable transfers cannot reach the same alloca on both ends.");
3882 Pass.Worklist.insert(X: AI);
3883 }
3884
3885 Type *OtherPtrTy = OtherPtr->getType();
3886 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
3887
3888 // Compute the relative offset for the other pointer within the transfer.
3889 unsigned OffsetWidth = DL.getIndexSizeInBits(AS: OtherAS);
3890 APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
3891 Align OtherAlign =
3892 (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
3893 OtherAlign =
3894 commonAlignment(A: OtherAlign, Offset: OtherOffset.zextOrTrunc(width: 64).getZExtValue());
3895
3896 if (EmitMemCpy) {
3897 // Compute the other pointer, folding as much as possible to produce
3898 // a single, simple GEP in most cases.
3899 OtherPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
3900 NamePrefix: OtherPtr->getName() + ".");
3901
3902 Value *OurPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3903 Type *SizeTy = II.getLength()->getType();
3904 Constant *Size = ConstantInt::get(Ty: SizeTy, V: NewEndOffset - NewBeginOffset);
3905
3906 Value *DestPtr, *SrcPtr;
3907 MaybeAlign DestAlign, SrcAlign;
3908 // Note: IsDest is true iff we're copying into the new alloca slice
3909 if (IsDest) {
3910 DestPtr = OurPtr;
3911 DestAlign = SliceAlign;
3912 SrcPtr = OtherPtr;
3913 SrcAlign = OtherAlign;
3914 } else {
3915 DestPtr = OtherPtr;
3916 DestAlign = OtherAlign;
3917 SrcPtr = OurPtr;
3918 SrcAlign = SliceAlign;
3919 }
3920 CallInst *New = IRB.CreateMemCpy(Dst: DestPtr, DstAlign: DestAlign, Src: SrcPtr, SrcAlign,
3921 Size, isVolatile: II.isVolatile());
3922 if (AATags)
3923 New->setAAMetadata(AATags.shift(Offset: NewBeginOffset - BeginOffset));
3924
3925 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DestPtr->getType()), 0);
3926 if (IsDest) {
3927 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8,
3928 OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
3929 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
3930 Val: DestPtr->stripAndAccumulateConstantOffsets(
3931 DL, Offset, /*AllowNonInbounds*/ true))) {
3932 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
3933 SliceSizeInBits: SliceSize * 8, OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
3934 }
3935 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3936 return false;
3937 }
3938
3939 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
3940 NewEndOffset == NewAllocaEndOffset;
3941 uint64_t Size = NewEndOffset - NewBeginOffset;
3942 unsigned BeginIndex = VecTy ? getIndex(Offset: NewBeginOffset) : 0;
3943 unsigned EndIndex = VecTy ? getIndex(Offset: NewEndOffset) : 0;
3944 unsigned NumElements = EndIndex - BeginIndex;
3945 IntegerType *SubIntTy =
3946 IntTy ? Type::getIntNTy(C&: IntTy->getContext(), N: Size * 8) : nullptr;
3947
3948 // Reset the other pointer type to match the register type we're going to
3949 // use, but using the address space of the original other pointer.
3950 Type *OtherTy;
3951 if (VecTy && !IsWholeAlloca) {
3952 if (NumElements == 1)
3953 OtherTy = VecTy->getElementType();
3954 else
3955 OtherTy = FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: NumElements);
3956 } else if (IntTy && !IsWholeAlloca) {
3957 OtherTy = SubIntTy;
3958 } else {
3959 OtherTy = NewAllocaTy;
3960 }
3961
3962 Value *AdjPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
3963 NamePrefix: OtherPtr->getName() + ".");
3964 MaybeAlign SrcAlign = OtherAlign;
3965 MaybeAlign DstAlign = SliceAlign;
3966 if (!IsDest)
3967 std::swap(a&: SrcAlign, b&: DstAlign);
3968
3969 Value *SrcPtr;
3970 Value *DstPtr;
3971
3972 if (IsDest) {
3973 DstPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3974 SrcPtr = AdjPtr;
3975 } else {
3976 DstPtr = AdjPtr;
3977 SrcPtr = getPtrToNewAI(AddrSpace: II.getSourceAddressSpace(), IsVolatile: II.isVolatile());
3978 }
3979
3980 Value *Src;
3981 if (VecTy && !IsWholeAlloca && !IsDest) {
3982 Src =
3983 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3984 Src = extractVector(IRB, V: Src, BeginIndex, EndIndex, Name: "vec");
3985 } else if (IntTy && !IsWholeAlloca && !IsDest) {
3986 Src =
3987 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3988 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: IntTy);
3989 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3990 Src = extractInteger(DL, IRB, V: Src, Ty: SubIntTy, Offset, Name: "extract");
3991 } else {
3992 LoadInst *Load = IRB.CreateAlignedLoad(Ty: OtherTy, Ptr: SrcPtr, Align: SrcAlign,
3993 isVolatile: II.isVolatile(), Name: "copyload");
3994 Load->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3995 LLVMContext::MD_access_group});
3996 if (AATags)
3997 Load->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3998 AccessTy: Load->getType(), DL));
3999 Src = Load;
4000 }
4001
4002 if (VecTy && !IsWholeAlloca && IsDest) {
4003 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4004 Name: "oldload");
4005 Src = insertVector(IRB, Old, V: Src, BeginIndex, Name: "vec");
4006 } else if (IntTy && !IsWholeAlloca && IsDest) {
4007 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4008 Name: "oldload");
4009 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
4010 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4011 Src = insertInteger(DL, IRB, Old, V: Src, Offset, Name: "insert");
4012 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: NewAllocaTy);
4013 }
4014
4015 StoreInst *Store = cast<StoreInst>(
4016 Val: IRB.CreateAlignedStore(Val: Src, Ptr: DstPtr, Align: DstAlign, isVolatile: II.isVolatile()));
4017 Store->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4018 LLVMContext::MD_access_group});
4019 if (AATags)
4020 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4021 AccessTy: Src->getType(), DL));
4022
4023 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DstPtr->getType()), 0);
4024 if (IsDest) {
4025
4026 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
4027 Inst: Store, Dest: DstPtr, Value: Src, DL);
4028 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4029 Val: DstPtr->stripAndAccumulateConstantOffsets(
4030 DL, Offset, /*AllowNonInbounds*/ true))) {
4031 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8, SliceSizeInBits: SliceSize * 8,
4032 OldInst: &II, Inst: Store, Dest: DstPtr, Value: Src, DL);
4033 }
4034
4035 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4036 return !II.isVolatile();
4037 }
4038
4039 bool visitIntrinsicInst(IntrinsicInst &II) {
4040 assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
4041 "Unexpected intrinsic!");
4042 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4043
4044 // Record this instruction for deletion.
4045 Pass.DeadInsts.push_back(Elt: &II);
4046
4047 if (II.isDroppable()) {
4048 assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4049 // TODO For now we forget assumed information, this can be improved.
4050 OldPtr->dropDroppableUsesIn(Usr&: II);
4051 return true;
4052 }
4053
4054 assert(II.getArgOperand(0) == OldPtr);
4055 Type *PointerTy = IRB.getPtrTy(AddrSpace: OldPtr->getType()->getPointerAddressSpace());
4056 Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4057 Value *New;
4058 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4059 New = IRB.CreateLifetimeStart(Ptr);
4060 else
4061 New = IRB.CreateLifetimeEnd(Ptr);
4062
4063 (void)New;
4064 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4065
4066 return true;
4067 }
4068
4069 void fixLoadStoreAlign(Instruction &Root) {
4070 // This algorithm implements the same visitor loop as
4071 // hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4072 // or store found.
4073 SmallPtrSet<Instruction *, 4> Visited;
4074 SmallVector<Instruction *, 4> Uses;
4075 Visited.insert(Ptr: &Root);
4076 Uses.push_back(Elt: &Root);
4077 do {
4078 Instruction *I = Uses.pop_back_val();
4079
4080 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
4081 LI->setAlignment(std::min(a: LI->getAlign(), b: getSliceAlign()));
4082 continue;
4083 }
4084 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
4085 SI->setAlignment(std::min(a: SI->getAlign(), b: getSliceAlign()));
4086 continue;
4087 }
4088
4089 assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||
4090 isa<PHINode>(I) || isa<SelectInst>(I) ||
4091 isa<GetElementPtrInst>(I));
4092 for (User *U : I->users())
4093 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
4094 Uses.push_back(Elt: cast<Instruction>(Val: U));
4095 } while (!Uses.empty());
4096 }
4097
4098 bool visitPHINode(PHINode &PN) {
4099 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4100 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4101 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4102
4103 // We would like to compute a new pointer in only one place, but have it be
4104 // as local as possible to the PHI. To do that, we re-use the location of
4105 // the old pointer, which necessarily must be in the right position to
4106 // dominate the PHI.
4107 IRBuilderBase::InsertPointGuard Guard(IRB);
4108 if (isa<PHINode>(Val: OldPtr))
4109 IRB.SetInsertPoint(TheBB: OldPtr->getParent(),
4110 IP: OldPtr->getParent()->getFirstInsertionPt());
4111 else
4112 IRB.SetInsertPoint(OldPtr);
4113 IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4114
4115 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4116 // Replace the operands which were using the old pointer.
4117 std::replace(first: PN.op_begin(), last: PN.op_end(), old_value: cast<Value>(Val: OldPtr), new_value: NewPtr);
4118
4119 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4120 deleteIfTriviallyDead(V: OldPtr);
4121
4122 // Fix the alignment of any loads or stores using this PHI node.
4123 fixLoadStoreAlign(Root&: PN);
4124
4125 // PHIs can't be promoted on their own, but often can be speculated. We
4126 // check the speculation outside of the rewriter so that we see the
4127 // fully-rewritten alloca.
4128 PHIUsers.insert(X: &PN);
4129 return true;
4130 }
4131
4132 bool visitSelectInst(SelectInst &SI) {
4133 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4134 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
4135 "Pointer isn't an operand!");
4136 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4137 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4138
4139 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4140 // Replace the operands which were using the old pointer.
4141 if (SI.getOperand(i_nocapture: 1) == OldPtr)
4142 SI.setOperand(i_nocapture: 1, Val_nocapture: NewPtr);
4143 if (SI.getOperand(i_nocapture: 2) == OldPtr)
4144 SI.setOperand(i_nocapture: 2, Val_nocapture: NewPtr);
4145
4146 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4147 deleteIfTriviallyDead(V: OldPtr);
4148
4149 // Fix the alignment of any loads or stores using this select.
4150 fixLoadStoreAlign(Root&: SI);
4151
4152 // Selects can't be promoted on their own, but often can be speculated. We
4153 // check the speculation outside of the rewriter so that we see the
4154 // fully-rewritten alloca.
4155 SelectUsers.insert(X: &SI);
4156 return true;
4157 }
4158};
4159
4160/// Visitor to rewrite aggregate loads and stores as scalar.
4161///
4162/// This pass aggressively rewrites all aggregate loads and stores on
4163/// a particular pointer (or any pointer derived from it which we can identify)
4164/// with scalar loads and stores.
4165class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4166 // Befriend the base class so it can delegate to private visit methods.
4167 friend class InstVisitor<AggLoadStoreRewriter, bool>;
4168
4169 /// Queue of pointer uses to analyze and potentially rewrite.
4170 SmallVector<Use *, 8> Queue;
4171
4172 /// Set to prevent us from cycling with phi nodes and loops.
4173 SmallPtrSet<User *, 8> Visited;
4174
4175 /// The current pointer use being rewritten. This is used to dig up the used
4176 /// value (as opposed to the user).
4177 Use *U = nullptr;
4178
4179 /// Used to calculate offsets, and hence alignment, of subobjects.
4180 const DataLayout &DL;
4181
4182 IRBuilderTy &IRB;
4183
4184public:
4185 AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4186 : DL(DL), IRB(IRB) {}
4187
4188 /// Rewrite loads and stores through a pointer and all pointers derived from
4189 /// it.
4190 bool rewrite(Instruction &I) {
4191 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4192 enqueueUsers(I);
4193 bool Changed = false;
4194 while (!Queue.empty()) {
4195 U = Queue.pop_back_val();
4196 Changed |= visit(I: cast<Instruction>(Val: U->getUser()));
4197 }
4198 return Changed;
4199 }
4200
4201private:
4202 /// Enqueue all the users of the given instruction for further processing.
4203 /// This uses a set to de-duplicate users.
4204 void enqueueUsers(Instruction &I) {
4205 for (Use &U : I.uses())
4206 if (Visited.insert(Ptr: U.getUser()).second)
4207 Queue.push_back(Elt: &U);
4208 }
4209
4210 // Conservative default is to not rewrite anything.
4211 bool visitInstruction(Instruction &I) { return false; }
4212
4213 /// Generic recursive split emission class.
4214 template <typename Derived> class OpSplitter {
4215 protected:
4216 /// The builder used to form new instructions.
4217 IRBuilderTy &IRB;
4218
4219 /// The indices which to be used with insert- or extractvalue to select the
4220 /// appropriate value within the aggregate.
4221 SmallVector<unsigned, 4> Indices;
4222
4223 /// The indices to a GEP instruction which will move Ptr to the correct slot
4224 /// within the aggregate.
4225 SmallVector<Value *, 4> GEPIndices;
4226
4227 /// The base pointer of the original op, used as a base for GEPing the
4228 /// split operations.
4229 Value *Ptr;
4230
4231 /// The base pointee type being GEPed into.
4232 Type *BaseTy;
4233
4234 /// Known alignment of the base pointer.
4235 Align BaseAlign;
4236
4237 /// To calculate offset of each component so we can correctly deduce
4238 /// alignments.
4239 const DataLayout &DL;
4240
4241 /// Initialize the splitter with an insertion point, Ptr and start with a
4242 /// single zero GEP index.
4243 OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4244 Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4245 : IRB(IRB), GEPIndices(1, IRB.getInt32(C: 0)), Ptr(Ptr), BaseTy(BaseTy),
4246 BaseAlign(BaseAlign), DL(DL) {
4247 IRB.SetInsertPoint(InsertionPoint);
4248 }
4249
4250 public:
4251 /// Generic recursive split emission routine.
4252 ///
4253 /// This method recursively splits an aggregate op (load or store) into
4254 /// scalar or vector ops. It splits recursively until it hits a single value
4255 /// and emits that single value operation via the template argument.
4256 ///
4257 /// The logic of this routine relies on GEPs and insertvalue and
4258 /// extractvalue all operating with the same fundamental index list, merely
4259 /// formatted differently (GEPs need actual values).
4260 ///
4261 /// \param Ty The type being split recursively into smaller ops.
4262 /// \param Agg The aggregate value being built up or stored, depending on
4263 /// whether this is splitting a load or a store respectively.
4264 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
4265 if (Ty->isSingleValueType()) {
4266 unsigned Offset = DL.getIndexedOffsetInType(ElemTy: BaseTy, Indices: GEPIndices);
4267 return static_cast<Derived *>(this)->emitFunc(
4268 Ty, Agg, commonAlignment(A: BaseAlign, Offset), Name);
4269 }
4270
4271 if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
4272 unsigned OldSize = Indices.size();
4273 (void)OldSize;
4274 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
4275 ++Idx) {
4276 assert(Indices.size() == OldSize && "Did not return to the old size");
4277 Indices.push_back(Elt: Idx);
4278 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4279 emitSplitOps(Ty: ATy->getElementType(), Agg, Name: Name + "." + Twine(Idx));
4280 GEPIndices.pop_back();
4281 Indices.pop_back();
4282 }
4283 return;
4284 }
4285
4286 if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4287 unsigned OldSize = Indices.size();
4288 (void)OldSize;
4289 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
4290 ++Idx) {
4291 assert(Indices.size() == OldSize && "Did not return to the old size");
4292 Indices.push_back(Elt: Idx);
4293 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4294 emitSplitOps(Ty: STy->getElementType(N: Idx), Agg, Name: Name + "." + Twine(Idx));
4295 GEPIndices.pop_back();
4296 Indices.pop_back();
4297 }
4298 return;
4299 }
4300
4301 llvm_unreachable("Only arrays and structs are aggregate loadable types");
4302 }
4303 };
4304
4305 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4306 AAMDNodes AATags;
4307 // A vector to hold the split components that we want to emit
4308 // separate fake uses for.
4309 SmallVector<Value *, 4> Components;
4310 // A vector to hold all the fake uses of the struct that we are splitting.
4311 // Usually there should only be one, but we are handling the general case.
4312 SmallVector<Instruction *, 1> FakeUses;
4313
4314 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4315 AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4316 IRBuilderTy &IRB)
4317 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4318 IRB),
4319 AATags(AATags) {}
4320
4321 /// Emit a leaf load of a single value. This is called at the leaves of the
4322 /// recursive emission to actually load values.
4323 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4324 assert(Ty->isSingleValueType());
4325 // Load the single value and insert it using the indices.
4326 Value *GEP =
4327 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4328 LoadInst *Load =
4329 IRB.CreateAlignedLoad(Ty, Ptr: GEP, Align: Alignment, Name: Name + ".load");
4330
4331 APInt Offset(
4332 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4333 if (AATags &&
4334 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset))
4335 Load->setAAMetadata(
4336 AATags.adjustForAccess(Offset: Offset.getZExtValue(), AccessTy: Load->getType(), DL));
4337 // Record the load so we can generate a fake use for this aggregate
4338 // component.
4339 Components.push_back(Elt: Load);
4340
4341 Agg = IRB.CreateInsertValue(Agg, Val: Load, Idxs: Indices, Name: Name + ".insert");
4342 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4343 }
4344
4345 // Stash the fake uses that use the value generated by this instruction.
4346 void recordFakeUses(LoadInst &LI) {
4347 for (Use &U : LI.uses())
4348 if (auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser()))
4349 if (II->getIntrinsicID() == Intrinsic::fake_use)
4350 FakeUses.push_back(Elt: II);
4351 }
4352
4353 // Replace all fake uses of the aggregate with a series of fake uses, one
4354 // for each split component.
4355 void emitFakeUses() {
4356 for (Instruction *I : FakeUses) {
4357 IRB.SetInsertPoint(I);
4358 for (auto *V : Components)
4359 IRB.CreateIntrinsic(ID: Intrinsic::fake_use, Args: {V});
4360 I->eraseFromParent();
4361 }
4362 }
4363 };
4364
4365 bool visitLoadInst(LoadInst &LI) {
4366 assert(LI.getPointerOperand() == *U);
4367 if (!LI.isSimple() || LI.getType()->isSingleValueType())
4368 return false;
4369
4370 // We have an aggregate being loaded, split it apart.
4371 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4372 LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4373 getAdjustedAlignment(I: &LI, Offset: 0), DL, IRB);
4374 Splitter.recordFakeUses(LI);
4375 Value *V = PoisonValue::get(T: LI.getType());
4376 Splitter.emitSplitOps(Ty: LI.getType(), Agg&: V, Name: LI.getName() + ".fca");
4377 Splitter.emitFakeUses();
4378 Visited.erase(Ptr: &LI);
4379 LI.replaceAllUsesWith(V);
4380 LI.eraseFromParent();
4381 return true;
4382 }
4383
4384 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4385 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4386 AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4387 const DataLayout &DL, IRBuilderTy &IRB)
4388 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4389 DL, IRB),
4390 AATags(AATags), AggStore(AggStore) {}
4391 AAMDNodes AATags;
4392 StoreInst *AggStore;
4393 /// Emit a leaf store of a single value. This is called at the leaves of the
4394 /// recursive emission to actually produce stores.
4395 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4396 assert(Ty->isSingleValueType());
4397 // Extract the single value and store it using the indices.
4398 //
4399 // The gep and extractvalue values are factored out of the CreateStore
4400 // call to make the output independent of the argument evaluation order.
4401 Value *ExtractValue =
4402 IRB.CreateExtractValue(Agg, Idxs: Indices, Name: Name + ".extract");
4403 Value *InBoundsGEP =
4404 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4405 StoreInst *Store =
4406 IRB.CreateAlignedStore(Val: ExtractValue, Ptr: InBoundsGEP, Align: Alignment);
4407
4408 APInt Offset(
4409 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4410 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset);
4411 if (AATags) {
4412 Store->setAAMetadata(AATags.adjustForAccess(
4413 Offset: Offset.getZExtValue(), AccessTy: ExtractValue->getType(), DL));
4414 }
4415
4416 // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4417 // If we cannot (because there's an intervening non-const or unbounded
4418 // gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4419 // this instruction.
4420 Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets();
4421 if (auto *OldAI = dyn_cast<AllocaInst>(Val: Base)) {
4422 uint64_t SizeInBits =
4423 DL.getTypeSizeInBits(Ty: Store->getValueOperand()->getType());
4424 migrateDebugInfo(OldAlloca: OldAI, /*IsSplit*/ true, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
4425 SliceSizeInBits: SizeInBits, OldInst: AggStore, Inst: Store,
4426 Dest: Store->getPointerOperand(), Value: Store->getValueOperand(),
4427 DL);
4428 } else {
4429 assert(at::getDVRAssignmentMarkers(Store).empty() &&
4430 "AT: unexpected debug.assign linked to store through "
4431 "unbounded GEP");
4432 }
4433 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4434 }
4435 };
4436
4437 bool visitStoreInst(StoreInst &SI) {
4438 if (!SI.isSimple() || SI.getPointerOperand() != *U)
4439 return false;
4440 Value *V = SI.getValueOperand();
4441 if (V->getType()->isSingleValueType())
4442 return false;
4443
4444 // We have an aggregate being stored, split it apart.
4445 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4446 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4447 getAdjustedAlignment(I: &SI, Offset: 0), DL, IRB);
4448 Splitter.emitSplitOps(Ty: V->getType(), Agg&: V, Name: V->getName() + ".fca");
4449 Visited.erase(Ptr: &SI);
4450 // The stores replacing SI each have markers describing fragments of the
4451 // assignment so delete the assignment markers linked to SI.
4452 at::deleteAssignmentMarkers(Inst: &SI);
4453 SI.eraseFromParent();
4454 return true;
4455 }
4456
4457 bool visitBitCastInst(BitCastInst &BC) {
4458 enqueueUsers(I&: BC);
4459 return false;
4460 }
4461
4462 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4463 enqueueUsers(I&: ASC);
4464 return false;
4465 }
4466
4467 // Unfold gep (select cond, ptr1, ptr2), idx
4468 // => select cond, gep(ptr1, idx), gep(ptr2, idx)
4469 // and gep ptr, (select cond, idx1, idx2)
4470 // => select cond, gep(ptr, idx1), gep(ptr, idx2)
4471 // We also allow for i1 zext indices, which are equivalent to selects.
4472 bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4473 // Check whether the GEP has exactly one select operand and all indices
4474 // will become constant after the transform.
4475 Instruction *Sel = dyn_cast<SelectInst>(Val: GEPI.getPointerOperand());
4476 for (Value *Op : GEPI.indices()) {
4477 if (auto *SI = dyn_cast<SelectInst>(Val: Op)) {
4478 if (Sel)
4479 return false;
4480
4481 Sel = SI;
4482 if (!isa<ConstantInt>(Val: SI->getTrueValue()) ||
4483 !isa<ConstantInt>(Val: SI->getFalseValue()))
4484 return false;
4485 continue;
4486 }
4487 if (auto *ZI = dyn_cast<ZExtInst>(Val: Op)) {
4488 if (Sel)
4489 return false;
4490 Sel = ZI;
4491 if (!ZI->getSrcTy()->isIntegerTy(BitWidth: 1))
4492 return false;
4493 continue;
4494 }
4495
4496 if (!isa<ConstantInt>(Val: Op))
4497 return false;
4498 }
4499
4500 if (!Sel)
4501 return false;
4502
4503 LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4504 dbgs() << " original: " << *Sel << "\n";
4505 dbgs() << " " << GEPI << "\n";);
4506
4507 auto GetNewOps = [&](Value *SelOp) {
4508 SmallVector<Value *> NewOps;
4509 for (Value *Op : GEPI.operands())
4510 if (Op == Sel)
4511 NewOps.push_back(Elt: SelOp);
4512 else
4513 NewOps.push_back(Elt: Op);
4514 return NewOps;
4515 };
4516
4517 Value *Cond, *True, *False;
4518 Instruction *MDFrom = nullptr;
4519 if (auto *SI = dyn_cast<SelectInst>(Val: Sel)) {
4520 Cond = SI->getCondition();
4521 True = SI->getTrueValue();
4522 False = SI->getFalseValue();
4523 MDFrom = SI;
4524 } else {
4525 Cond = Sel->getOperand(i: 0);
4526 True = ConstantInt::get(Ty: Sel->getType(), V: 1);
4527 False = ConstantInt::get(Ty: Sel->getType(), V: 0);
4528 }
4529 SmallVector<Value *> TrueOps = GetNewOps(True);
4530 SmallVector<Value *> FalseOps = GetNewOps(False);
4531
4532 IRB.SetInsertPoint(&GEPI);
4533 GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4534
4535 Type *Ty = GEPI.getSourceElementType();
4536 Value *NTrue = IRB.CreateGEP(Ty, Ptr: TrueOps[0], IdxList: ArrayRef(TrueOps).drop_front(),
4537 Name: True->getName() + ".sroa.gep", NW);
4538
4539 Value *NFalse =
4540 IRB.CreateGEP(Ty, Ptr: FalseOps[0], IdxList: ArrayRef(FalseOps).drop_front(),
4541 Name: False->getName() + ".sroa.gep", NW);
4542
4543 Value *NSel = MDFrom
4544 ? IRB.CreateSelect(C: Cond, True: NTrue, False: NFalse,
4545 Name: Sel->getName() + ".sroa.sel", MDFrom)
4546 : IRB.CreateSelectWithUnknownProfile(
4547 C: Cond, True: NTrue, False: NFalse, DEBUG_TYPE,
4548 Name: Sel->getName() + ".sroa.sel");
4549 Visited.erase(Ptr: &GEPI);
4550 GEPI.replaceAllUsesWith(V: NSel);
4551 GEPI.eraseFromParent();
4552 Instruction *NSelI = cast<Instruction>(Val: NSel);
4553 Visited.insert(Ptr: NSelI);
4554 enqueueUsers(I&: *NSelI);
4555
4556 LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4557 dbgs() << " " << *NFalse << "\n";
4558 dbgs() << " " << *NSel << "\n";);
4559
4560 return true;
4561 }
4562
4563 // Unfold gep (phi ptr1, ptr2), idx
4564 // => phi ((gep ptr1, idx), (gep ptr2, idx))
4565 // and gep ptr, (phi idx1, idx2)
4566 // => phi ((gep ptr, idx1), (gep ptr, idx2))
4567 bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4568 // To prevent infinitely expanding recursive phis, bail if the GEP pointer
4569 // operand (looking through the phi if it is the phi we want to unfold) is
4570 // an instruction besides a static alloca.
4571 PHINode *Phi = dyn_cast<PHINode>(Val: GEPI.getPointerOperand());
4572 auto IsInvalidPointerOperand = [](Value *V) {
4573 if (!isa<Instruction>(Val: V))
4574 return false;
4575 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
4576 return !AI->isStaticAlloca();
4577 return true;
4578 };
4579 if (Phi) {
4580 if (any_of(Range: Phi->operands(), P: IsInvalidPointerOperand))
4581 return false;
4582 } else {
4583 if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4584 return false;
4585 }
4586 // Check whether the GEP has exactly one phi operand (including the pointer
4587 // operand) and all indices will become constant after the transform.
4588 for (Value *Op : GEPI.indices()) {
4589 if (auto *SI = dyn_cast<PHINode>(Val: Op)) {
4590 if (Phi)
4591 return false;
4592
4593 Phi = SI;
4594 if (!all_of(Range: Phi->incoming_values(),
4595 P: [](Value *V) { return isa<ConstantInt>(Val: V); }))
4596 return false;
4597 continue;
4598 }
4599
4600 if (!isa<ConstantInt>(Val: Op))
4601 return false;
4602 }
4603
4604 if (!Phi)
4605 return false;
4606
4607 LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4608 dbgs() << " original: " << *Phi << "\n";
4609 dbgs() << " " << GEPI << "\n";);
4610
4611 auto GetNewOps = [&](Value *PhiOp) {
4612 SmallVector<Value *> NewOps;
4613 for (Value *Op : GEPI.operands())
4614 if (Op == Phi)
4615 NewOps.push_back(Elt: PhiOp);
4616 else
4617 NewOps.push_back(Elt: Op);
4618 return NewOps;
4619 };
4620
4621 IRB.SetInsertPoint(Phi);
4622 PHINode *NewPhi = IRB.CreatePHI(Ty: GEPI.getType(), NumReservedValues: Phi->getNumIncomingValues(),
4623 Name: Phi->getName() + ".sroa.phi");
4624
4625 Type *SourceTy = GEPI.getSourceElementType();
4626 // We only handle arguments, constants, and static allocas here, so we can
4627 // insert GEPs at the end of the entry block.
4628 IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4629 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4630 Value *Op = Phi->getIncomingValue(i: I);
4631 BasicBlock *BB = Phi->getIncomingBlock(i: I);
4632 Value *NewGEP;
4633 if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) {
4634 NewGEP = NewPhi->getIncomingValue(i: NI);
4635 } else {
4636 SmallVector<Value *> NewOps = GetNewOps(Op);
4637 NewGEP =
4638 IRB.CreateGEP(Ty: SourceTy, Ptr: NewOps[0], IdxList: ArrayRef(NewOps).drop_front(),
4639 Name: Phi->getName() + ".sroa.gep", NW: GEPI.getNoWrapFlags());
4640 }
4641 NewPhi->addIncoming(V: NewGEP, BB);
4642 }
4643
4644 Visited.erase(Ptr: &GEPI);
4645 GEPI.replaceAllUsesWith(V: NewPhi);
4646 GEPI.eraseFromParent();
4647 Visited.insert(Ptr: NewPhi);
4648 enqueueUsers(I&: *NewPhi);
4649
4650 LLVM_DEBUG(dbgs() << " to: ";
4651 for (Value *In
4652 : NewPhi->incoming_values()) dbgs()
4653 << "\n " << *In;
4654 dbgs() << "\n " << *NewPhi << '\n');
4655
4656 return true;
4657 }
4658
4659 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4660 if (unfoldGEPSelect(GEPI))
4661 return true;
4662
4663 if (unfoldGEPPhi(GEPI))
4664 return true;
4665
4666 enqueueUsers(I&: GEPI);
4667 return false;
4668 }
4669
4670 bool visitPHINode(PHINode &PN) {
4671 enqueueUsers(I&: PN);
4672 return false;
4673 }
4674
4675 bool visitSelectInst(SelectInst &SI) {
4676 enqueueUsers(I&: SI);
4677 return false;
4678 }
4679};
4680
4681} // end anonymous namespace
4682
4683/// Strip aggregate type wrapping.
4684///
4685/// This removes no-op aggregate types wrapping an underlying type. It will
4686/// strip as many layers of types as it can without changing either the type
4687/// size or the allocated size.
4688static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
4689 if (Ty->isSingleValueType())
4690 return Ty;
4691
4692 uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4693 uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4694
4695 Type *InnerTy;
4696 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Val: Ty)) {
4697 InnerTy = ArrTy->getElementType();
4698 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4699 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4700 unsigned Index = SL->getElementContainingOffset(FixedOffset: 0);
4701 InnerTy = STy->getElementType(N: Index);
4702 } else {
4703 return Ty;
4704 }
4705
4706 if (AllocSize > DL.getTypeAllocSize(Ty: InnerTy).getFixedValue() ||
4707 TypeSize > DL.getTypeSizeInBits(Ty: InnerTy).getFixedValue())
4708 return Ty;
4709
4710 return stripAggregateTypeWrapping(DL, Ty: InnerTy);
4711}
4712
4713/// Try to find a partition of the aggregate type passed in for a given
4714/// offset and size.
4715///
4716/// This recurses through the aggregate type and tries to compute a subtype
4717/// based on the offset and size. When the offset and size span a sub-section
4718/// of an array, it will even compute a new array type for that sub-section,
4719/// and the same for structs.
4720///
4721/// Note that this routine is very strict and tries to find a partition of the
4722/// type which produces the *exact* right offset and size. It is not forgiving
4723/// when the size or offset cause either end of type-based partition to be off.
4724/// Also, this is a best-effort routine. It is reasonable to give up and not
4725/// return a type if necessary.
4726static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
4727 uint64_t Size) {
4728 if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4729 return stripAggregateTypeWrapping(DL, Ty);
4730 if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() ||
4731 (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4732 return nullptr;
4733
4734 if (isa<ArrayType>(Val: Ty) || isa<VectorType>(Val: Ty)) {
4735 Type *ElementTy;
4736 uint64_t TyNumElements;
4737 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
4738 ElementTy = AT->getElementType();
4739 TyNumElements = AT->getNumElements();
4740 } else {
4741 // FIXME: This isn't right for vectors with non-byte-sized or
4742 // non-power-of-two sized elements.
4743 auto *VT = cast<FixedVectorType>(Val: Ty);
4744 ElementTy = VT->getElementType();
4745 TyNumElements = VT->getNumElements();
4746 }
4747 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4748 uint64_t NumSkippedElements = Offset / ElementSize;
4749 if (NumSkippedElements >= TyNumElements)
4750 return nullptr;
4751 Offset -= NumSkippedElements * ElementSize;
4752
4753 // First check if we need to recurse.
4754 if (Offset > 0 || Size < ElementSize) {
4755 // Bail if the partition ends in a different array element.
4756 if ((Offset + Size) > ElementSize)
4757 return nullptr;
4758 // Recurse through the element type trying to peel off offset bytes.
4759 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4760 }
4761 assert(Offset == 0);
4762
4763 if (Size == ElementSize)
4764 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4765 assert(Size > ElementSize);
4766 uint64_t NumElements = Size / ElementSize;
4767 if (NumElements * ElementSize != Size)
4768 return nullptr;
4769 return ArrayType::get(ElementType: ElementTy, NumElements);
4770 }
4771
4772 StructType *STy = dyn_cast<StructType>(Val: Ty);
4773 if (!STy)
4774 return nullptr;
4775
4776 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4777
4778 if (SL->getSizeInBits().isScalable())
4779 return nullptr;
4780
4781 if (Offset >= SL->getSizeInBytes())
4782 return nullptr;
4783 uint64_t EndOffset = Offset + Size;
4784 if (EndOffset > SL->getSizeInBytes())
4785 return nullptr;
4786
4787 unsigned Index = SL->getElementContainingOffset(FixedOffset: Offset);
4788 Offset -= SL->getElementOffset(Idx: Index);
4789
4790 Type *ElementTy = STy->getElementType(N: Index);
4791 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4792 if (Offset >= ElementSize)
4793 return nullptr; // The offset points into alignment padding.
4794
4795 // See if any partition must be contained by the element.
4796 if (Offset > 0 || Size < ElementSize) {
4797 if ((Offset + Size) > ElementSize)
4798 return nullptr;
4799 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4800 }
4801 assert(Offset == 0);
4802
4803 if (Size == ElementSize)
4804 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4805
4806 StructType::element_iterator EI = STy->element_begin() + Index,
4807 EE = STy->element_end();
4808 if (EndOffset < SL->getSizeInBytes()) {
4809 unsigned EndIndex = SL->getElementContainingOffset(FixedOffset: EndOffset);
4810 if (Index == EndIndex)
4811 return nullptr; // Within a single element and its padding.
4812
4813 // Don't try to form "natural" types if the elements don't line up with the
4814 // expected size.
4815 // FIXME: We could potentially recurse down through the last element in the
4816 // sub-struct to find a natural end point.
4817 if (SL->getElementOffset(Idx: EndIndex) != EndOffset)
4818 return nullptr;
4819
4820 assert(Index < EndIndex);
4821 EE = STy->element_begin() + EndIndex;
4822 }
4823
4824 // Try to build up a sub-structure.
4825 StructType *SubTy =
4826 StructType::get(Context&: STy->getContext(), Elements: ArrayRef(EI, EE), isPacked: STy->isPacked());
4827 const StructLayout *SubSL = DL.getStructLayout(Ty: SubTy);
4828 if (Size != SubSL->getSizeInBytes())
4829 return nullptr; // The sub-struct doesn't have quite the size needed.
4830
4831 return SubTy;
4832}
4833
4834/// Pre-split loads and stores to simplify rewriting.
4835///
4836/// We want to break up the splittable load+store pairs as much as
4837/// possible. This is important to do as a preprocessing step, as once we
4838/// start rewriting the accesses to partitions of the alloca we lose the
4839/// necessary information to correctly split apart paired loads and stores
4840/// which both point into this alloca. The case to consider is something like
4841/// the following:
4842///
4843/// %a = alloca [12 x i8]
4844/// %gep1 = getelementptr i8, ptr %a, i32 0
4845/// %gep2 = getelementptr i8, ptr %a, i32 4
4846/// %gep3 = getelementptr i8, ptr %a, i32 8
4847/// store float 0.0, ptr %gep1
4848/// store float 1.0, ptr %gep2
4849/// %v = load i64, ptr %gep1
4850/// store i64 %v, ptr %gep2
4851/// %f1 = load float, ptr %gep2
4852/// %f2 = load float, ptr %gep3
4853///
4854/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
4855/// promote everything so we recover the 2 SSA values that should have been
4856/// there all along.
4857///
4858/// \returns true if any changes are made.
4859bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
4860 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
4861
4862 // Track the loads and stores which are candidates for pre-splitting here, in
4863 // the order they first appear during the partition scan. These give stable
4864 // iteration order and a basis for tracking which loads and stores we
4865 // actually split.
4866 SmallVector<LoadInst *, 4> Loads;
4867 SmallVector<StoreInst *, 4> Stores;
4868
4869 // We need to accumulate the splits required of each load or store where we
4870 // can find them via a direct lookup. This is important to cross-check loads
4871 // and stores against each other. We also track the slice so that we can kill
4872 // all the slices that end up split.
4873 struct SplitOffsets {
4874 Slice *S;
4875 std::vector<uint64_t> Splits;
4876 };
4877 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
4878
4879 // Track loads out of this alloca which cannot, for any reason, be pre-split.
4880 // This is important as we also cannot pre-split stores of those loads!
4881 // FIXME: This is all pretty gross. It means that we can be more aggressive
4882 // in pre-splitting when the load feeding the store happens to come from
4883 // a separate alloca. Put another way, the effectiveness of SROA would be
4884 // decreased by a frontend which just concatenated all of its local allocas
4885 // into one big flat alloca. But defeating such patterns is exactly the job
4886 // SROA is tasked with! Sadly, to not have this discrepancy we would have
4887 // change store pre-splitting to actually force pre-splitting of the load
4888 // that feeds it *and all stores*. That makes pre-splitting much harder, but
4889 // maybe it would make it more principled?
4890 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
4891
4892 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
4893 for (auto &P : AS.partitions()) {
4894 for (Slice &S : P) {
4895 Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
4896 if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
4897 // If this is a load we have to track that it can't participate in any
4898 // pre-splitting. If this is a store of a load we have to track that
4899 // that load also can't participate in any pre-splitting.
4900 if (auto *LI = dyn_cast<LoadInst>(Val: I))
4901 UnsplittableLoads.insert(Ptr: LI);
4902 else if (auto *SI = dyn_cast<StoreInst>(Val: I))
4903 if (auto *LI = dyn_cast<LoadInst>(Val: SI->getValueOperand()))
4904 UnsplittableLoads.insert(Ptr: LI);
4905 continue;
4906 }
4907 assert(P.endOffset() > S.beginOffset() &&
4908 "Empty or backwards partition!");
4909
4910 // Determine if this is a pre-splittable slice.
4911 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
4912 assert(!LI->isVolatile() && "Cannot split volatile loads!");
4913
4914 // The load must be used exclusively to store into other pointers for
4915 // us to be able to arbitrarily pre-split it. The stores must also be
4916 // simple to avoid changing semantics.
4917 auto IsLoadSimplyStored = [](LoadInst *LI) {
4918 for (User *LU : LI->users()) {
4919 auto *SI = dyn_cast<StoreInst>(Val: LU);
4920 if (!SI || !SI->isSimple())
4921 return false;
4922 }
4923 return true;
4924 };
4925 if (!IsLoadSimplyStored(LI)) {
4926 UnsplittableLoads.insert(Ptr: LI);
4927 continue;
4928 }
4929
4930 Loads.push_back(Elt: LI);
4931 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4932 if (S.getUse() != &SI->getOperandUse(i: SI->getPointerOperandIndex()))
4933 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
4934 continue;
4935 auto *StoredLoad = dyn_cast<LoadInst>(Val: SI->getValueOperand());
4936 if (!StoredLoad || !StoredLoad->isSimple())
4937 continue;
4938 assert(!SI->isVolatile() && "Cannot split volatile stores!");
4939
4940 Stores.push_back(Elt: SI);
4941 } else {
4942 // Other uses cannot be pre-split.
4943 continue;
4944 }
4945
4946 // Record the initial split.
4947 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
4948 auto &Offsets = SplitOffsetsMap[I];
4949 assert(Offsets.Splits.empty() &&
4950 "Should not have splits the first time we see an instruction!");
4951 Offsets.S = &S;
4952 Offsets.Splits.push_back(x: P.endOffset() - S.beginOffset());
4953 }
4954
4955 // Now scan the already split slices, and add a split for any of them which
4956 // we're going to pre-split.
4957 for (Slice *S : P.splitSliceTails()) {
4958 auto SplitOffsetsMapI =
4959 SplitOffsetsMap.find(Val: cast<Instruction>(Val: S->getUse()->getUser()));
4960 if (SplitOffsetsMapI == SplitOffsetsMap.end())
4961 continue;
4962 auto &Offsets = SplitOffsetsMapI->second;
4963
4964 assert(Offsets.S == S && "Found a mismatched slice!");
4965 assert(!Offsets.Splits.empty() &&
4966 "Cannot have an empty set of splits on the second partition!");
4967 assert(Offsets.Splits.back() ==
4968 P.beginOffset() - Offsets.S->beginOffset() &&
4969 "Previous split does not end where this one begins!");
4970
4971 // Record each split. The last partition's end isn't needed as the size
4972 // of the slice dictates that.
4973 if (S->endOffset() > P.endOffset())
4974 Offsets.Splits.push_back(x: P.endOffset() - Offsets.S->beginOffset());
4975 }
4976 }
4977
4978 // We may have split loads where some of their stores are split stores. For
4979 // such loads and stores, we can only pre-split them if their splits exactly
4980 // match relative to their starting offset. We have to verify this prior to
4981 // any rewriting.
4982 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
4983 // Lookup the load we are storing in our map of split
4984 // offsets.
4985 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
4986 // If it was completely unsplittable, then we're done,
4987 // and this store can't be pre-split.
4988 if (UnsplittableLoads.count(Ptr: LI))
4989 return true;
4990
4991 auto LoadOffsetsI = SplitOffsetsMap.find(Val: LI);
4992 if (LoadOffsetsI == SplitOffsetsMap.end())
4993 return false; // Unrelated loads are definitely safe.
4994 auto &LoadOffsets = LoadOffsetsI->second;
4995
4996 // Now lookup the store's offsets.
4997 auto &StoreOffsets = SplitOffsetsMap[SI];
4998
4999 // If the relative offsets of each split in the load and
5000 // store match exactly, then we can split them and we
5001 // don't need to remove them here.
5002 if (LoadOffsets.Splits == StoreOffsets.Splits)
5003 return false;
5004
5005 LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
5006 << " " << *LI << "\n"
5007 << " " << *SI << "\n");
5008
5009 // We've found a store and load that we need to split
5010 // with mismatched relative splits. Just give up on them
5011 // and remove both instructions from our list of
5012 // candidates.
5013 UnsplittableLoads.insert(Ptr: LI);
5014 return true;
5015 });
5016 // Now we have to go *back* through all the stores, because a later store may
5017 // have caused an earlier store's load to become unsplittable and if it is
5018 // unsplittable for the later store, then we can't rely on it being split in
5019 // the earlier store either.
5020 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads](StoreInst *SI) {
5021 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5022 return UnsplittableLoads.count(Ptr: LI);
5023 });
5024 // Once we've established all the loads that can't be split for some reason,
5025 // filter any that made it into our list out.
5026 llvm::erase_if(C&: Loads, P: [&UnsplittableLoads](LoadInst *LI) {
5027 return UnsplittableLoads.count(Ptr: LI);
5028 });
5029
5030 // If no loads or stores are left, there is no pre-splitting to be done for
5031 // this alloca.
5032 if (Loads.empty() && Stores.empty())
5033 return false;
5034
5035 // From here on, we can't fail and will be building new accesses, so rig up
5036 // an IR builder.
5037 IRBuilderTy IRB(&AI);
5038
5039 // Collect the new slices which we will merge into the alloca slices.
5040 SmallVector<Slice, 4> NewSlices;
5041
5042 // Track any allocas we end up splitting loads and stores for so we iterate
5043 // on them.
5044 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
5045
5046 // At this point, we have collected all of the loads and stores we can
5047 // pre-split, and the specific splits needed for them. We actually do the
5048 // splitting in a specific order in order to handle when one of the loads in
5049 // the value operand to one of the stores.
5050 //
5051 // First, we rewrite all of the split loads, and just accumulate each split
5052 // load in a parallel structure. We also build the slices for them and append
5053 // them to the alloca slices.
5054 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
5055 std::vector<LoadInst *> SplitLoads;
5056 const DataLayout &DL = AI.getDataLayout();
5057 for (LoadInst *LI : Loads) {
5058 SplitLoads.clear();
5059
5060 auto &Offsets = SplitOffsetsMap[LI];
5061 unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5062 assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&
5063 "Load must have type size equal to store size");
5064 assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&
5065 "Load must be >= slice size");
5066
5067 uint64_t BaseOffset = Offsets.S->beginOffset();
5068 assert(BaseOffset + SliceSize > BaseOffset &&
5069 "Cannot represent alloca access size using 64-bit integers!");
5070
5071 Instruction *BasePtr = cast<Instruction>(Val: LI->getPointerOperand());
5072 IRB.SetInsertPoint(LI);
5073
5074 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5075
5076 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5077 int Idx = 0, Size = Offsets.Splits.size();
5078 for (;;) {
5079 auto *PartTy = Type::getIntNTy(C&: LI->getContext(), N: PartSize * 8);
5080 auto AS = LI->getPointerAddressSpace();
5081 auto *PartPtrTy = LI->getPointerOperandType();
5082 LoadInst *PLoad = IRB.CreateAlignedLoad(
5083 Ty: PartTy,
5084 Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5085 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5086 PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5087 Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5088 /*IsVolatile*/ isVolatile: false, Name: LI->getName());
5089 PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5090 LLVMContext::MD_access_group});
5091
5092 // Append this load onto the list of split loads so we can find it later
5093 // to rewrite the stores.
5094 SplitLoads.push_back(x: PLoad);
5095
5096 // Now build a new slice for the alloca.
5097 NewSlices.push_back(
5098 Elt: Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5099 &PLoad->getOperandUse(i: PLoad->getPointerOperandIndex()),
5100 /*IsSplittable*/ false));
5101 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5102 << ", " << NewSlices.back().endOffset()
5103 << "): " << *PLoad << "\n");
5104
5105 // See if we've handled all the splits.
5106 if (Idx >= Size)
5107 break;
5108
5109 // Setup the next partition.
5110 PartOffset = Offsets.Splits[Idx];
5111 ++Idx;
5112 PartSize = (Idx < Size ? Offsets.Splits[Idx] : SliceSize) - PartOffset;
5113 }
5114
5115 // Now that we have the split loads, do the slow walk over all uses of the
5116 // load and rewrite them as split stores, or save the split loads to use
5117 // below if the store is going to be split there anyways.
5118 bool DeferredStores = false;
5119 for (User *LU : LI->users()) {
5120 StoreInst *SI = cast<StoreInst>(Val: LU);
5121 if (!Stores.empty() && SplitOffsetsMap.count(Val: SI)) {
5122 DeferredStores = true;
5123 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5124 << "\n");
5125 continue;
5126 }
5127
5128 Value *StoreBasePtr = SI->getPointerOperand();
5129 IRB.SetInsertPoint(SI);
5130 AAMDNodes AATags = SI->getAAMetadata();
5131
5132 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5133
5134 for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5135 LoadInst *PLoad = SplitLoads[Idx];
5136 uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1];
5137 auto *PartPtrTy = SI->getPointerOperandType();
5138
5139 auto AS = SI->getPointerAddressSpace();
5140 StoreInst *PStore = IRB.CreateAlignedStore(
5141 Val: PLoad,
5142 Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5143 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5144 PointerTy: PartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5145 Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5146 /*IsVolatile*/ isVolatile: false);
5147 PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5148 LLVMContext::MD_access_group,
5149 LLVMContext::MD_DIAssignID});
5150
5151 if (AATags)
5152 PStore->setAAMetadata(
5153 AATags.adjustForAccess(Offset: PartOffset, AccessTy: PLoad->getType(), DL));
5154 LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5155 }
5156
5157 // We want to immediately iterate on any allocas impacted by splitting
5158 // this store, and we have to track any promotable alloca (indicated by
5159 // a direct store) as needing to be resplit because it is no longer
5160 // promotable.
5161 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: StoreBasePtr)) {
5162 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5163 Worklist.insert(X: OtherAI);
5164 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5165 Val: StoreBasePtr->stripInBoundsOffsets())) {
5166 Worklist.insert(X: OtherAI);
5167 }
5168
5169 // Mark the original store as dead.
5170 DeadInsts.push_back(Elt: SI);
5171 }
5172
5173 // Save the split loads if there are deferred stores among the users.
5174 if (DeferredStores)
5175 SplitLoadsMap.insert(KV: std::make_pair(x&: LI, y: std::move(SplitLoads)));
5176
5177 // Mark the original load as dead and kill the original slice.
5178 DeadInsts.push_back(Elt: LI);
5179 Offsets.S->kill();
5180 }
5181
5182 // Second, we rewrite all of the split stores. At this point, we know that
5183 // all loads from this alloca have been split already. For stores of such
5184 // loads, we can simply look up the pre-existing split loads. For stores of
5185 // other loads, we split those loads first and then write split stores of
5186 // them.
5187 for (StoreInst *SI : Stores) {
5188 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5189 IntegerType *Ty = cast<IntegerType>(Val: LI->getType());
5190 assert(Ty->getBitWidth() % 8 == 0);
5191 uint64_t StoreSize = Ty->getBitWidth() / 8;
5192 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
5193
5194 auto &Offsets = SplitOffsetsMap[SI];
5195 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5196 "Slice size should always match load size exactly!");
5197 uint64_t BaseOffset = Offsets.S->beginOffset();
5198 assert(BaseOffset + StoreSize > BaseOffset &&
5199 "Cannot represent alloca access size using 64-bit integers!");
5200
5201 Value *LoadBasePtr = LI->getPointerOperand();
5202 Instruction *StoreBasePtr = cast<Instruction>(Val: SI->getPointerOperand());
5203
5204 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5205
5206 // Check whether we have an already split load.
5207 auto SplitLoadsMapI = SplitLoadsMap.find(Val: LI);
5208 std::vector<LoadInst *> *SplitLoads = nullptr;
5209 if (SplitLoadsMapI != SplitLoadsMap.end()) {
5210 SplitLoads = &SplitLoadsMapI->second;
5211 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
5212 "Too few split loads for the number of splits in the store!");
5213 } else {
5214 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5215 }
5216
5217 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5218 int Idx = 0, Size = Offsets.Splits.size();
5219 for (;;) {
5220 auto *PartTy = Type::getIntNTy(C&: Ty->getContext(), N: PartSize * 8);
5221 auto *LoadPartPtrTy = LI->getPointerOperandType();
5222 auto *StorePartPtrTy = SI->getPointerOperandType();
5223
5224 // Either lookup a split load or create one.
5225 LoadInst *PLoad;
5226 if (SplitLoads) {
5227 PLoad = (*SplitLoads)[Idx];
5228 } else {
5229 IRB.SetInsertPoint(LI);
5230 auto AS = LI->getPointerAddressSpace();
5231 PLoad = IRB.CreateAlignedLoad(
5232 Ty: PartTy,
5233 Ptr: getAdjustedPtr(IRB, DL, Ptr: LoadBasePtr,
5234 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5235 PointerTy: LoadPartPtrTy, NamePrefix: LoadBasePtr->getName() + "."),
5236 Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5237 /*IsVolatile*/ isVolatile: false, Name: LI->getName());
5238 PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5239 LLVMContext::MD_access_group});
5240 }
5241
5242 // And store this partition.
5243 IRB.SetInsertPoint(SI);
5244 auto AS = SI->getPointerAddressSpace();
5245 StoreInst *PStore = IRB.CreateAlignedStore(
5246 Val: PLoad,
5247 Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5248 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5249 PointerTy: StorePartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5250 Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5251 /*IsVolatile*/ isVolatile: false);
5252 PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5253 LLVMContext::MD_access_group});
5254
5255 // Now build a new slice for the alloca.
5256 NewSlices.push_back(
5257 Elt: Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5258 &PStore->getOperandUse(i: PStore->getPointerOperandIndex()),
5259 /*IsSplittable*/ false));
5260 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5261 << ", " << NewSlices.back().endOffset()
5262 << "): " << *PStore << "\n");
5263 if (!SplitLoads) {
5264 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5265 }
5266
5267 // See if we've finished all the splits.
5268 if (Idx >= Size)
5269 break;
5270
5271 // Setup the next partition.
5272 PartOffset = Offsets.Splits[Idx];
5273 ++Idx;
5274 PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset;
5275 }
5276
5277 // We want to immediately iterate on any allocas impacted by splitting
5278 // this load, which is only relevant if it isn't a load of this alloca and
5279 // thus we didn't already split the loads above. We also have to keep track
5280 // of any promotable allocas we split loads on as they can no longer be
5281 // promoted.
5282 if (!SplitLoads) {
5283 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: LoadBasePtr)) {
5284 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5285 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5286 Worklist.insert(X: OtherAI);
5287 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5288 Val: LoadBasePtr->stripInBoundsOffsets())) {
5289 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5290 Worklist.insert(X: OtherAI);
5291 }
5292 }
5293
5294 // Mark the original store as dead now that we've split it up and kill its
5295 // slice. Note that we leave the original load in place unless this store
5296 // was its only use. It may in turn be split up if it is an alloca load
5297 // for some other alloca, but it may be a normal load. This may introduce
5298 // redundant loads, but where those can be merged the rest of the optimizer
5299 // should handle the merging, and this uncovers SSA splits which is more
5300 // important. In practice, the original loads will almost always be fully
5301 // split and removed eventually, and the splits will be merged by any
5302 // trivial CSE, including instcombine.
5303 if (LI->hasOneUse()) {
5304 assert(*LI->user_begin() == SI && "Single use isn't this store!");
5305 DeadInsts.push_back(Elt: LI);
5306 }
5307 DeadInsts.push_back(Elt: SI);
5308 Offsets.S->kill();
5309 }
5310
5311 // Remove the killed slices that have ben pre-split.
5312 llvm::erase_if(C&: AS, P: [](const Slice &S) { return S.isDead(); });
5313
5314 // Insert our new slices. This will sort and merge them into the sorted
5315 // sequence.
5316 AS.insert(NewSlices);
5317
5318 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5319#ifndef NDEBUG
5320 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5321 LLVM_DEBUG(AS.print(dbgs(), I, " "));
5322#endif
5323
5324 // Finally, don't try to promote any allocas that new require re-splitting.
5325 // They have already been added to the worklist above.
5326 PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5327
5328 return true;
5329}
5330
5331/// Try to canonicalize a homogeneous struct partition to a vector type.
5332///
5333/// We can do this if all the elements of the struct are the same and the
5334/// corresponding vector has the same byte-level layout. This can sometimes
5335/// eliminate allocas because structs cannot get promoted to LLVM values, but
5336/// vectors can.
5337///
5338/// We only apply this transformation when all users of the partition are memory
5339/// intrinsics. Otherwise, if there is a load or store of some other type to the
5340/// partition, SROA would select that type.
5341///
5342/// Applying this transformation too early may hinder memcpyopt, which may
5343/// generate better code when eliminating allocas. For example, see
5344/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5345/// this before memcpyopt can initialize previously uninitialized memory when
5346/// the alloca gets promoted to an SSA value. For another example, see
5347/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5348/// this before memcpyopt can result in promoting an alloca so that we load a
5349/// temporary value instead of copying the temporary value into memory, whereas
5350/// memcpyopt eliminates the temporary altogether.
5351///
5352/// As such, we only apply this transformation after memcpyopt has run. We gate
5353/// this transformation by the "AggregateToVector" pass option.
5354static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
5355 Partition &P,
5356 const DataLayout &DL) {
5357 unsigned NumElts = STy->getNumElements();
5358
5359 Type *EltTy = STy->getElementType(N: 0);
5360 if (!llvm::all_equal(Range: STy->elements()))
5361 return nullptr;
5362
5363 bool IsIntegralPointerTy =
5364 EltTy->isPointerTy() && !DL.isNonIntegralPointerType(Ty: EltTy);
5365 if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5366 !IsIntegralPointerTy)
5367 return nullptr;
5368
5369 // Ensure the struct is tightly packed so that the bit-layout is the same as
5370 // the corresponding vector. For example, this prevents a miscompile for
5371 // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5372 // tightly packed elements and trailing padding.
5373 if (DL.getTypeSizeInBits(Ty: EltTy) != DL.getTypeAllocSizeInBits(Ty: EltTy))
5374 return nullptr;
5375
5376 auto *VTy = FixedVectorType::get(ElementType: EltTy, NumElts);
5377 TypeSize StructSize = DL.getStructLayout(Ty: STy)->getSizeInBytes();
5378 TypeSize VectorSize = DL.getTypeStoreSize(Ty: VTy);
5379 // After ruling out per-element padding, make sure a vector load/store
5380 // covers the same number of bytes as the struct layout.
5381 if (StructSize != VectorSize)
5382 return nullptr;
5383
5384 auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5385 if (S.isDead())
5386 return true;
5387 auto *U = S.getUse();
5388 if (!U)
5389 return true;
5390
5391 User *Usr = U->getUser();
5392 if (isa<LifetimeIntrinsic>(Val: Usr) || isa<DbgInfoIntrinsic>(Val: Usr))
5393 return true;
5394
5395 return isa<MemIntrinsic>(Val: Usr);
5396 };
5397
5398 for (const Slice &S : P)
5399 if (!IsIgnorableOrMemIntrinsicSlice(S))
5400 return nullptr;
5401
5402 for (const Slice *S : P.splitSliceTails())
5403 if (!IsIgnorableOrMemIntrinsicSlice(*S))
5404 return nullptr;
5405
5406 return VTy;
5407}
5408
5409/// Select a partition type for an alloca partition.
5410///
5411/// Try to compute a friendly type for this partition of the alloca. This
5412/// won't always succeed, in which case we fall back to a legal integer type
5413/// or an i8 array of an appropriate size.
5414///
5415/// \returns A tuple with the following elements:
5416/// - PartitionType: The computed type for this partition.
5417/// - IsIntegerWideningViable: True if integer widening promotion is used.
5418/// - VectorType: The vector type if vector promotion is used, otherwise
5419/// nullptr.
5420static std::tuple<Type *, bool, VectorType *>
5421selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
5422 LLVMContext &C, bool AggregateToVector) {
5423 auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5424 VectorType *SelectedVecTy, bool SelectedIntWidening) {
5425 LLVM_DEBUG({
5426 dbgs() << "selectPartitionType path=" << Path
5427 << " func=" << AI.getFunction()->getName() << " alloca=";
5428 if (AI.hasName())
5429 dbgs() << AI.getName();
5430 else
5431 dbgs() << "<unnamed>";
5432 dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5433 << ") size=" << P.size();
5434 if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5435 dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5436 if (SelectedTy)
5437 dbgs() << " chosen=" << *SelectedTy;
5438 if (SelectedVecTy)
5439 dbgs() << " vec=" << *SelectedVecTy;
5440 dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5441 });
5442 };
5443 // First check if the partition is viable for vector promotion.
5444 //
5445 // We prefer vector promotion over integer widening promotion when:
5446 // - The vector element type is a floating-point type.
5447 // - All the loads/stores to the alloca are vector loads/stores to the
5448 // entire alloca or load/store a single element of the vector.
5449 //
5450 // Otherwise when there is an integer vector with mixed type loads/stores we
5451 // prefer integer widening promotion because it's more likely the user is
5452 // doing bitwise arithmetic and we generate better code.
5453 VectorType *VecTy =
5454 isVectorPromotionViable(P, DL, VScale: AI.getFunction()->getVScaleValue());
5455 // If the vector element type is a floating-point type, we prefer vector
5456 // promotion. If the vector has one element, let the below code select
5457 // whether we promote with the vector or scalar.
5458 if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5459 VecTy->getElementCount().getFixedValue() > 1) {
5460 LogSelection("direct-fp-vecty", VecTy, VecTy, false);
5461 return {VecTy, false, VecTy};
5462 }
5463
5464 // Check if there is a common type that all slices of the partition use that
5465 // spans the partition.
5466 auto [CommonUseTy, LargestIntTy] =
5467 findCommonType(B: P.begin(), E: P.end(), EndOffset: P.endOffset());
5468 if (CommonUseTy) {
5469 TypeSize CommonUseSize = DL.getTypeAllocSize(Ty: CommonUseTy);
5470 if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5471 // We prefer vector promotion here because if vector promotion is viable
5472 // and there is a common type used, then it implies the second listed
5473 // condition for preferring vector promotion is true.
5474 if (VecTy) {
5475 LogSelection("common-type-vecty", VecTy, VecTy, false);
5476 return {VecTy, false, VecTy};
5477 }
5478 bool IntWiden = isIntegerWideningViable(P, AllocaTy: CommonUseTy, DL);
5479 LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
5480 return {CommonUseTy, IntWiden, nullptr};
5481 }
5482 }
5483
5484 // Can we find an appropriate subtype in the original allocated
5485 // type?
5486 if (Type *TypePartitionTy = getTypePartition(DL, Ty: AI.getAllocatedType(),
5487 Offset: P.beginOffset(), Size: P.size())) {
5488 // If the partition is an integer array that can be spanned by a legal
5489 // integer type, prefer to represent it as a legal integer type because
5490 // it's more likely to be promotable.
5491 if (TypePartitionTy->isArrayTy() &&
5492 TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5493 DL.isLegalInteger(Width: P.size() * 8))
5494 TypePartitionTy = Type::getIntNTy(C, N: P.size() * 8);
5495 // There was no common type used, so we prefer integer widening promotion.
5496 if (isIntegerWideningViable(P, AllocaTy: TypePartitionTy, DL)) {
5497 LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
5498 return {TypePartitionTy, true, nullptr};
5499 }
5500 if (VecTy) {
5501 LogSelection("type-partition-vecty", VecTy, VecTy, false);
5502 return {VecTy, false, VecTy};
5503 }
5504 // If we couldn't promote with TypePartitionTy, try with the largest
5505 // integer type used.
5506 if (LargestIntTy &&
5507 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size() &&
5508 isIntegerWideningViable(P, AllocaTy: LargestIntTy, DL)) {
5509 LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
5510 return {LargestIntTy, true, nullptr};
5511 }
5512
5513 // Try homogeneous struct to vector canonicalization when requested. Running
5514 // this too early can hide memcpy chains from MemCpyOpt.
5515 if (AggregateToVector) {
5516 if (auto *STy = dyn_cast<StructType>(Val: TypePartitionTy)) {
5517 if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5518 LogSelection("struct-fallback-vecty", VTy, nullptr, false);
5519 return {VTy, false, nullptr};
5520 }
5521 }
5522 }
5523
5524 // Fallback to TypePartitionTy and we probably won't promote.
5525 LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
5526 return {TypePartitionTy, false, nullptr};
5527 }
5528
5529 // Select the largest integer type used if it spans the partition.
5530 if (LargestIntTy &&
5531 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size()) {
5532 LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
5533 return {LargestIntTy, false, nullptr};
5534 }
5535
5536 // Select a legal integer type if it spans the partition.
5537 if (DL.isLegalInteger(Width: P.size() * 8)) {
5538 Type *IntTy = Type::getIntNTy(C, N: P.size() * 8);
5539 LogSelection("legal-int-fallback", IntTy, nullptr, false);
5540 return {IntTy, false, nullptr};
5541 }
5542
5543 // Fallback to an i8 array.
5544 Type *ArrayTy = ArrayType::get(ElementType: Type::getInt8Ty(C), NumElements: P.size());
5545 LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
5546 return {ArrayTy, false, nullptr};
5547}
5548
5549/// Rewrite an alloca partition's users.
5550///
5551/// This routine drives both of the rewriting goals of the SROA pass. It tries
5552/// to rewrite uses of an alloca partition to be conducive for SSA value
5553/// promotion. If the partition needs a new, more refined alloca, this will
5554/// build that new alloca, preserving as much type information as possible, and
5555/// rewrite the uses of the old alloca to point at the new one and have the
5556/// appropriate new offsets. It also evaluates how successful the rewrite was
5557/// at enabling promotion and if it was successful queues the alloca to be
5558/// promoted.
5559std::pair<AllocaInst *, uint64_t>
5560SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5561 const DataLayout &DL = AI.getDataLayout();
5562 // Select the type for the new alloca that spans the partition.
5563 auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5564 selectPartitionType(P, DL, AI, C&: *C, AggregateToVector);
5565
5566 // Check for the case where we're going to rewrite to a new alloca of the
5567 // exact same type as the original, and with the same access offsets. In that
5568 // case, re-use the existing alloca, but still run through the rewriter to
5569 // perform phi and select speculation.
5570 // P.beginOffset() can be non-zero even with the same type in a case with
5571 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5572 AllocaInst *NewAI;
5573 if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == 0) {
5574 NewAI = &AI;
5575 // FIXME: We should be able to bail at this point with "nothing changed".
5576 // FIXME: We might want to defer PHI speculation until after here.
5577 // FIXME: return nullptr;
5578 } else {
5579 // Make sure the alignment is compatible with P.beginOffset().
5580 const Align Alignment = commonAlignment(A: AI.getAlign(), Offset: P.beginOffset());
5581 NewAI =
5582 new AllocaInst(PartitionTy, AI.getAddressSpace(), nullptr, Alignment,
5583 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()),
5584 AI.getIterator());
5585 tryEnforceAlignment(V: NewAI, PrefAlign: DL.getPrefTypeAlign(Ty: PartitionTy), DL);
5586 // Copy the old AI debug location over to the new one.
5587 NewAI->setDebugLoc(AI.getDebugLoc());
5588 ++NumNewAllocas;
5589 }
5590
5591 LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5592 << "," << P.endOffset() << ") to: " << *NewAI << "\n");
5593
5594 // Track the high watermark on the worklist as it is only relevant for
5595 // promoted allocas. We will reset it to this point if the alloca is not in
5596 // fact scheduled for promotion.
5597 unsigned PPWOldSize = PostPromotionWorklist.size();
5598 unsigned NumUses = 0;
5599 SmallSetVector<PHINode *, 8> PHIUsers;
5600 SmallSetVector<SelectInst *, 8> SelectUsers;
5601
5602 AllocaSliceRewriter Rewriter(
5603 DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5604 IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5605 bool Promotable = true;
5606 // Check whether we can have tree-structured merge.
5607 if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5608 NumUses += DeletedValues->size() + 1;
5609 for (Value *V : *DeletedValues)
5610 DeadInsts.push_back(Elt: V);
5611 } else {
5612 for (Slice *S : P.splitSliceTails()) {
5613 Promotable &= Rewriter.visit(I: S);
5614 ++NumUses;
5615 }
5616 for (Slice &S : P) {
5617 Promotable &= Rewriter.visit(I: &S);
5618 ++NumUses;
5619 }
5620 }
5621
5622 NumAllocaPartitionUses += NumUses;
5623 MaxUsesPerAllocaPartition.updateMax(V: NumUses);
5624
5625 // Now that we've processed all the slices in the new partition, check if any
5626 // PHIs or Selects would block promotion.
5627 for (PHINode *PHI : PHIUsers)
5628 if (!isSafePHIToSpeculate(PN&: *PHI)) {
5629 Promotable = false;
5630 PHIUsers.clear();
5631 SelectUsers.clear();
5632 break;
5633 }
5634
5635 SmallVector<std::pair<SelectInst *, RewriteableMemOps>, 2>
5636 NewSelectsToRewrite;
5637 NewSelectsToRewrite.reserve(N: SelectUsers.size());
5638 for (SelectInst *Sel : SelectUsers) {
5639 std::optional<RewriteableMemOps> Ops =
5640 isSafeSelectToSpeculate(SI&: *Sel, PreserveCFG);
5641 if (!Ops) {
5642 Promotable = false;
5643 PHIUsers.clear();
5644 SelectUsers.clear();
5645 NewSelectsToRewrite.clear();
5646 break;
5647 }
5648 NewSelectsToRewrite.emplace_back(Args: std::make_pair(x&: Sel, y&: *Ops));
5649 }
5650
5651 if (Promotable) {
5652 for (Use *U : AS.getDeadUsesIfPromotable()) {
5653 auto *OldInst = dyn_cast<Instruction>(Val: U->get());
5654 Value::dropDroppableUse(U&: *U);
5655 if (OldInst)
5656 if (isInstructionTriviallyDead(I: OldInst))
5657 DeadInsts.push_back(Elt: OldInst);
5658 }
5659 if (PHIUsers.empty() && SelectUsers.empty()) {
5660 // Promote the alloca.
5661 PromotableAllocas.insert(X: NewAI);
5662 } else {
5663 // If we have either PHIs or Selects to speculate, add them to those
5664 // worklists and re-queue the new alloca so that we promote in on the
5665 // next iteration.
5666 SpeculatablePHIs.insert_range(R&: PHIUsers);
5667 SelectsToRewrite.reserve(NumEntries: SelectsToRewrite.size() +
5668 NewSelectsToRewrite.size());
5669 for (auto &&KV : llvm::make_range(
5670 x: std::make_move_iterator(i: NewSelectsToRewrite.begin()),
5671 y: std::make_move_iterator(i: NewSelectsToRewrite.end())))
5672 SelectsToRewrite.insert(KV: std::move(KV));
5673 Worklist.insert(X: NewAI);
5674 }
5675 } else {
5676 // Drop any post-promotion work items if promotion didn't happen.
5677 while (PostPromotionWorklist.size() > PPWOldSize)
5678 PostPromotionWorklist.pop_back();
5679
5680 // We couldn't promote and we didn't create a new partition, nothing
5681 // happened.
5682 if (NewAI == &AI)
5683 return {nullptr, 0};
5684
5685 // If we can't promote the alloca, iterate on it to check for new
5686 // refinements exposed by splitting the current alloca. Don't iterate on an
5687 // alloca which didn't actually change and didn't get promoted.
5688 Worklist.insert(X: NewAI);
5689 }
5690
5691 return {NewAI, DL.getTypeSizeInBits(Ty: PartitionTy).getFixedValue()};
5692}
5693
5694// There isn't a shared interface to get the "address" parts out of a
5695// dbg.declare and dbg.assign, so provide some wrappers.
5696bool isKillAddress(const DbgVariableRecord *DVR) {
5697 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5698 return DVR->isKillAddress();
5699 return DVR->isKillLocation();
5700}
5701
5702const DIExpression *getAddressExpression(const DbgVariableRecord *DVR) {
5703 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5704 return DVR->getAddressExpression();
5705 return DVR->getExpression();
5706}
5707
5708/// Create or replace an existing fragment in a DIExpression with \p Frag.
5709/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5710/// operation, add \p BitExtractOffset to the offset part.
5711///
5712/// Returns the new expression, or nullptr if this fails (see details below).
5713///
5714/// This function is similar to DIExpression::createFragmentExpression except
5715/// for 3 important distinctions:
5716/// 1. The new fragment isn't relative to an existing fragment.
5717/// 2. It assumes the computed location is a memory location. This means we
5718/// don't need to perform checks that creating the fragment preserves the
5719/// expression semantics.
5720/// 3. Existing extract_bits are modified independently of fragment changes
5721/// using \p BitExtractOffset. A change to the fragment offset or size
5722/// may affect a bit extract. But a bit extract offset can change
5723/// independently of the fragment dimensions.
5724///
5725/// Returns the new expression, or nullptr if one couldn't be created.
5726/// Ideally this is only used to signal that a bit-extract has become
5727/// zero-sized (and thus the new debug record has no size and can be
5728/// dropped), however, it fails for other reasons too - see the FIXME below.
5729///
5730/// FIXME: To keep the change that introduces this function NFC it bails
5731/// in some situations unecessarily, e.g. when fragment and bit extract
5732/// sizes differ.
5733static DIExpression *createOrReplaceFragment(const DIExpression *Expr,
5734 DIExpression::FragmentInfo Frag,
5735 int64_t BitExtractOffset) {
5736 SmallVector<uint64_t, 8> Ops;
5737 bool HasFragment = false;
5738 bool HasBitExtract = false;
5739
5740 for (auto &Op : Expr->expr_ops()) {
5741 if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5742 HasFragment = true;
5743 continue;
5744 }
5745 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val: Op)) {
5746 HasBitExtract = true;
5747 int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5748 int64_t ExtractSizeInBits = Extract.getSizeInBits();
5749
5750 // DIExpression::createFragmentExpression doesn't know how to handle
5751 // a fragment that is smaller than the extract. Copy the behaviour
5752 // (bail) to avoid non-NFC changes.
5753 // FIXME: Don't do this.
5754 if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5755 return nullptr;
5756
5757 assert(BitExtractOffset <= 0);
5758 int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5759
5760 // DIExpression::createFragmentExpression doesn't know what to do
5761 // if the new extract starts "outside" the existing one. Copy the
5762 // behaviour (bail) to avoid non-NFC changes.
5763 // FIXME: Don't do this.
5764 if (AdjustedOffset < 0)
5765 return nullptr;
5766
5767 Ops.push_back(Elt: Op.getOp());
5768 Ops.push_back(Elt: std::max<int64_t>(a: 0, b: AdjustedOffset));
5769 Ops.push_back(Elt: ExtractSizeInBits);
5770 continue;
5771 }
5772 Op.appendToVector(V&: Ops);
5773 }
5774
5775 // Unsupported by createFragmentExpression, so don't support it here yet to
5776 // preserve NFC-ness.
5777 if (HasFragment && HasBitExtract)
5778 return nullptr;
5779
5780 if (!HasBitExtract) {
5781 Ops.push_back(Elt: dwarf::DW_OP_LLVM_fragment);
5782 Ops.push_back(Elt: Frag.OffsetInBits);
5783 Ops.push_back(Elt: Frag.SizeInBits);
5784 }
5785 return DIExpression::get(Context&: Expr->getContext(), Elements: Ops);
5786}
5787
5788/// Insert a new DbgRecord.
5789/// \p Orig Original to copy record type, debug loc and variable from, and
5790/// additionally value and value expression for dbg_assign records.
5791/// \p NewAddr Location's new base address.
5792/// \p NewAddrExpr New expression to apply to address.
5793/// \p BeforeInst Insert position.
5794/// \p NewFragment New fragment (absolute, non-relative).
5795/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
5796static void
5797insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr,
5798 DIExpression *NewAddrExpr, Instruction *BeforeInst,
5799 std::optional<DIExpression::FragmentInfo> NewFragment,
5800 int64_t BitExtractAdjustment) {
5801 (void)DIB;
5802
5803 // A dbg_assign puts fragment info in the value expression only. The address
5804 // expression has already been built: NewAddrExpr. A dbg_declare puts the
5805 // new fragment info into NewAddrExpr (as it only has one expression).
5806 DIExpression *NewFragmentExpr =
5807 Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
5808 if (NewFragment)
5809 NewFragmentExpr = createOrReplaceFragment(Expr: NewFragmentExpr, Frag: *NewFragment,
5810 BitExtractOffset: BitExtractAdjustment);
5811 if (!NewFragmentExpr)
5812 return;
5813
5814 if (Orig->isDbgDeclare()) {
5815 DbgVariableRecord *DVR = DbgVariableRecord::createDVRDeclare(
5816 Address: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5817 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5818 Here: BeforeInst->getIterator());
5819 return;
5820 }
5821
5822 if (Orig->isDbgValue()) {
5823 DbgVariableRecord *DVR = DbgVariableRecord::createDbgVariableRecord(
5824 Location: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5825 // Drop debug information if the expression doesn't start with a
5826 // DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
5827 // describes the address of alloca rather than the value inside the alloca.
5828 if (!NewFragmentExpr->startsWithDeref())
5829 DVR->setKillAddress();
5830 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5831 Here: BeforeInst->getIterator());
5832 return;
5833 }
5834
5835 // Apply a DIAssignID to the store if it doesn't already have it.
5836 if (!NewAddr->hasMetadata(KindID: LLVMContext::MD_DIAssignID)) {
5837 NewAddr->setMetadata(KindID: LLVMContext::MD_DIAssignID,
5838 Node: DIAssignID::getDistinct(Context&: NewAddr->getContext()));
5839 }
5840
5841 DbgVariableRecord *NewAssign = DbgVariableRecord::createLinkedDVRAssign(
5842 LinkedInstr: NewAddr, Val: Orig->getValue(), Variable: Orig->getVariable(), Expression: NewFragmentExpr, Address: NewAddr,
5843 AddressExpression: NewAddrExpr, DI: Orig->getDebugLoc());
5844 LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
5845 (void)NewAssign;
5846}
5847
5848/// Walks the slices of an alloca and form partitions based on them,
5849/// rewriting each of their uses.
5850bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
5851 if (AS.begin() == AS.end())
5852 return false;
5853
5854 unsigned NumPartitions = 0;
5855 bool Changed = false;
5856 const DataLayout &DL = AI.getModule()->getDataLayout();
5857
5858 // First try to pre-split loads and stores.
5859 Changed |= presplitLoadsAndStores(AI, AS);
5860
5861 // Now that we have identified any pre-splitting opportunities,
5862 // mark loads and stores unsplittable except for the following case.
5863 // We leave a slice splittable if all other slices are disjoint or fully
5864 // included in the slice, such as whole-alloca loads and stores.
5865 // If we fail to split these during pre-splitting, we want to force them
5866 // to be rewritten into a partition.
5867 bool IsSorted = true;
5868
5869 uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
5870 // We can split at the begin and end offsets of each slice, but only if those
5871 // offsets don't lie inside another slice. Because slices are ordered by
5872 // increasing begin offset, and then decreasing end offset, we can consider
5873 // the slices as being split up into sets with the same begin offset where we
5874 // can ignore every slice except the first (the begin offset will already be
5875 // handled as the begin offset of the set, and the end offset we know is not a
5876 // splittable offset as it's inside the first slice of the set).
5877 SparseBitVector<> SplittableOffset;
5878 uint64_t CurBegin = 0, CurEnd = 0;
5879 for (Slice &S : AS) {
5880 // Check if we have a new set of slices
5881 if (S.beginOffset() > CurBegin || S.endOffset() > CurEnd) {
5882 // If the start isn't inside the previous set it's splittable
5883 if (S.beginOffset() >= CurEnd) {
5884 SplittableOffset.set(S.beginOffset());
5885 }
5886 // If the previous end is inside this slice then remove it
5887 if (CurEnd > S.beginOffset() && CurEnd < S.endOffset()) {
5888 SplittableOffset.reset(Idx: CurEnd);
5889 }
5890 CurBegin = S.beginOffset();
5891 // If the end offset isn't inside the previous set it's splittable. We
5892 // also don't update the end offset in that case, as the next set may also
5893 // be inside the previous set.
5894 if (S.endOffset() > CurEnd) {
5895 CurEnd = S.endOffset();
5896 SplittableOffset.set(CurEnd);
5897 }
5898 }
5899 }
5900
5901 for (Slice &S : AS) {
5902 if (!S.isSplittable())
5903 continue;
5904
5905 if ((S.beginOffset() > AllocaSize ||
5906 SplittableOffset.test(Idx: S.beginOffset())) &&
5907 (S.endOffset() > AllocaSize || SplittableOffset.test(Idx: S.endOffset())))
5908 continue;
5909
5910 if (isa<LoadInst>(Val: S.getUse()->getUser()) ||
5911 isa<StoreInst>(Val: S.getUse()->getUser())) {
5912 S.makeUnsplittable();
5913 IsSorted = false;
5914 }
5915 }
5916
5917 if (!IsSorted)
5918 llvm::stable_sort(Range&: AS);
5919
5920 /// Describes the allocas introduced by rewritePartition in order to migrate
5921 /// the debug info.
5922 struct Fragment {
5923 AllocaInst *Alloca;
5924 uint64_t Offset;
5925 uint64_t Size;
5926 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
5927 : Alloca(AI), Offset(O), Size(S) {}
5928 };
5929 SmallVector<Fragment, 4> Fragments;
5930
5931 // Rewrite each partition.
5932 for (auto &P : AS.partitions()) {
5933 auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
5934 if (NewAI) {
5935 Changed = true;
5936 if (NewAI != &AI) {
5937 uint64_t SizeOfByte = 8;
5938 // Don't include any padding.
5939 uint64_t Size = std::min(a: ActiveBits, b: P.size() * SizeOfByte);
5940 Fragments.push_back(
5941 Elt: Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
5942 }
5943 }
5944 ++NumPartitions;
5945 }
5946
5947 NumAllocaPartitions += NumPartitions;
5948 MaxPartitionsPerAlloca.updateMax(V: NumPartitions);
5949
5950 // Migrate debug information from the old alloca to the new alloca(s)
5951 // and the individual partitions.
5952 auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
5953 // Can't overlap with undef memory.
5954 if (isKillAddress(DVR: DbgVariable))
5955 return;
5956
5957 const Value *DbgPtr = DbgVariable->getAddress();
5958 DIExpression::FragmentInfo VarFrag =
5959 DbgVariable->getFragmentOrEntireVariable();
5960 // Get the address expression constant offset if one exists and the ops
5961 // that come after it.
5962 int64_t CurrentExprOffsetInBytes = 0;
5963 SmallVector<uint64_t> PostOffsetOps;
5964 if (!getAddressExpression(DVR: DbgVariable)
5965 ->extractLeadingOffset(OffsetInBytes&: CurrentExprOffsetInBytes, RemainingOps&: PostOffsetOps))
5966 return; // Couldn't interpret this DIExpression - drop the var.
5967
5968 // Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
5969 int64_t ExtractOffsetInBits = 0;
5970 for (auto Op : getAddressExpression(DVR: DbgVariable)->expr_ops()) {
5971 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val&: Op)) {
5972 ExtractOffsetInBits = Extract.getOffsetInBits();
5973 break;
5974 }
5975 }
5976
5977 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
5978 for (auto Fragment : Fragments) {
5979 int64_t OffsetFromLocationInBits;
5980 std::optional<DIExpression::FragmentInfo> NewDbgFragment;
5981 // Find the variable fragment that the new alloca slice covers.
5982 // Drop debug info for this variable fragment if we can't compute an
5983 // intersect between it and the alloca slice.
5984 if (!DIExpression::calculateFragmentIntersect(
5985 DL, SliceStart: &AI, SliceOffsetInBits: Fragment.Offset, SliceSizeInBits: Fragment.Size, DbgPtr,
5986 DbgPtrOffsetInBits: CurrentExprOffsetInBytes * 8, DbgExtractOffsetInBits: ExtractOffsetInBits, VarFrag,
5987 Result&: NewDbgFragment, OffsetFromLocationInBits))
5988 continue; // Do not migrate this fragment to this slice.
5989
5990 // Zero sized fragment indicates there's no intersect between the variable
5991 // fragment and the alloca slice. Skip this slice for this variable
5992 // fragment.
5993 if (NewDbgFragment && !NewDbgFragment->SizeInBits)
5994 continue; // Do not migrate this fragment to this slice.
5995
5996 // No fragment indicates DbgVariable's variable or fragment exactly
5997 // overlaps the slice; copy its fragment (or nullopt if there isn't one).
5998 if (!NewDbgFragment)
5999 NewDbgFragment = DbgVariable->getFragment();
6000
6001 // Reduce the new expression offset by the bit-extract offset since
6002 // we'll be keeping that.
6003 int64_t OffestFromNewAllocaInBits =
6004 OffsetFromLocationInBits - ExtractOffsetInBits;
6005 // We need to adjust an existing bit extract if the offset expression
6006 // can't eat the slack (i.e., if the new offset would be negative).
6007 int64_t BitExtractOffset =
6008 std::min<int64_t>(a: 0, b: OffestFromNewAllocaInBits);
6009 // The magnitude of a negative value indicates the number of bits into
6010 // the existing variable fragment that the memory region begins. The new
6011 // variable fragment already excludes those bits - the new DbgPtr offset
6012 // only needs to be applied if it's positive.
6013 OffestFromNewAllocaInBits =
6014 std::max(a: int64_t(0), b: OffestFromNewAllocaInBits);
6015
6016 // Rebuild the expression:
6017 // {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
6018 // Add NewDbgFragment later, because dbg.assigns don't want it in the
6019 // address expression but the value expression instead.
6020 DIExpression *NewExpr = DIExpression::get(Context&: AI.getContext(), Elements: PostOffsetOps);
6021 if (OffestFromNewAllocaInBits > 0) {
6022 int64_t OffsetInBytes = (OffestFromNewAllocaInBits + 7) / 8;
6023 NewExpr = DIExpression::prepend(Expr: NewExpr, /*flags=*/Flags: 0, Offset: OffsetInBytes);
6024 }
6025
6026 // Remove any existing intrinsics on the new alloca describing
6027 // the variable fragment.
6028 auto RemoveOne = [DbgVariable](auto *OldDII) {
6029 auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
6030 return LHS->getVariable() == RHS->getVariable() &&
6031 LHS->getDebugLoc()->getInlinedAt() ==
6032 RHS->getDebugLoc()->getInlinedAt();
6033 };
6034 if (SameVariableFragment(OldDII, DbgVariable))
6035 OldDII->eraseFromParent();
6036 };
6037 for_each(Range: findDVRDeclares(V: Fragment.Alloca), F: RemoveOne);
6038 for_each(Range: findDVRValues(V: Fragment.Alloca), F: RemoveOne);
6039 insertNewDbgInst(DIB, Orig: DbgVariable, NewAddr: Fragment.Alloca, NewAddrExpr: NewExpr, BeforeInst: &AI,
6040 NewFragment: NewDbgFragment, BitExtractAdjustment: BitExtractOffset);
6041 }
6042 };
6043
6044 // Migrate debug information from the old alloca to the new alloca(s)
6045 // and the individual partitions.
6046 for_each(Range: findDVRDeclares(V: &AI), F: MigrateOne);
6047 for_each(Range: findDVRValues(V: &AI), F: MigrateOne);
6048 for_each(Range: at::getDVRAssignmentMarkers(Inst: &AI), F: MigrateOne);
6049
6050 return Changed;
6051}
6052
6053/// Clobber a use with poison, deleting the used value if it becomes dead.
6054void SROA::clobberUse(Use &U) {
6055 Value *OldV = U;
6056 // Replace the use with an poison value.
6057 U = PoisonValue::get(T: OldV->getType());
6058
6059 // Check for this making an instruction dead. We have to garbage collect
6060 // all the dead instructions to ensure the uses of any alloca end up being
6061 // minimal.
6062 if (Instruction *OldI = dyn_cast<Instruction>(Val: OldV))
6063 if (isInstructionTriviallyDead(I: OldI)) {
6064 DeadInsts.push_back(Elt: OldI);
6065 }
6066}
6067
6068/// A basic LoadAndStorePromoter that does not remove store nodes.
6069class BasicLoadAndStorePromoter : public LoadAndStorePromoter {
6070public:
6071 BasicLoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S,
6072 Type *ZeroType)
6073 : LoadAndStorePromoter(Insts, S), ZeroType(ZeroType) {}
6074 bool shouldDelete(Instruction *I) const override {
6075 return !isa<StoreInst>(Val: I) && !isa<AllocaInst>(Val: I);
6076 }
6077
6078 Value *getValueToUseForAlloca(Instruction *I) const override {
6079 return UndefValue::get(T: ZeroType);
6080 }
6081
6082private:
6083 Type *ZeroType;
6084};
6085
6086bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6087 // Look through each "partition", looking for slices with the same start/end
6088 // that do not overlap with any before them. The slices are sorted by
6089 // increasing beginOffset. We don't use AS.partitions(), as it will use a more
6090 // sophisticated algorithm that takes splittable slices into account.
6091 LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6092 bool AllSameAndValid = true;
6093 Type *PartitionType = nullptr;
6094 SmallVector<Instruction *> Insts;
6095 uint64_t BeginOffset = 0;
6096 uint64_t EndOffset = 0;
6097
6098 auto Flush = [&]() {
6099 if (AllSameAndValid && !Insts.empty()) {
6100 LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6101 << EndOffset << ")\n");
6102 SmallVector<PHINode *, 4> NewPHIs;
6103 SSAUpdater SSA(&NewPHIs);
6104 Insts.push_back(Elt: &AI);
6105 BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6106 Promoter.run(Insts);
6107 }
6108 AllSameAndValid = true;
6109 PartitionType = nullptr;
6110 Insts.clear();
6111 };
6112
6113 for (Slice &S : AS) {
6114 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
6115 if (isAssumeLikeIntrinsic(I: User)) {
6116 LLVM_DEBUG({
6117 dbgs() << "Ignoring slice: ";
6118 AS.print(dbgs(), &S);
6119 });
6120 continue;
6121 }
6122 if (S.beginOffset() >= EndOffset) {
6123 Flush();
6124 BeginOffset = S.beginOffset();
6125 EndOffset = S.endOffset();
6126 } else if (S.beginOffset() != BeginOffset || S.endOffset() != EndOffset) {
6127 if (AllSameAndValid) {
6128 LLVM_DEBUG({
6129 dbgs() << "Slice does not match range [" << BeginOffset << ", "
6130 << EndOffset << ")";
6131 AS.print(dbgs(), &S);
6132 });
6133 AllSameAndValid = false;
6134 }
6135 EndOffset = std::max(a: EndOffset, b: S.endOffset());
6136 continue;
6137 }
6138
6139 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
6140 Type *UserTy = LI->getType();
6141 // LoadAndStorePromoter requires all the types to be the same.
6142 if (!LI->isSimple() || (PartitionType && UserTy != PartitionType))
6143 AllSameAndValid = false;
6144 PartitionType = UserTy;
6145 Insts.push_back(Elt: User);
6146 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
6147 Type *UserTy = SI->getValueOperand()->getType();
6148 if (!SI->isSimple() || (PartitionType && UserTy != PartitionType))
6149 AllSameAndValid = false;
6150 PartitionType = UserTy;
6151 Insts.push_back(Elt: User);
6152 } else {
6153 AllSameAndValid = false;
6154 }
6155 }
6156
6157 Flush();
6158 return true;
6159}
6160
6161/// Analyze an alloca for SROA.
6162///
6163/// This analyzes the alloca to ensure we can reason about it, builds
6164/// the slices of the alloca, and then hands it off to be split and
6165/// rewritten as needed.
6166std::pair<bool /*Changed*/, bool /*CFGChanged*/>
6167SROA::runOnAlloca(AllocaInst &AI) {
6168 bool Changed = false;
6169 bool CFGChanged = false;
6170
6171 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6172 ++NumAllocasAnalyzed;
6173
6174 // Special case dead allocas, as they're trivial.
6175 if (AI.use_empty()) {
6176 AI.eraseFromParent();
6177 Changed = true;
6178 return {Changed, CFGChanged};
6179 }
6180 const DataLayout &DL = AI.getDataLayout();
6181
6182 // Skip alloca forms that this analysis can't handle.
6183 std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6184 if (AI.isArrayAllocation() || !Size || Size->isScalable() || Size->isZero())
6185 return {Changed, CFGChanged};
6186
6187 // First, split any FCA loads and stores touching this alloca to promote
6188 // better splitting and promotion opportunities.
6189 IRBuilderTy IRB(&AI);
6190 AggLoadStoreRewriter AggRewriter(DL, IRB);
6191 Changed |= AggRewriter.rewrite(I&: AI);
6192
6193 // Build the slices using a recursive instruction-visiting builder.
6194 AllocaSlices AS(DL, AI);
6195 LLVM_DEBUG(AS.print(dbgs()));
6196 if (AS.isEscaped())
6197 return {Changed, CFGChanged};
6198
6199 if (AS.isEscapedReadOnly()) {
6200 Changed |= propagateStoredValuesToLoads(AI, AS);
6201 return {Changed, CFGChanged};
6202 }
6203
6204 // Delete all the dead users of this alloca before splitting and rewriting it.
6205 for (Instruction *DeadUser : AS.getDeadUsers()) {
6206 // Free up everything used by this instruction.
6207 for (Use &DeadOp : DeadUser->operands())
6208 clobberUse(U&: DeadOp);
6209
6210 // Now replace the uses of this instruction.
6211 DeadUser->replaceAllUsesWith(V: PoisonValue::get(T: DeadUser->getType()));
6212
6213 // And mark it for deletion.
6214 DeadInsts.push_back(Elt: DeadUser);
6215 Changed = true;
6216 }
6217 for (Use *DeadOp : AS.getDeadOperands()) {
6218 clobberUse(U&: *DeadOp);
6219 Changed = true;
6220 }
6221
6222 // No slices to split. Leave the dead alloca for a later pass to clean up.
6223 if (AS.begin() == AS.end())
6224 return {Changed, CFGChanged};
6225
6226 Changed |= splitAlloca(AI, AS);
6227
6228 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6229 while (!SpeculatablePHIs.empty())
6230 speculatePHINodeLoads(IRB, PN&: *SpeculatablePHIs.pop_back_val());
6231
6232 LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6233 auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6234 while (!RemainingSelectsToRewrite.empty()) {
6235 const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6236 CFGChanged |=
6237 rewriteSelectInstMemOps(SI&: *K, Ops: V, IRB, DTU: PreserveCFG ? nullptr : DTU);
6238 }
6239
6240 return {Changed, CFGChanged};
6241}
6242
6243/// Delete the dead instructions accumulated in this run.
6244///
6245/// Recursively deletes the dead instructions we've accumulated. This is done
6246/// at the very end to maximize locality of the recursive delete and to
6247/// minimize the problems of invalidated instruction pointers as such pointers
6248/// are used heavily in the intermediate stages of the algorithm.
6249///
6250/// We also record the alloca instructions deleted here so that they aren't
6251/// subsequently handed to mem2reg to promote.
6252bool SROA::deleteDeadInstructions(
6253 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6254 bool Changed = false;
6255 while (!DeadInsts.empty()) {
6256 Instruction *I = dyn_cast_or_null<Instruction>(Val: DeadInsts.pop_back_val());
6257 if (!I)
6258 continue;
6259 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6260
6261 // If the instruction is an alloca, find the possible dbg.declare connected
6262 // to it, and remove it too. We must do this before calling RAUW or we will
6263 // not be able to find it.
6264 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: I)) {
6265 DeletedAllocas.insert(Ptr: AI);
6266 for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6267 OldDII->eraseFromParent();
6268 }
6269
6270 at::deleteAssignmentMarkers(Inst: I);
6271 I->replaceAllUsesWith(V: UndefValue::get(T: I->getType()));
6272
6273 for (Use &Operand : I->operands())
6274 if (Instruction *U = dyn_cast<Instruction>(Val&: Operand)) {
6275 // Zero out the operand and see if it becomes trivially dead.
6276 Operand = nullptr;
6277 if (isInstructionTriviallyDead(I: U))
6278 DeadInsts.push_back(Elt: U);
6279 }
6280
6281 ++NumDeleted;
6282 I->eraseFromParent();
6283 Changed = true;
6284 }
6285 return Changed;
6286}
6287/// Promote the allocas, using the best available technique.
6288///
6289/// This attempts to promote whatever allocas have been identified as viable in
6290/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6291/// This function returns whether any promotion occurred.
6292bool SROA::promoteAllocas() {
6293 if (PromotableAllocas.empty())
6294 return false;
6295
6296 if (SROASkipMem2Reg) {
6297 LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6298 } else {
6299 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6300 NumPromoted += PromotableAllocas.size();
6301 PromoteMemToReg(Allocas: PromotableAllocas.getArrayRef(), DT&: DTU->getDomTree(), AC);
6302 }
6303
6304 PromotableAllocas.clear();
6305 return true;
6306}
6307
6308std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
6309 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6310
6311 const DataLayout &DL = F.getDataLayout();
6312 BasicBlock &EntryBB = F.getEntryBlock();
6313 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(x: EntryBB.end());
6314 I != E; ++I) {
6315 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val&: I)) {
6316 std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6317 if (Size && Size->isScalable() && isAllocaPromotable(AI))
6318 PromotableAllocas.insert(X: AI);
6319 else
6320 Worklist.insert(X: AI);
6321 }
6322 }
6323
6324 bool Changed = false;
6325 bool CFGChanged = false;
6326 // A set of deleted alloca instruction pointers which should be removed from
6327 // the list of promotable allocas.
6328 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
6329
6330 do {
6331 while (!Worklist.empty()) {
6332 auto [IterationChanged, IterationCFGChanged] =
6333 runOnAlloca(AI&: *Worklist.pop_back_val());
6334 Changed |= IterationChanged;
6335 CFGChanged |= IterationCFGChanged;
6336
6337 Changed |= deleteDeadInstructions(DeletedAllocas);
6338
6339 // Remove the deleted allocas from various lists so that we don't try to
6340 // continue processing them.
6341 if (!DeletedAllocas.empty()) {
6342 Worklist.set_subtract(DeletedAllocas);
6343 PostPromotionWorklist.set_subtract(DeletedAllocas);
6344 PromotableAllocas.set_subtract(DeletedAllocas);
6345 DeletedAllocas.clear();
6346 }
6347 }
6348
6349 Changed |= promoteAllocas();
6350
6351 Worklist = PostPromotionWorklist;
6352 PostPromotionWorklist.clear();
6353 } while (!Worklist.empty());
6354
6355 assert((!CFGChanged || Changed) && "Can not only modify the CFG.");
6356 assert((!CFGChanged || !PreserveCFG) &&
6357 "Should not have modified the CFG when told to preserve it.");
6358
6359 if (Changed && isAssignmentTrackingEnabled(M: *F.getParent())) {
6360 for (auto &BB : F) {
6361 RemoveRedundantDbgInstrs(BB: &BB);
6362 }
6363 }
6364
6365 return {Changed, CFGChanged};
6366}
6367
6368PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
6369 DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
6370 AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
6371 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6372 auto [Changed, CFGChanged] =
6373 SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6374 if (!Changed)
6375 return PreservedAnalyses::all();
6376 PreservedAnalyses PA;
6377 if (!CFGChanged)
6378 PA.preserveSet<CFGAnalyses>();
6379 PA.preserve<DominatorTreeAnalysis>();
6380 return PA;
6381}
6382
6383void SROAPass::printPipeline(
6384 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6385 static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
6386 OS, MapClassName2PassName);
6387 OS << '<'
6388 << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6389 : "modify-cfg");
6390 if (Options.AggregateToVector)
6391 OS << ";aggregate-to-vector";
6392 OS << '>';
6393}
6394
6395SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
6396
6397namespace {
6398
6399/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6400class SROALegacyPass : public FunctionPass {
6401 SROAOptions Options;
6402
6403public:
6404 static char ID;
6405
6406 SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
6407 : FunctionPass(ID), Options(Options) {
6408 initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
6409 }
6410
6411 bool runOnFunction(Function &F) override {
6412 if (skipFunction(F))
6413 return false;
6414
6415 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6416 AssumptionCache &AC =
6417 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6418 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6419 auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6420 return Changed;
6421 }
6422
6423 void getAnalysisUsage(AnalysisUsage &AU) const override {
6424 AU.addRequired<AssumptionCacheTracker>();
6425 AU.addRequired<DominatorTreeWrapperPass>();
6426 AU.addPreserved<GlobalsAAWrapperPass>();
6427 AU.addPreserved<DominatorTreeWrapperPass>();
6428 }
6429
6430 StringRef getPassName() const override { return "SROA"; }
6431};
6432
6433} // end anonymous namespace
6434
6435char SROALegacyPass::ID = 0;
6436
6437FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
6438 return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
6439 : SROAOptions::ModifyCFG,
6440 AggregateToVector));
6441}
6442
6443INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6444 "Scalar Replacement Of Aggregates", false, false)
6445INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6446INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
6447INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
6448 false, false)
6449