1//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This transformation implements the well known scalar replacement of
10/// aggregates transformation. It tries to identify promotable elements of an
11/// aggregate alloca, and promote them to registers. It will also try to
12/// convert uses of an element (or set of elements) of an alloca into a vector
13/// or bitfield-style integer scalar if appropriate.
14///
15/// It works to do this with minimal slicing of the alloca so that regions
16/// which are merely transferred in and out of external memory remain unchanged
17/// and are not decomposed to scalar code.
18///
19/// Because this also performs alloca promotion, it can be thought of as also
20/// serving the purpose of SSA formation. The algorithm iterates on the
21/// function until all opportunities for promotion have been realized.
22///
23//===----------------------------------------------------------------------===//
24
25#include "llvm/Transforms/Scalar/SROA.h"
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/MapVector.h"
30#include "llvm/ADT/PointerIntPair.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SetVector.h"
33#include "llvm/ADT/SmallBitVector.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/ADT/Twine.h"
39#include "llvm/ADT/iterator.h"
40#include "llvm/ADT/iterator_range.h"
41#include "llvm/Analysis/AssumptionCache.h"
42#include "llvm/Analysis/DomTreeUpdater.h"
43#include "llvm/Analysis/GlobalsModRef.h"
44#include "llvm/Analysis/Loads.h"
45#include "llvm/Analysis/PtrUseVisitor.h"
46#include "llvm/Analysis/ValueTracking.h"
47#include "llvm/Analysis/VectorUtils.h"
48#include "llvm/IR/BasicBlock.h"
49#include "llvm/IR/Constant.h"
50#include "llvm/IR/ConstantFolder.h"
51#include "llvm/IR/Constants.h"
52#include "llvm/IR/DIBuilder.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfo.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalAlias.h"
60#include "llvm/IR/IRBuilder.h"
61#include "llvm/IR/InstVisitor.h"
62#include "llvm/IR/Instruction.h"
63#include "llvm/IR/Instructions.h"
64#include "llvm/IR/IntrinsicInst.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/Module.h"
68#include "llvm/IR/Operator.h"
69#include "llvm/IR/PassManager.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/InitializePasses.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/Debug.h"
81#include "llvm/Support/ErrorHandling.h"
82#include "llvm/Support/raw_ostream.h"
83#include "llvm/Transforms/Scalar.h"
84#include "llvm/Transforms/Utils/BasicBlockUtils.h"
85#include "llvm/Transforms/Utils/Local.h"
86#include "llvm/Transforms/Utils/PromoteMemToReg.h"
87#include "llvm/Transforms/Utils/SSAUpdater.h"
88#include <algorithm>
89#include <cassert>
90#include <cstddef>
91#include <cstdint>
92#include <cstring>
93#include <iterator>
94#include <string>
95#include <tuple>
96#include <utility>
97#include <variant>
98#include <vector>
99
100using namespace llvm;
101
102#define DEBUG_TYPE "sroa"
103
104STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112STATISTIC(NumLoadsPredicated,
113 "Number of loads rewritten into predicated loads to allow promotion");
114STATISTIC(
115 NumStoresPredicated,
116 "Number of stores rewritten into predicated loads to allow promotion");
117STATISTIC(NumDeleted, "Number of instructions deleted");
118STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120namespace llvm {
121/// Disable running mem2reg during SROA in order to test or debug SROA.
122static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(Val: false),
123 cl::Hidden);
124extern cl::opt<bool> ProfcheckDisableMetadataFixes;
125} // namespace llvm
126
127namespace {
128
129class AllocaSliceRewriter;
130class AllocaSlices;
131class Partition;
132
133class SelectHandSpeculativity {
134 unsigned char Storage = 0; // None are speculatable by default.
135 using TrueVal = Bitfield::Element<bool, 0, 1>; // Low 0'th bit.
136 using FalseVal = Bitfield::Element<bool, 1, 1>; // Low 1'th bit.
137public:
138 SelectHandSpeculativity() = default;
139 SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
140 bool isSpeculatable(bool isTrueVal) const;
141 bool areAllSpeculatable() const;
142 bool areAnySpeculatable() const;
143 bool areNoneSpeculatable() const;
144 // For interop as int half of PointerIntPair.
145 explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
146 explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
147};
148static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
149
150using PossiblySpeculatableLoad =
151 PointerIntPair<LoadInst *, 2, SelectHandSpeculativity>;
152using UnspeculatableStore = StoreInst *;
153using RewriteableMemOp =
154 std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
155using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
156
157/// An optimization pass providing Scalar Replacement of Aggregates.
158///
159/// This pass takes allocations which can be completely analyzed (that is, they
160/// don't escape) and tries to turn them into scalar SSA values. There are
161/// a few steps to this process.
162///
163/// 1) It takes allocations of aggregates and analyzes the ways in which they
164/// are used to try to split them into smaller allocations, ideally of
165/// a single scalar data type. It will split up memcpy and memset accesses
166/// as necessary and try to isolate individual scalar accesses.
167/// 2) It will transform accesses into forms which are suitable for SSA value
168/// promotion. This can be replacing a memset with a scalar store of an
169/// integer value, or it can involve speculating operations on a PHI or
170/// select to be a PHI or select of the results.
171/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
172/// onto insert and extract operations on a vector value, and convert them to
173/// this form. By doing so, it will enable promotion of vector aggregates to
174/// SSA vector values.
175class SROA {
176 LLVMContext *const C;
177 DomTreeUpdater *const DTU;
178 AssumptionCache *const AC;
179 const bool PreserveCFG;
180 const bool AggregateToVector;
181
182 /// Worklist of alloca instructions to simplify.
183 ///
184 /// Each alloca in the function is added to this. Each new alloca formed gets
185 /// added to it as well to recursively simplify unless that alloca can be
186 /// directly promoted. Finally, each time we rewrite a use of an alloca other
187 /// the one being actively rewritten, we add it back onto the list if not
188 /// already present to ensure it is re-visited.
189 SmallSetVector<AllocaInst *, 16> Worklist;
190
191 /// A collection of instructions to delete.
192 /// We try to batch deletions to simplify code and make things a bit more
193 /// efficient. We also make sure there is no dangling pointers.
194 SmallVector<WeakVH, 8> DeadInsts;
195
196 /// Post-promotion worklist.
197 ///
198 /// Sometimes we discover an alloca which has a high probability of becoming
199 /// viable for SROA after a round of promotion takes place. In those cases,
200 /// the alloca is enqueued here for re-processing.
201 ///
202 /// Note that we have to be very careful to clear allocas out of this list in
203 /// the event they are deleted.
204 SmallSetVector<AllocaInst *, 16> PostPromotionWorklist;
205
206 /// A collection of alloca instructions we can directly promote.
207 SetVector<AllocaInst *, SmallVector<AllocaInst *>,
208 SmallPtrSet<AllocaInst *, 16>, 16>
209 PromotableAllocas;
210
211 /// A worklist of PHIs to speculate prior to promoting allocas.
212 ///
213 /// All of these PHIs have been checked for the safety of speculation and by
214 /// being speculated will allow promoting allocas currently in the promotable
215 /// queue.
216 SmallSetVector<PHINode *, 8> SpeculatablePHIs;
217
218 /// A worklist of select instructions to rewrite prior to promoting
219 /// allocas.
220 SmallMapVector<SelectInst *, RewriteableMemOps, 8> SelectsToRewrite;
221
222 /// Select instructions that use an alloca and are subsequently loaded can be
223 /// rewritten to load both input pointers and then select between the result,
224 /// allowing the load of the alloca to be promoted.
225 /// From this:
226 /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other
227 /// %V = load <type>, ptr %P2
228 /// to:
229 /// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
230 /// %V2 = load <type>, ptr %Other
231 /// %V = select i1 %cond, <type> %V1, <type> %V2
232 ///
233 /// We can do this to a select if its only uses are loads
234 /// and if either the operand to the select can be loaded unconditionally,
235 /// or if we are allowed to perform CFG modifications.
236 /// If found an intervening bitcast with a single use of the load,
237 /// allow the promotion.
238 static std::optional<RewriteableMemOps>
239 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
240
241public:
242 SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
243 SROAOptions Options)
244 : C(C), DTU(DTU), AC(AC),
245 PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
246 AggregateToVector(Options.AggregateToVector) {}
247
248 /// Main run method used by both the SROAPass and by the legacy pass.
249 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
250
251private:
252 friend class AllocaSliceRewriter;
253
254 bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
255 std::pair<AllocaInst *, uint64_t>
256 rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
257 bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
258 bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
259 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runOnAlloca(AllocaInst &AI);
260 void clobberUse(Use &U);
261 bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
262 bool promoteAllocas();
263};
264
265} // end anonymous namespace
266
267/// Calculate the fragment of a variable to use when slicing a store
268/// based on the slice dimensions, existing fragment, and base storage
269/// fragment.
270/// Results:
271/// UseFrag - Use Target as the new fragment.
272/// UseNoFrag - The new slice already covers the whole variable.
273/// Skip - The new alloca slice doesn't include this variable.
274/// FIXME: Can we use calculateFragmentIntersect instead?
275namespace {
276enum FragCalcResult { UseFrag, UseNoFrag, Skip };
277}
278static FragCalcResult
279calculateFragment(DILocalVariable *Variable,
280 uint64_t NewStorageSliceOffsetInBits,
281 uint64_t NewStorageSliceSizeInBits,
282 std::optional<DIExpression::FragmentInfo> StorageFragment,
283 std::optional<DIExpression::FragmentInfo> CurrentFragment,
284 DIExpression::FragmentInfo &Target) {
285 // If the base storage describes part of the variable apply the offset and
286 // the size constraint.
287 if (StorageFragment) {
288 Target.SizeInBits =
289 std::min(a: NewStorageSliceSizeInBits, b: StorageFragment->SizeInBits);
290 Target.OffsetInBits =
291 NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits;
292 } else {
293 Target.SizeInBits = NewStorageSliceSizeInBits;
294 Target.OffsetInBits = NewStorageSliceOffsetInBits;
295 }
296
297 // If this slice extracts the entirety of an independent variable from a
298 // larger alloca, do not produce a fragment expression, as the variable is
299 // not fragmented.
300 if (!CurrentFragment) {
301 if (auto Size = Variable->getSizeInBits()) {
302 // Treat the current fragment as covering the whole variable.
303 CurrentFragment = DIExpression::FragmentInfo(*Size, 0);
304 if (Target == CurrentFragment)
305 return UseNoFrag;
306 }
307 }
308
309 // No additional work to do if there isn't a fragment already, or there is
310 // but it already exactly describes the new assignment.
311 if (!CurrentFragment || *CurrentFragment == Target)
312 return UseFrag;
313
314 // Reject the target fragment if it doesn't fit wholly within the current
315 // fragment. TODO: We could instead chop up the target to fit in the case of
316 // a partial overlap.
317 if (Target.startInBits() < CurrentFragment->startInBits() ||
318 Target.endInBits() > CurrentFragment->endInBits())
319 return Skip;
320
321 // Target fits within the current fragment, return it.
322 return UseFrag;
323}
324
325static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
326 return DebugVariable(DVR->getVariable(), std::nullopt,
327 DVR->getDebugLoc().getInlinedAt());
328}
329
330/// Find linked dbg.assign and generate a new one with the correct
331/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
332/// value component is copied from the old dbg.assign to the new.
333/// \param OldAlloca Alloca for the variable before splitting.
334/// \param IsSplit True if the store (not necessarily alloca)
335/// is being split.
336/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
337/// \param SliceSizeInBits New number of bits being written to.
338/// \param OldInst Instruction that is being split.
339/// \param Inst New instruction performing this part of the
340/// split store.
341/// \param Dest Store destination.
342/// \param Value Stored value.
343/// \param DL Datalayout.
344static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
345 uint64_t OldAllocaOffsetInBits,
346 uint64_t SliceSizeInBits, Instruction *OldInst,
347 Instruction *Inst, Value *Dest, Value *Value,
348 const DataLayout &DL) {
349 // If we want allocas to be migrated using this helper then we need to ensure
350 // that the BaseFragments map code still works. A simple solution would be
351 // to choose to always clone alloca dbg_assigns (rather than sometimes
352 // "stealing" them).
353 assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
354
355 auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(Inst: OldInst);
356 // Nothing to do if OldInst has no linked dbg.assign intrinsics.
357 if (DVRAssignMarkerRange.empty())
358 return;
359
360 LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
361 LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
362 LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
363 LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
364 << "\n");
365 LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
366 LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
367 LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
368 LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
369 if (Value)
370 LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
371
372 /// Map of aggregate variables to their fragment associated with OldAlloca.
373 DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>>
374 BaseFragments;
375 for (auto *DVR : at::getDVRAssignmentMarkers(Inst: OldAlloca))
376 BaseFragments[getAggregateVariable(DVR)] =
377 DVR->getExpression()->getFragmentInfo();
378
379 // The new inst needs a DIAssignID unique metadata tag (if OldInst has
380 // one). It shouldn't already have one: assert this assumption.
381 assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
382 DIAssignID *NewID = nullptr;
383 auto &Ctx = Inst->getContext();
384 DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
385 assert(OldAlloca->isStaticAlloca());
386
387 auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
388 LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
389 << "\n");
390 auto *Expr = DbgAssign->getExpression();
391 bool SetKillLocation = false;
392
393 if (IsSplit) {
394 std::optional<DIExpression::FragmentInfo> BaseFragment;
395 {
396 auto R = BaseFragments.find(Val: getAggregateVariable(DVR: DbgAssign));
397 if (R == BaseFragments.end())
398 return;
399 BaseFragment = R->second;
400 }
401 std::optional<DIExpression::FragmentInfo> CurrentFragment =
402 Expr->getFragmentInfo();
403 DIExpression::FragmentInfo NewFragment;
404 FragCalcResult Result = calculateFragment(
405 Variable: DbgAssign->getVariable(), NewStorageSliceOffsetInBits: OldAllocaOffsetInBits, NewStorageSliceSizeInBits: SliceSizeInBits,
406 StorageFragment: BaseFragment, CurrentFragment, Target&: NewFragment);
407
408 if (Result == Skip)
409 return;
410 if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
411 if (CurrentFragment) {
412 // Rewrite NewFragment to be relative to the existing one (this is
413 // what createFragmentExpression wants). CalculateFragment has
414 // already resolved the size for us. FIXME: Should it return the
415 // relative fragment too?
416 NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits;
417 }
418 // Add the new fragment info to the existing expression if possible.
419 if (auto E = DIExpression::createFragmentExpression(
420 Expr, OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits)) {
421 Expr = *E;
422 } else {
423 // Otherwise, add the new fragment info to an empty expression and
424 // discard the value component of this dbg.assign as the value cannot
425 // be computed with the new fragment.
426 Expr = *DIExpression::createFragmentExpression(
427 Expr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
428 OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits);
429 SetKillLocation = true;
430 }
431 }
432 }
433
434 // If we haven't created a DIAssignID ID do that now and attach it to Inst.
435 if (!NewID) {
436 NewID = DIAssignID::getDistinct(Context&: Ctx);
437 Inst->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: NewID);
438 }
439
440 DbgVariableRecord *NewAssign;
441 if (IsSplit) {
442 ::Value *NewValue = Value ? Value : DbgAssign->getValue();
443 NewAssign = cast<DbgVariableRecord>(Val: cast<DbgRecord *>(
444 Val: DIB.insertDbgAssign(LinkedInstr: Inst, Val: NewValue, SrcVar: DbgAssign->getVariable(), ValExpr: Expr,
445 Addr: Dest, AddrExpr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
446 DL: DbgAssign->getDebugLoc())));
447 } else {
448 // The store is not split, simply steal the existing dbg_assign.
449 NewAssign = DbgAssign;
450 NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
451 NewAssign->setAddress(Dest);
452 if (Value)
453 NewAssign->replaceVariableLocationOp(OpIdx: 0u, NewValue: Value);
454 assert(Expr == NewAssign->getExpression());
455 }
456
457 // If we've updated the value but the original dbg.assign has an arglist
458 // then kill it now - we can't use the requested new value.
459 // We can't replace the DIArgList with the new value as it'd leave
460 // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
461 // an arglist). And we can't keep the DIArgList in case the linked store
462 // is being split - in which case the DIArgList + expression may no longer
463 // be computing the correct value.
464 // This should be a very rare situation as it requires the value being
465 // stored to differ from the dbg.assign (i.e., the value has been
466 // represented differently in the debug intrinsic for some reason).
467 SetKillLocation |=
468 Value && (DbgAssign->hasArgList() ||
469 !DbgAssign->getExpression()->isSingleLocationExpression());
470 if (SetKillLocation)
471 NewAssign->setKillLocation();
472
473 // We could use more precision here at the cost of some additional (code)
474 // complexity - if the original dbg.assign was adjacent to its store, we
475 // could position this new dbg.assign adjacent to its store rather than the
476 // old dbg.assgn. That would result in interleaved dbg.assigns rather than
477 // what we get now:
478 // split store !1
479 // split store !2
480 // dbg.assign !1
481 // dbg.assign !2
482 // This (current behaviour) results results in debug assignments being
483 // noted as slightly offset (in code) from the store. In practice this
484 // should have little effect on the debugging experience due to the fact
485 // that all the split stores should get the same line number.
486 if (NewAssign != DbgAssign) {
487 NewAssign->moveBefore(MoveBefore: DbgAssign->getIterator());
488 NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
489 }
490 LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
491 };
492
493 for_each(Range&: DVRAssignMarkerRange, F: MigrateDbgAssign);
494}
495
496namespace {
497
498/// A custom IRBuilder inserter which prefixes all names, but only in
499/// Assert builds.
500class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
501 std::string Prefix;
502
503 Twine getNameWithPrefix(const Twine &Name) const {
504 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
505 }
506
507public:
508 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
509
510 void InsertHelper(Instruction *I, const Twine &Name,
511 BasicBlock::iterator InsertPt) const override {
512 IRBuilderDefaultInserter::InsertHelper(I, Name: getNameWithPrefix(Name),
513 InsertPt);
514 }
515};
516
517/// Provide a type for IRBuilder that drops names in release builds.
518using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
519
520/// A used slice of an alloca.
521///
522/// This structure represents a slice of an alloca used by some instruction. It
523/// stores both the begin and end offsets of this use, a pointer to the use
524/// itself, and a flag indicating whether we can classify the use as splittable
525/// or not when forming partitions of the alloca.
526class Slice {
527 /// The beginning offset of the range.
528 uint64_t BeginOffset = 0;
529
530 /// The ending offset, not included in the range.
531 uint64_t EndOffset = 0;
532
533 /// Storage for both the use of this slice and whether it can be
534 /// split.
535 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
536
537public:
538 Slice() = default;
539
540 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
541 : BeginOffset(BeginOffset), EndOffset(EndOffset),
542 UseAndIsSplittable(U, IsSplittable) {}
543
544 uint64_t beginOffset() const { return BeginOffset; }
545 uint64_t endOffset() const { return EndOffset; }
546
547 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
548 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
549
550 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
551
552 bool isDead() const { return getUse() == nullptr; }
553 void kill() { UseAndIsSplittable.setPointer(nullptr); }
554
555 /// Support for ordering ranges.
556 ///
557 /// This provides an ordering over ranges such that start offsets are
558 /// always increasing, and within equal start offsets, the end offsets are
559 /// decreasing. Thus the spanning range comes first in a cluster with the
560 /// same start position.
561 bool operator<(const Slice &RHS) const {
562 if (beginOffset() < RHS.beginOffset())
563 return true;
564 if (beginOffset() > RHS.beginOffset())
565 return false;
566 if (isSplittable() != RHS.isSplittable())
567 return !isSplittable();
568 if (endOffset() > RHS.endOffset())
569 return true;
570 return false;
571 }
572
573 /// Support comparison with a single offset to allow binary searches.
574 [[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
575 return LHS.beginOffset() < RHSOffset;
576 }
577 [[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
578 return LHSOffset < RHS.beginOffset();
579 }
580
581 bool operator==(const Slice &RHS) const {
582 return isSplittable() == RHS.isSplittable() &&
583 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
584 }
585 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
586};
587
588/// Representation of the alloca slices.
589///
590/// This class represents the slices of an alloca which are formed by its
591/// various uses. If a pointer escapes, we can't fully build a representation
592/// for the slices used and we reflect that in this structure. The uses are
593/// stored, sorted by increasing beginning offset and with unsplittable slices
594/// starting at a particular offset before splittable slices.
595class AllocaSlices {
596public:
597 /// Construct the slices of a particular alloca.
598 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
599
600 /// Test whether a pointer to the allocation escapes our analysis.
601 ///
602 /// If this is true, the slices are never fully built and should be
603 /// ignored.
604 bool isEscaped() const { return PointerEscapingInstr; }
605 bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
606
607 /// Support for iterating over the slices.
608 /// @{
609 using iterator = SmallVectorImpl<Slice>::iterator;
610 using range = iterator_range<iterator>;
611
612 iterator begin() { return Slices.begin(); }
613 iterator end() { return Slices.end(); }
614
615 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
616 using const_range = iterator_range<const_iterator>;
617
618 const_iterator begin() const { return Slices.begin(); }
619 const_iterator end() const { return Slices.end(); }
620 /// @}
621
622 /// Erase a range of slices.
623 void erase(iterator Start, iterator Stop) { Slices.erase(CS: Start, CE: Stop); }
624
625 /// Insert new slices for this alloca.
626 ///
627 /// This moves the slices into the alloca's slices collection, and re-sorts
628 /// everything so that the usual ordering properties of the alloca's slices
629 /// hold.
630 void insert(ArrayRef<Slice> NewSlices) {
631 int OldSize = Slices.size();
632 Slices.append(in_start: NewSlices.begin(), in_end: NewSlices.end());
633 auto SliceI = Slices.begin() + OldSize;
634 std::stable_sort(first: SliceI, last: Slices.end());
635 std::inplace_merge(first: Slices.begin(), middle: SliceI, last: Slices.end());
636 }
637
638 // Forward declare the iterator and range accessor for walking the
639 // partitions.
640 class partition_iterator;
641 iterator_range<partition_iterator> partitions();
642
643 /// Access the dead users for this alloca.
644 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
645
646 /// Access Uses that should be dropped if the alloca is promotable.
647 ArrayRef<Use *> getDeadUsesIfPromotable() const {
648 return DeadUseIfPromotable;
649 }
650
651 /// Access the dead operands referring to this alloca.
652 ///
653 /// These are operands which have cannot actually be used to refer to the
654 /// alloca as they are outside its range and the user doesn't correct for
655 /// that. These mostly consist of PHI node inputs and the like which we just
656 /// need to replace with undef.
657 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
658
659#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
660 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
661 void printSlice(raw_ostream &OS, const_iterator I,
662 StringRef Indent = " ") const;
663 void printUse(raw_ostream &OS, const_iterator I,
664 StringRef Indent = " ") const;
665 void print(raw_ostream &OS) const;
666 void dump(const_iterator I) const;
667 void dump() const;
668#endif
669
670private:
671 template <typename DerivedT, typename RetT = void> class BuilderBase;
672 class SliceBuilder;
673
674 friend class AllocaSlices::SliceBuilder;
675
676#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
677 /// Handle to alloca instruction to simplify method interfaces.
678 AllocaInst &AI;
679#endif
680
681 /// The instruction responsible for this alloca not having a known set
682 /// of slices.
683 ///
684 /// When an instruction (potentially) escapes the pointer to the alloca, we
685 /// store a pointer to that here and abort trying to form slices of the
686 /// alloca. This will be null if the alloca slices are analyzed successfully.
687 Instruction *PointerEscapingInstr;
688 Instruction *PointerEscapingInstrReadOnly;
689
690 /// The slices of the alloca.
691 ///
692 /// We store a vector of the slices formed by uses of the alloca here. This
693 /// vector is sorted by increasing begin offset, and then the unsplittable
694 /// slices before the splittable ones. See the Slice inner class for more
695 /// details.
696 SmallVector<Slice, 8> Slices;
697
698 /// Instructions which will become dead if we rewrite the alloca.
699 ///
700 /// Note that these are not separated by slice. This is because we expect an
701 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
702 /// all these instructions can simply be removed and replaced with poison as
703 /// they come from outside of the allocated space.
704 SmallVector<Instruction *, 8> DeadUsers;
705
706 /// Uses which will become dead if can promote the alloca.
707 SmallVector<Use *, 8> DeadUseIfPromotable;
708
709 /// Operands which will become dead if we rewrite the alloca.
710 ///
711 /// These are operands that in their particular use can be replaced with
712 /// poison when we rewrite the alloca. These show up in out-of-bounds inputs
713 /// to PHI nodes and the like. They aren't entirely dead (there might be
714 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
715 /// want to swap this particular input for poison to simplify the use lists of
716 /// the alloca.
717 SmallVector<Use *, 8> DeadOperands;
718};
719
720/// A partition of the slices.
721///
722/// An ephemeral representation for a range of slices which can be viewed as
723/// a partition of the alloca. This range represents a span of the alloca's
724/// memory which cannot be split, and provides access to all of the slices
725/// overlapping some part of the partition.
726///
727/// Objects of this type are produced by traversing the alloca's slices, but
728/// are only ephemeral and not persistent.
729class Partition {
730private:
731 friend class AllocaSlices;
732 friend class AllocaSlices::partition_iterator;
733
734 using iterator = AllocaSlices::iterator;
735
736 /// The beginning and ending offsets of the alloca for this
737 /// partition.
738 uint64_t BeginOffset = 0, EndOffset = 0;
739
740 /// The start and end iterators of this partition.
741 iterator SI, SJ;
742
743 /// A collection of split slice tails overlapping the partition.
744 SmallVector<Slice *, 4> SplitTails;
745
746 /// Raw constructor builds an empty partition starting and ending at
747 /// the given iterator.
748 Partition(iterator SI) : SI(SI), SJ(SI) {}
749
750public:
751 /// The start offset of this partition.
752 ///
753 /// All of the contained slices start at or after this offset.
754 uint64_t beginOffset() const { return BeginOffset; }
755
756 /// The end offset of this partition.
757 ///
758 /// All of the contained slices end at or before this offset.
759 uint64_t endOffset() const { return EndOffset; }
760
761 /// The size of the partition.
762 ///
763 /// Note that this can never be zero.
764 uint64_t size() const {
765 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
766 return EndOffset - BeginOffset;
767 }
768
769 /// Test whether this partition contains no slices, and merely spans
770 /// a region occupied by split slices.
771 bool empty() const { return SI == SJ; }
772
773 /// \name Iterate slices that start within the partition.
774 /// These may be splittable or unsplittable. They have a begin offset >= the
775 /// partition begin offset.
776 /// @{
777 // FIXME: We should probably define a "concat_iterator" helper and use that
778 // to stitch together pointee_iterators over the split tails and the
779 // contiguous iterators of the partition. That would give a much nicer
780 // interface here. We could then additionally expose filtered iterators for
781 // split, unsplit, and unsplittable splices based on the usage patterns.
782 iterator begin() const { return SI; }
783 iterator end() const { return SJ; }
784 /// @}
785
786 /// Get the sequence of split slice tails.
787 ///
788 /// These tails are of slices which start before this partition but are
789 /// split and overlap into the partition. We accumulate these while forming
790 /// partitions.
791 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
792};
793
794} // end anonymous namespace
795
796/// An iterator over partitions of the alloca's slices.
797///
798/// This iterator implements the core algorithm for partitioning the alloca's
799/// slices. It is a forward iterator as we don't support backtracking for
800/// efficiency reasons, and re-use a single storage area to maintain the
801/// current set of split slices.
802///
803/// It is templated on the slice iterator type to use so that it can operate
804/// with either const or non-const slice iterators.
805class AllocaSlices::partition_iterator
806 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
807 Partition> {
808 friend class AllocaSlices;
809
810 /// Most of the state for walking the partitions is held in a class
811 /// with a nice interface for examining them.
812 Partition P;
813
814 /// We need to keep the end of the slices to know when to stop.
815 AllocaSlices::iterator SE;
816
817 /// We also need to keep track of the maximum split end offset seen.
818 /// FIXME: Do we really?
819 uint64_t MaxSplitSliceEndOffset = 0;
820
821 /// Sets the partition to be empty at given iterator, and sets the
822 /// end iterator.
823 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
824 : P(SI), SE(SE) {
825 // If not already at the end, advance our state to form the initial
826 // partition.
827 if (SI != SE)
828 advance();
829 }
830
831 /// Advance the iterator to the next partition.
832 ///
833 /// Requires that the iterator not be at the end of the slices.
834 void advance() {
835 assert((P.SI != SE || !P.SplitTails.empty()) &&
836 "Cannot advance past the end of the slices!");
837
838 // Clear out any split uses which have ended.
839 if (!P.SplitTails.empty()) {
840 if (P.EndOffset >= MaxSplitSliceEndOffset) {
841 // If we've finished all splits, this is easy.
842 P.SplitTails.clear();
843 MaxSplitSliceEndOffset = 0;
844 } else {
845 // Remove the uses which have ended in the prior partition. This
846 // cannot change the max split slice end because we just checked that
847 // the prior partition ended prior to that max.
848 llvm::erase_if(C&: P.SplitTails,
849 P: [&](Slice *S) { return S->endOffset() <= P.EndOffset; });
850 assert(llvm::any_of(P.SplitTails,
851 [&](Slice *S) {
852 return S->endOffset() == MaxSplitSliceEndOffset;
853 }) &&
854 "Could not find the current max split slice offset!");
855 assert(llvm::all_of(P.SplitTails,
856 [&](Slice *S) {
857 return S->endOffset() <= MaxSplitSliceEndOffset;
858 }) &&
859 "Max split slice end offset is not actually the max!");
860 }
861 }
862
863 // If P.SI is already at the end, then we've cleared the split tail and
864 // now have an end iterator.
865 if (P.SI == SE) {
866 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
867 return;
868 }
869
870 // If we had a non-empty partition previously, set up the state for
871 // subsequent partitions.
872 if (P.SI != P.SJ) {
873 // Accumulate all the splittable slices which started in the old
874 // partition into the split list.
875 for (Slice &S : P)
876 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
877 P.SplitTails.push_back(Elt: &S);
878 MaxSplitSliceEndOffset =
879 std::max(a: S.endOffset(), b: MaxSplitSliceEndOffset);
880 }
881
882 // Start from the end of the previous partition.
883 P.SI = P.SJ;
884
885 // If P.SI is now at the end, we at most have a tail of split slices.
886 if (P.SI == SE) {
887 P.BeginOffset = P.EndOffset;
888 P.EndOffset = MaxSplitSliceEndOffset;
889 return;
890 }
891
892 // If the we have split slices and the next slice is after a gap and is
893 // not splittable immediately form an empty partition for the split
894 // slices up until the next slice begins.
895 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
896 !P.SI->isSplittable()) {
897 P.BeginOffset = P.EndOffset;
898 P.EndOffset = P.SI->beginOffset();
899 return;
900 }
901 }
902
903 // OK, we need to consume new slices. Set the end offset based on the
904 // current slice, and step SJ past it. The beginning offset of the
905 // partition is the beginning offset of the next slice unless we have
906 // pre-existing split slices that are continuing, in which case we begin
907 // at the prior end offset.
908 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
909 P.EndOffset = P.SI->endOffset();
910 ++P.SJ;
911
912 // There are two strategies to form a partition based on whether the
913 // partition starts with an unsplittable slice or a splittable slice.
914 if (!P.SI->isSplittable()) {
915 // When we're forming an unsplittable region, it must always start at
916 // the first slice and will extend through its end.
917 assert(P.BeginOffset == P.SI->beginOffset());
918
919 // Form a partition including all of the overlapping slices with this
920 // unsplittable slice.
921 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
922 if (!P.SJ->isSplittable())
923 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
924 ++P.SJ;
925 }
926
927 // We have a partition across a set of overlapping unsplittable
928 // partitions.
929 return;
930 }
931
932 // If we're starting with a splittable slice, then we need to form
933 // a synthetic partition spanning it and any other overlapping splittable
934 // splices.
935 assert(P.SI->isSplittable() && "Forming a splittable partition!");
936
937 // Collect all of the overlapping splittable slices.
938 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
939 P.SJ->isSplittable()) {
940 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
941 ++P.SJ;
942 }
943
944 // Back upiP.EndOffset if we ended the span early when encountering an
945 // unsplittable slice. This synthesizes the early end offset of
946 // a partition spanning only splittable slices.
947 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
948 assert(!P.SJ->isSplittable());
949 P.EndOffset = P.SJ->beginOffset();
950 }
951 }
952
953public:
954 bool operator==(const partition_iterator &RHS) const {
955 assert(SE == RHS.SE &&
956 "End iterators don't match between compared partition iterators!");
957
958 // The observed positions of partitions is marked by the P.SI iterator and
959 // the emptiness of the split slices. The latter is only relevant when
960 // P.SI == SE, as the end iterator will additionally have an empty split
961 // slices list, but the prior may have the same P.SI and a tail of split
962 // slices.
963 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
964 assert(P.SJ == RHS.P.SJ &&
965 "Same set of slices formed two different sized partitions!");
966 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
967 "Same slice position with differently sized non-empty split "
968 "slice tails!");
969 return true;
970 }
971 return false;
972 }
973
974 partition_iterator &operator++() {
975 advance();
976 return *this;
977 }
978
979 Partition &operator*() { return P; }
980};
981
982/// A forward range over the partitions of the alloca's slices.
983///
984/// This accesses an iterator range over the partitions of the alloca's
985/// slices. It computes these partitions on the fly based on the overlapping
986/// offsets of the slices and the ability to split them. It will visit "empty"
987/// partitions to cover regions of the alloca only accessed via split
988/// slices.
989iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
990 return make_range(x: partition_iterator(begin(), end()),
991 y: partition_iterator(end(), end()));
992}
993
994static Value *foldSelectInst(SelectInst &SI) {
995 // If the condition being selected on is a constant or the same value is
996 // being selected between, fold the select. Yes this does (rarely) happen
997 // early on.
998 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: SI.getCondition()))
999 return SI.getOperand(i_nocapture: 1 + CI->isZero());
1000 if (SI.getOperand(i_nocapture: 1) == SI.getOperand(i_nocapture: 2))
1001 return SI.getOperand(i_nocapture: 1);
1002
1003 return nullptr;
1004}
1005
1006/// A helper that folds a PHI node or a select.
1007static Value *foldPHINodeOrSelectInst(Instruction &I) {
1008 if (PHINode *PN = dyn_cast<PHINode>(Val: &I)) {
1009 // If PN merges together the same value, return that value.
1010 return PN->hasConstantValue();
1011 }
1012 return foldSelectInst(SI&: cast<SelectInst>(Val&: I));
1013}
1014
1015/// Builder for the alloca slices.
1016///
1017/// This class builds a set of alloca slices by recursively visiting the uses
1018/// of an alloca and making a slice for each load and store at each offset.
1019class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1020 friend class PtrUseVisitor<SliceBuilder>;
1021 friend class InstVisitor<SliceBuilder>;
1022
1023 using Base = PtrUseVisitor<SliceBuilder>;
1024
1025 const uint64_t AllocSize;
1026 AllocaSlices &AS;
1027
1028 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
1029 SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
1030
1031 /// Set to de-duplicate dead instructions found in the use walk.
1032 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
1033
1034public:
1035 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1036 : PtrUseVisitor<SliceBuilder>(DL),
1037 AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1038
1039private:
1040 void markAsDead(Instruction &I) {
1041 if (VisitedDeadInsts.insert(Ptr: &I).second)
1042 AS.DeadUsers.push_back(Elt: &I);
1043 }
1044
1045 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1046 bool IsSplittable = false) {
1047 // Completely skip uses which have a zero size or start either before or
1048 // past the end of the allocation.
1049 if (Size == 0 || Offset.uge(RHS: AllocSize)) {
1050 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1051 << Offset
1052 << " which has zero size or starts outside of the "
1053 << AllocSize << " byte alloca:\n"
1054 << " alloca: " << AS.AI << "\n"
1055 << " use: " << I << "\n");
1056 return markAsDead(I);
1057 }
1058
1059 uint64_t BeginOffset = Offset.getZExtValue();
1060 uint64_t EndOffset = BeginOffset + Size;
1061
1062 // Clamp the end offset to the end of the allocation. Note that this is
1063 // formulated to handle even the case where "BeginOffset + Size" overflows.
1064 // This may appear superficially to be something we could ignore entirely,
1065 // but that is not so! There may be widened loads or PHI-node uses where
1066 // some instructions are dead but not others. We can't completely ignore
1067 // them, and so have to record at least the information here.
1068 assert(AllocSize >= BeginOffset); // Established above.
1069 if (Size > AllocSize - BeginOffset) {
1070 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1071 << Offset << " to remain within the " << AllocSize
1072 << " byte alloca:\n"
1073 << " alloca: " << AS.AI << "\n"
1074 << " use: " << I << "\n");
1075 EndOffset = AllocSize;
1076 }
1077
1078 AS.Slices.push_back(Elt: Slice(BeginOffset, EndOffset, U, IsSplittable));
1079 }
1080
1081 void visitBitCastInst(BitCastInst &BC) {
1082 if (BC.use_empty())
1083 return markAsDead(I&: BC);
1084
1085 return Base::visitBitCastInst(BC);
1086 }
1087
1088 void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1089 if (ASC.use_empty())
1090 return markAsDead(I&: ASC);
1091
1092 return Base::visitAddrSpaceCastInst(ASC);
1093 }
1094
1095 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1096 if (GEPI.use_empty())
1097 return markAsDead(I&: GEPI);
1098
1099 return Base::visitGetElementPtrInst(GEPI);
1100 }
1101
1102 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
1103 uint64_t Size, bool IsVolatile) {
1104 // We allow splitting of non-volatile loads and stores where the type is an
1105 // integer type. These may be used to implement 'memcpy' or other "transfer
1106 // of bits" patterns.
1107 bool IsSplittable =
1108 Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1109
1110 insertUse(I, Offset, Size, IsSplittable);
1111 }
1112
1113 void visitLoadInst(LoadInst &LI) {
1114 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
1115 "All simple FCA loads should have been pre-split");
1116
1117 // If there is a load with an unknown offset, we can still perform store
1118 // to load forwarding for other known-offset loads.
1119 if (!IsOffsetKnown)
1120 return PI.setEscapedReadOnly(&LI);
1121
1122 TypeSize Size = DL.getTypeStoreSize(Ty: LI.getType());
1123 if (Size.isScalable()) {
1124 unsigned VScale = LI.getFunction()->getVScaleValue();
1125 if (!VScale)
1126 return PI.setAborted(&LI);
1127
1128 Size = TypeSize::getFixed(ExactSize: Size.getKnownMinValue() * VScale);
1129 }
1130
1131 return handleLoadOrStore(Ty: LI.getType(), I&: LI, Offset, Size: Size.getFixedValue(),
1132 IsVolatile: LI.isVolatile());
1133 }
1134
1135 void visitStoreInst(StoreInst &SI) {
1136 Value *ValOp = SI.getValueOperand();
1137 if (ValOp == *U)
1138 return PI.setEscapedAndAborted(&SI);
1139 if (!IsOffsetKnown)
1140 return PI.setAborted(&SI);
1141
1142 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValOp->getType());
1143 if (StoreSize.isScalable()) {
1144 unsigned VScale = SI.getFunction()->getVScaleValue();
1145 if (!VScale)
1146 return PI.setAborted(&SI);
1147
1148 StoreSize = TypeSize::getFixed(ExactSize: StoreSize.getKnownMinValue() * VScale);
1149 }
1150
1151 uint64_t Size = StoreSize.getFixedValue();
1152
1153 // If this memory access can be shown to *statically* extend outside the
1154 // bounds of the allocation, it's behavior is undefined, so simply
1155 // ignore it. Note that this is more strict than the generic clamping
1156 // behavior of insertUse. We also try to handle cases which might run the
1157 // risk of overflow.
1158 // FIXME: We should instead consider the pointer to have escaped if this
1159 // function is being instrumented for addressing bugs or race conditions.
1160 if (Size > AllocSize || Offset.ugt(RHS: AllocSize - Size)) {
1161 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1162 << Offset << " which extends past the end of the "
1163 << AllocSize << " byte alloca:\n"
1164 << " alloca: " << AS.AI << "\n"
1165 << " use: " << SI << "\n");
1166 return markAsDead(I&: SI);
1167 }
1168
1169 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1170 "All simple FCA stores should have been pre-split");
1171 handleLoadOrStore(Ty: ValOp->getType(), I&: SI, Offset, Size, IsVolatile: SI.isVolatile());
1172 }
1173
1174 void visitMemSetInst(MemSetInst &II) {
1175 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1176 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1177 if ((Length && Length->getValue() == 0) ||
1178 (IsOffsetKnown && Offset.uge(RHS: AllocSize)))
1179 // Zero-length mem transfer intrinsics can be ignored entirely.
1180 return markAsDead(I&: II);
1181
1182 if (!IsOffsetKnown)
1183 return PI.setAborted(&II);
1184
1185 insertUse(I&: II, Offset,
1186 Size: Length ? Length->getLimitedValue()
1187 : AllocSize - Offset.getLimitedValue(),
1188 IsSplittable: (bool)Length);
1189 }
1190
1191 void visitMemTransferInst(MemTransferInst &II) {
1192 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1193 if (Length && Length->getValue() == 0)
1194 // Zero-length mem transfer intrinsics can be ignored entirely.
1195 return markAsDead(I&: II);
1196
1197 // Because we can visit these intrinsics twice, also check to see if the
1198 // first time marked this instruction as dead. If so, skip it.
1199 if (VisitedDeadInsts.count(Ptr: &II))
1200 return;
1201
1202 if (!IsOffsetKnown)
1203 return PI.setAborted(&II);
1204
1205 // This side of the transfer is completely out-of-bounds, and so we can
1206 // nuke the entire transfer. However, we also need to nuke the other side
1207 // if already added to our partitions.
1208 // FIXME: Yet another place we really should bypass this when
1209 // instrumenting for ASan.
1210 if (Offset.uge(RHS: AllocSize)) {
1211 auto MTPI = MemTransferSliceMap.find(Val: &II);
1212 if (MTPI != MemTransferSliceMap.end())
1213 AS.Slices[MTPI->second].kill();
1214 return markAsDead(I&: II);
1215 }
1216
1217 uint64_t RawOffset = Offset.getLimitedValue();
1218 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1219
1220 // Check for the special case where the same exact value is used for both
1221 // source and dest.
1222 if (*U == II.getRawDest() && *U == II.getRawSource()) {
1223 // For non-volatile transfers this is a no-op.
1224 if (!II.isVolatile())
1225 return markAsDead(I&: II);
1226
1227 return insertUse(I&: II, Offset, Size, /*IsSplittable=*/false);
1228 }
1229
1230 // If we have seen both source and destination for a mem transfer, then
1231 // they both point to the same alloca.
1232 bool Inserted;
1233 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
1234 std::tie(args&: MTPI, args&: Inserted) =
1235 MemTransferSliceMap.insert(KV: std::make_pair(x: &II, y: AS.Slices.size()));
1236 unsigned PrevIdx = MTPI->second;
1237 if (!Inserted) {
1238 Slice &PrevP = AS.Slices[PrevIdx];
1239
1240 // Check if the begin offsets match and this is a non-volatile transfer.
1241 // In that case, we can completely elide the transfer.
1242 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1243 PrevP.kill();
1244 return markAsDead(I&: II);
1245 }
1246
1247 // Otherwise we have an offset transfer within the same alloca. We can't
1248 // split those.
1249 PrevP.makeUnsplittable();
1250 }
1251
1252 // Insert the use now that we've fixed up the splittable nature.
1253 insertUse(I&: II, Offset, Size, /*IsSplittable=*/Inserted && Length);
1254
1255 // Check that we ended up with a valid index in the map.
1256 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1257 "Map index doesn't point back to a slice with this user.");
1258 }
1259
1260 // Disable SRoA for any intrinsics except for lifetime invariants.
1261 // FIXME: What about debug intrinsics? This matches old behavior, but
1262 // doesn't make sense.
1263 void visitIntrinsicInst(IntrinsicInst &II) {
1264 if (II.isDroppable()) {
1265 AS.DeadUseIfPromotable.push_back(Elt: U);
1266 return;
1267 }
1268
1269 if (!IsOffsetKnown)
1270 return PI.setAborted(&II);
1271
1272 if (II.isLifetimeStartOrEnd()) {
1273 insertUse(I&: II, Offset, Size: AllocSize, IsSplittable: true);
1274 return;
1275 }
1276
1277 Base::visitIntrinsicInst(II);
1278 }
1279
1280 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
1281 // We consider any PHI or select that results in a direct load or store of
1282 // the same offset to be a viable use for slicing purposes. These uses
1283 // are considered unsplittable and the size is the maximum loaded or stored
1284 // size.
1285 SmallPtrSet<Instruction *, 4> Visited;
1286 SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses;
1287 Visited.insert(Ptr: Root);
1288 Uses.push_back(Elt: std::make_pair(x: cast<Instruction>(Val&: *U), y&: Root));
1289 const DataLayout &DL = Root->getDataLayout();
1290 // If there are no loads or stores, the access is dead. We mark that as
1291 // a size zero access.
1292 Size = 0;
1293 do {
1294 Instruction *I, *UsedI;
1295 std::tie(args&: UsedI, args&: I) = Uses.pop_back_val();
1296
1297 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1298 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
1299 if (LoadSize.isScalable()) {
1300 PI.setAborted(LI);
1301 return nullptr;
1302 }
1303 Size = std::max(a: Size, b: LoadSize.getFixedValue());
1304 continue;
1305 }
1306 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1307 Value *Op = SI->getOperand(i_nocapture: 0);
1308 if (Op == UsedI)
1309 return SI;
1310 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Op->getType());
1311 if (StoreSize.isScalable()) {
1312 PI.setAborted(SI);
1313 return nullptr;
1314 }
1315 Size = std::max(a: Size, b: StoreSize.getFixedValue());
1316 continue;
1317 }
1318
1319 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1320 if (!GEP->hasAllZeroIndices())
1321 return GEP;
1322 } else if (!isa<BitCastInst>(Val: I) && !isa<PHINode>(Val: I) &&
1323 !isa<SelectInst>(Val: I) && !isa<AddrSpaceCastInst>(Val: I)) {
1324 return I;
1325 }
1326
1327 for (User *U : I->users())
1328 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
1329 Uses.push_back(Elt: std::make_pair(x&: I, y: cast<Instruction>(Val: U)));
1330 } while (!Uses.empty());
1331
1332 return nullptr;
1333 }
1334
1335 void visitPHINodeOrSelectInst(Instruction &I) {
1336 assert(isa<PHINode>(I) || isa<SelectInst>(I));
1337 if (I.use_empty())
1338 return markAsDead(I);
1339
1340 // If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1341 // instructions in this BB, which may be required during rewriting. Bail out
1342 // on these cases.
1343 if (isa<PHINode>(Val: I) && !I.getParent()->hasInsertionPt())
1344 return PI.setAborted(&I);
1345
1346 // TODO: We could use simplifyInstruction here to fold PHINodes and
1347 // SelectInsts. However, doing so requires to change the current
1348 // dead-operand-tracking mechanism. For instance, suppose neither loading
1349 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1350 // trap either. However, if we simply replace %U with undef using the
1351 // current dead-operand-tracking mechanism, "load (select undef, undef,
1352 // %other)" may trap because the select may return the first operand
1353 // "undef".
1354 if (Value *Result = foldPHINodeOrSelectInst(I)) {
1355 if (Result == *U)
1356 // If the result of the constant fold will be the pointer, recurse
1357 // through the PHI/select as if we had RAUW'ed it.
1358 enqueueUsers(I);
1359 else
1360 // Otherwise the operand to the PHI/select is dead, and we can replace
1361 // it with poison.
1362 AS.DeadOperands.push_back(Elt: U);
1363
1364 return;
1365 }
1366
1367 if (!IsOffsetKnown)
1368 return PI.setAborted(&I);
1369
1370 // See if we already have computed info on this node.
1371 uint64_t &Size = PHIOrSelectSizes[&I];
1372 if (!Size) {
1373 // This is a new PHI/Select, check for an unsafe use of it.
1374 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(Root: &I, Size))
1375 return PI.setAborted(UnsafeI);
1376 }
1377
1378 // For PHI and select operands outside the alloca, we can't nuke the entire
1379 // phi or select -- the other side might still be relevant, so we special
1380 // case them here and use a separate structure to track the operands
1381 // themselves which should be replaced with poison.
1382 // FIXME: This should instead be escaped in the event we're instrumenting
1383 // for address sanitization.
1384 if (Offset.uge(RHS: AllocSize)) {
1385 AS.DeadOperands.push_back(Elt: U);
1386 return;
1387 }
1388
1389 insertUse(I, Offset, Size);
1390 }
1391
1392 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(I&: PN); }
1393
1394 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(I&: SI); }
1395
1396 /// Disable SROA entirely if there are unhandled users of the alloca.
1397 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1398
1399 void visitCallBase(CallBase &CB) {
1400 // If the call operand is read-only and only does a read-only or address
1401 // capture, then we mark it as EscapedReadOnly.
1402 if (CB.isDataOperand(U) &&
1403 !capturesFullProvenance(CC: CB.getCaptureInfo(OpNo: U->getOperandNo())) &&
1404 CB.onlyReadsMemory(OpNo: U->getOperandNo())) {
1405 PI.setEscapedReadOnly(&CB);
1406 return;
1407 }
1408
1409 Base::visitCallBase(CB);
1410 }
1411};
1412
1413AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1414 :
1415#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1416 AI(AI),
1417#endif
1418 PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1419 SliceBuilder PB(DL, AI, *this);
1420 SliceBuilder::PtrInfo PtrI = PB.visitPtr(I&: AI);
1421 if (PtrI.isEscaped() || PtrI.isAborted()) {
1422 // FIXME: We should sink the escape vs. abort info into the caller nicely,
1423 // possibly by just storing the PtrInfo in the AllocaSlices.
1424 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1425 : PtrI.getAbortingInst();
1426 assert(PointerEscapingInstr && "Did not track a bad instruction");
1427 return;
1428 }
1429 PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1430
1431 llvm::erase_if(C&: Slices, P: [](const Slice &S) { return S.isDead(); });
1432
1433 // Sort the uses. This arranges for the offsets to be in ascending order,
1434 // and the sizes to be in descending order.
1435 llvm::stable_sort(Range&: Slices);
1436}
1437
1438#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1439
1440void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1441 StringRef Indent) const {
1442 printSlice(OS, I, Indent);
1443 OS << "\n";
1444 printUse(OS, I, Indent);
1445}
1446
1447void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1448 StringRef Indent) const {
1449 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1450 << " slice #" << (I - begin())
1451 << (I->isSplittable() ? " (splittable)" : "");
1452}
1453
1454void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1455 StringRef Indent) const {
1456 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1457}
1458
1459void AllocaSlices::print(raw_ostream &OS) const {
1460 if (PointerEscapingInstr) {
1461 OS << "Can't analyze slices for alloca: " << AI << "\n"
1462 << " A pointer to this alloca escaped by:\n"
1463 << " " << *PointerEscapingInstr << "\n";
1464 return;
1465 }
1466
1467 if (PointerEscapingInstrReadOnly)
1468 OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1469
1470 OS << "Slices of alloca: " << AI << "\n";
1471 for (const_iterator I = begin(), E = end(); I != E; ++I)
1472 print(OS, I);
1473}
1474
1475LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1476 print(dbgs(), I);
1477}
1478LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1479
1480#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1481
1482/// Walk the range of a partitioning looking for a common type to cover this
1483/// sequence of slices.
1484static std::pair<Type *, IntegerType *>
1485findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1486 uint64_t EndOffset) {
1487 Type *Ty = nullptr;
1488 bool TyIsCommon = true;
1489 IntegerType *ITy = nullptr;
1490
1491 // Note that we need to look at *every* alloca slice's Use to ensure we
1492 // always get consistent results regardless of the order of slices.
1493 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1494 Use *U = I->getUse();
1495 if (isa<IntrinsicInst>(Val: *U->getUser()))
1496 continue;
1497 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1498 continue;
1499
1500 Type *UserTy = nullptr;
1501 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
1502 UserTy = LI->getType();
1503 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
1504 UserTy = SI->getValueOperand()->getType();
1505 }
1506
1507 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(Val: UserTy)) {
1508 // If the type is larger than the partition, skip it. We only encounter
1509 // this for split integer operations where we want to use the type of the
1510 // entity causing the split. Also skip if the type is not a byte width
1511 // multiple.
1512 if (UserITy->getBitWidth() % 8 != 0 ||
1513 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
1514 continue;
1515
1516 // Track the largest bitwidth integer type used in this way in case there
1517 // is no common type.
1518 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1519 ITy = UserITy;
1520 }
1521
1522 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1523 // depend on types skipped above.
1524 if (!UserTy || (Ty && Ty != UserTy))
1525 TyIsCommon = false; // Give up on anything but an iN type.
1526 else
1527 Ty = UserTy;
1528 }
1529
1530 return {TyIsCommon ? Ty : nullptr, ITy};
1531}
1532
1533/// PHI instructions that use an alloca and are subsequently loaded can be
1534/// rewritten to load both input pointers in the pred blocks and then PHI the
1535/// results, allowing the load of the alloca to be promoted.
1536/// From this:
1537/// %P2 = phi [i32* %Alloca, i32* %Other]
1538/// %V = load i32* %P2
1539/// to:
1540/// %V1 = load i32* %Alloca -> will be mem2reg'd
1541/// ...
1542/// %V2 = load i32* %Other
1543/// ...
1544/// %V = phi [i32 %V1, i32 %V2]
1545///
1546/// We can do this to a select if its only uses are loads and if the operands
1547/// to the select can be loaded unconditionally.
1548///
1549/// FIXME: This should be hoisted into a generic utility, likely in
1550/// Transforms/Util/Local.h
1551static bool isSafePHIToSpeculate(PHINode &PN) {
1552 const DataLayout &DL = PN.getDataLayout();
1553
1554 // For now, we can only do this promotion if the load is in the same block
1555 // as the PHI, and if there are no stores between the phi and load.
1556 // TODO: Allow recursive phi users.
1557 // TODO: Allow stores.
1558 BasicBlock *BB = PN.getParent();
1559 Align MaxAlign;
1560 uint64_t APWidth = DL.getIndexTypeSizeInBits(Ty: PN.getType());
1561 Type *LoadType = nullptr;
1562 for (User *U : PN.users()) {
1563 LoadInst *LI = dyn_cast<LoadInst>(Val: U);
1564 if (!LI || !LI->isSimple())
1565 return false;
1566
1567 // For now we only allow loads in the same block as the PHI. This is
1568 // a common case that happens when instcombine merges two loads through
1569 // a PHI.
1570 if (LI->getParent() != BB)
1571 return false;
1572
1573 if (LoadType) {
1574 if (LoadType != LI->getType())
1575 return false;
1576 } else {
1577 LoadType = LI->getType();
1578 }
1579
1580 // Ensure that there are no instructions between the PHI and the load that
1581 // could store.
1582 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1583 if (BBI->mayWriteToMemory())
1584 return false;
1585
1586 MaxAlign = std::max(a: MaxAlign, b: LI->getAlign());
1587 }
1588
1589 if (!LoadType)
1590 return false;
1591
1592 APInt LoadSize =
1593 APInt(APWidth, DL.getTypeStoreSize(Ty: LoadType).getFixedValue());
1594
1595 // We can only transform this if it is safe to push the loads into the
1596 // predecessor blocks. The only thing to watch out for is that we can't put
1597 // a possibly trapping load in the predecessor if it is a critical edge.
1598 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1599 Instruction *TI = PN.getIncomingBlock(i: Idx)->getTerminator();
1600 Value *InVal = PN.getIncomingValue(i: Idx);
1601
1602 // If the value is produced by the terminator of the predecessor (an
1603 // invoke) or it has side-effects, there is no valid place to put a load
1604 // in the predecessor.
1605 if (TI == InVal || TI->mayHaveSideEffects())
1606 return false;
1607
1608 // If the predecessor has a single successor, then the edge isn't
1609 // critical.
1610 if (TI->getNumSuccessors() == 1)
1611 continue;
1612
1613 // If this pointer is always safe to load, or if we can prove that there
1614 // is already a load in the block, then we can move the load to the pred
1615 // block.
1616 if (isSafeToLoadUnconditionally(V: InVal, Alignment: MaxAlign, Size: LoadSize, DL, ScanFrom: TI))
1617 continue;
1618
1619 return false;
1620 }
1621
1622 return true;
1623}
1624
1625static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1626 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1627
1628 LoadInst *SomeLoad = cast<LoadInst>(Val: PN.user_back());
1629 Type *LoadTy = SomeLoad->getType();
1630 IRB.SetInsertPoint(&PN);
1631 PHINode *NewPN = IRB.CreatePHI(Ty: LoadTy, NumReservedValues: PN.getNumIncomingValues(),
1632 Name: PN.getName() + ".sroa.speculated");
1633
1634 // Get the AA tags and alignment to use from one of the loads. It does not
1635 // matter which one we get and if any differ.
1636 AAMDNodes AATags = SomeLoad->getAAMetadata();
1637 Align Alignment = SomeLoad->getAlign();
1638
1639 // Rewrite all loads of the PN to use the new PHI.
1640 while (!PN.use_empty()) {
1641 LoadInst *LI = cast<LoadInst>(Val: PN.user_back());
1642 LI->replaceAllUsesWith(V: NewPN);
1643 LI->eraseFromParent();
1644 }
1645
1646 // Inject loads into all of the pred blocks.
1647 DenseMap<BasicBlock *, Value *> InjectedLoads;
1648 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1649 BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1650 Value *InVal = PN.getIncomingValue(i: Idx);
1651
1652 // A PHI node is allowed to have multiple (duplicated) entries for the same
1653 // basic block, as long as the value is the same. So if we already injected
1654 // a load in the predecessor, then we should reuse the same load for all
1655 // duplicated entries.
1656 if (Value *V = InjectedLoads.lookup(Val: Pred)) {
1657 NewPN->addIncoming(V, BB: Pred);
1658 continue;
1659 }
1660
1661 Instruction *TI = Pred->getTerminator();
1662 IRB.SetInsertPoint(TI);
1663
1664 LoadInst *Load = IRB.CreateAlignedLoad(
1665 Ty: LoadTy, Ptr: InVal, Align: Alignment,
1666 Name: (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1667 ++NumLoadsSpeculated;
1668 if (AATags)
1669 Load->setAAMetadata(AATags);
1670 NewPN->addIncoming(V: Load, BB: Pred);
1671 InjectedLoads[Pred] = Load;
1672 }
1673
1674 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1675 PN.eraseFromParent();
1676}
1677
1678SelectHandSpeculativity &
1679SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1680 if (isTrueVal)
1681 Bitfield::set<SelectHandSpeculativity::TrueVal>(Packed&: Storage, Value: true);
1682 else
1683 Bitfield::set<SelectHandSpeculativity::FalseVal>(Packed&: Storage, Value: true);
1684 return *this;
1685}
1686
1687bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1688 return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Packed: Storage)
1689 : Bitfield::get<SelectHandSpeculativity::FalseVal>(Packed: Storage);
1690}
1691
1692bool SelectHandSpeculativity::areAllSpeculatable() const {
1693 return isSpeculatable(/*isTrueVal=*/true) &&
1694 isSpeculatable(/*isTrueVal=*/false);
1695}
1696
1697bool SelectHandSpeculativity::areAnySpeculatable() const {
1698 return isSpeculatable(/*isTrueVal=*/true) ||
1699 isSpeculatable(/*isTrueVal=*/false);
1700}
1701bool SelectHandSpeculativity::areNoneSpeculatable() const {
1702 return !areAnySpeculatable();
1703}
1704
1705static SelectHandSpeculativity
1706isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1707 assert(LI.isSimple() && "Only for simple loads");
1708 SelectHandSpeculativity Spec;
1709
1710 const DataLayout &DL = SI.getDataLayout();
1711 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1712 if (isSafeToLoadUnconditionally(V: Value, Ty: LI.getType(), Alignment: LI.getAlign(), DL,
1713 ScanFrom: &LI))
1714 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1715 else if (PreserveCFG)
1716 return Spec;
1717
1718 return Spec;
1719}
1720
1721std::optional<RewriteableMemOps>
1722SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1723 RewriteableMemOps Ops;
1724
1725 for (User *U : SI.users()) {
1726 if (auto *BC = dyn_cast<BitCastInst>(Val: U); BC && BC->hasOneUse())
1727 U = *BC->user_begin();
1728
1729 if (auto *Store = dyn_cast<StoreInst>(Val: U)) {
1730 // Note that atomic stores can be transformed; atomic semantics do not
1731 // have any meaning for a local alloca. Stores are not speculatable,
1732 // however, so if we can't turn it into a predicated store, we are done.
1733 if (Store->isVolatile() || PreserveCFG)
1734 return {}; // Give up on this `select`.
1735 Ops.emplace_back(Args&: Store);
1736 continue;
1737 }
1738
1739 auto *LI = dyn_cast<LoadInst>(Val: U);
1740
1741 // Note that atomic loads can be transformed;
1742 // atomic semantics do not have any meaning for a local alloca.
1743 if (!LI || LI->isVolatile())
1744 return {}; // Give up on this `select`.
1745
1746 PossiblySpeculatableLoad Load(LI);
1747 if (!LI->isSimple()) {
1748 // If the `load` is not simple, we can't speculatively execute it,
1749 // but we could handle this via a CFG modification. But can we?
1750 if (PreserveCFG)
1751 return {}; // Give up on this `select`.
1752 Ops.emplace_back(Args&: Load);
1753 continue;
1754 }
1755
1756 SelectHandSpeculativity Spec =
1757 isSafeLoadOfSelectToSpeculate(LI&: *LI, SI, PreserveCFG);
1758 if (PreserveCFG && !Spec.areAllSpeculatable())
1759 return {}; // Give up on this `select`.
1760
1761 Load.setInt(Spec);
1762 Ops.emplace_back(Args&: Load);
1763 }
1764
1765 return Ops;
1766}
1767
1768static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1769 IRBuilderTy &IRB) {
1770 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1771
1772 Value *TV = SI.getTrueValue();
1773 Value *FV = SI.getFalseValue();
1774 // Replace the given load of the select with a select of two loads.
1775
1776 assert(LI.isSimple() && "We only speculate simple loads");
1777
1778 IRB.SetInsertPoint(&LI);
1779
1780 LoadInst *TL =
1781 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: TV, Align: LI.getAlign(),
1782 Name: LI.getName() + ".sroa.speculate.load.true");
1783 LoadInst *FL =
1784 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: FV, Align: LI.getAlign(),
1785 Name: LI.getName() + ".sroa.speculate.load.false");
1786 NumLoadsSpeculated += 2;
1787
1788 // Transfer alignment and AA info if present.
1789 TL->setAlignment(LI.getAlign());
1790 FL->setAlignment(LI.getAlign());
1791
1792 AAMDNodes Tags = LI.getAAMetadata();
1793 if (Tags) {
1794 TL->setAAMetadata(Tags);
1795 FL->setAAMetadata(Tags);
1796 }
1797
1798 Value *V = IRB.CreateSelect(C: SI.getCondition(), True: TL, False: FL,
1799 Name: LI.getName() + ".sroa.speculated",
1800 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
1801
1802 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1803 LI.replaceAllUsesWith(V);
1804}
1805
1806template <typename T>
1807static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1808 SelectHandSpeculativity Spec,
1809 DomTreeUpdater &DTU) {
1810 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!");
1811 LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1812 BasicBlock *Head = I.getParent();
1813 Instruction *ThenTerm = nullptr;
1814 Instruction *ElseTerm = nullptr;
1815 if (Spec.areNoneSpeculatable())
1816 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1817 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU);
1818 else {
1819 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1820 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU,
1821 /*LI=*/nullptr, /*ThenBlock=*/nullptr);
1822 if (Spec.isSpeculatable(/*isTrueVal=*/true))
1823 cast<CondBrInst>(Val: Head->getTerminator())->swapSuccessors();
1824 }
1825 auto *HeadBI = cast<CondBrInst>(Val: Head->getTerminator());
1826 Spec = {}; // Do not use `Spec` beyond this point.
1827 BasicBlock *Tail = I.getParent();
1828 Tail->setName(Head->getName() + ".cont");
1829 PHINode *PN;
1830 if (isa<LoadInst>(I))
1831 PN = PHINode::Create(Ty: I.getType(), NumReservedValues: 2, NameStr: "", InsertBefore: I.getIterator());
1832 for (BasicBlock *SuccBB : successors(BB: Head)) {
1833 bool IsThen = SuccBB == HeadBI->getSuccessor(i: 0);
1834 int SuccIdx = IsThen ? 0 : 1;
1835 auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1836 auto &CondMemOp = cast<T>(*I.clone());
1837 if (NewMemOpBB != Head) {
1838 NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1839 if (isa<LoadInst>(I))
1840 ++NumLoadsPredicated;
1841 else
1842 ++NumStoresPredicated;
1843 } else {
1844 CondMemOp.dropUBImplyingAttrsAndMetadata();
1845 ++NumLoadsSpeculated;
1846 }
1847 CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
1848 Value *Ptr = SI.getOperand(i_nocapture: 1 + SuccIdx);
1849 CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
1850 if (isa<LoadInst>(I)) {
1851 CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
1852 PN->addIncoming(V: &CondMemOp, BB: NewMemOpBB);
1853 } else
1854 LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
1855 }
1856 if (isa<LoadInst>(I)) {
1857 PN->takeName(V: &I);
1858 LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
1859 I.replaceAllUsesWith(PN);
1860 }
1861}
1862
1863static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I,
1864 SelectHandSpeculativity Spec,
1865 DomTreeUpdater &DTU) {
1866 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1867 rewriteMemOpOfSelect(SI&: SelInst, I&: *LI, Spec, DTU);
1868 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1869 rewriteMemOpOfSelect(SI&: SelInst, I&: *SI, Spec, DTU);
1870 else
1871 llvm_unreachable_internal(msg: "Only for load and store.");
1872}
1873
1874static bool rewriteSelectInstMemOps(SelectInst &SI,
1875 const RewriteableMemOps &Ops,
1876 IRBuilderTy &IRB, DomTreeUpdater *DTU) {
1877 bool CFGChanged = false;
1878 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
1879
1880 for (const RewriteableMemOp &Op : Ops) {
1881 SelectHandSpeculativity Spec;
1882 Instruction *I;
1883 if (auto *const *US = std::get_if<UnspeculatableStore>(ptr: &Op)) {
1884 I = *US;
1885 } else {
1886 auto PSL = std::get<PossiblySpeculatableLoad>(v: Op);
1887 I = PSL.getPointer();
1888 Spec = PSL.getInt();
1889 }
1890 if (Spec.areAllSpeculatable()) {
1891 speculateSelectInstLoads(SI, LI&: cast<LoadInst>(Val&: *I), IRB);
1892 } else {
1893 assert(DTU && "Should not get here when not allowed to modify the CFG!");
1894 rewriteMemOpOfSelect(SelInst&: SI, I&: *I, Spec, DTU&: *DTU);
1895 CFGChanged = true;
1896 }
1897 I->eraseFromParent();
1898 }
1899
1900 for (User *U : make_early_inc_range(Range: SI.users()))
1901 cast<BitCastInst>(Val: U)->eraseFromParent();
1902 SI.eraseFromParent();
1903 return CFGChanged;
1904}
1905
1906/// Compute an adjusted pointer from Ptr by Offset bytes where the
1907/// resulting pointer has PointerTy.
1908static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
1909 APInt Offset, Type *PointerTy,
1910 const Twine &NamePrefix) {
1911 if (Offset != 0)
1912 Ptr = IRB.CreateInBoundsPtrAdd(Ptr, Offset: IRB.getInt(AI: Offset),
1913 Name: NamePrefix + "sroa_idx");
1914 return IRB.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: PointerTy,
1915 Name: NamePrefix + "sroa_cast");
1916}
1917
1918/// Compute the adjusted alignment for a load or store from an offset.
1919static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
1920 return commonAlignment(A: getLoadStoreAlignment(I), Offset);
1921}
1922
1923/// Test whether we can convert a value from the old to the new type.
1924///
1925/// This predicate should be used to guard calls to convertValue in order to
1926/// ensure that we only try to convert viable values. The strategy is that we
1927/// will peel off single element struct and array wrappings to get to an
1928/// underlying value, and convert that value.
1929static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
1930 unsigned VScale = 0) {
1931 if (OldTy == NewTy)
1932 return true;
1933
1934 // For integer types, we can't handle any bit-width differences. This would
1935 // break both vector conversions with extension and introduce endianness
1936 // issues when in conjunction with loads and stores.
1937 if (isa<IntegerType>(Val: OldTy) && isa<IntegerType>(Val: NewTy)) {
1938 assert(cast<IntegerType>(OldTy)->getBitWidth() !=
1939 cast<IntegerType>(NewTy)->getBitWidth() &&
1940 "We can't have the same bitwidth for different int types");
1941 return false;
1942 }
1943
1944 TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
1945 TypeSize OldSize = DL.getTypeSizeInBits(Ty: OldTy);
1946
1947 if ((isa<ScalableVectorType>(Val: NewTy) && isa<FixedVectorType>(Val: OldTy)) ||
1948 (isa<ScalableVectorType>(Val: OldTy) && isa<FixedVectorType>(Val: NewTy))) {
1949 // Conversion is only possible when the size of scalable vectors is known.
1950 if (!VScale)
1951 return false;
1952
1953 // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
1954 // a single domain (either fixed or scalable). Any additional conversion
1955 // between fixed and scalable types is handled through integer types.
1956 auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
1957 auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
1958
1959 if (isa<ScalableVectorType>(Val: NewTy)) {
1960 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: NewVTy), EltTy: OldVTy))
1961 return false;
1962
1963 NewSize = TypeSize::getFixed(ExactSize: NewSize.getKnownMinValue() * VScale);
1964 } else {
1965 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: OldVTy), EltTy: NewVTy))
1966 return false;
1967
1968 OldSize = TypeSize::getFixed(ExactSize: OldSize.getKnownMinValue() * VScale);
1969 }
1970 }
1971
1972 if (NewSize != OldSize)
1973 return false;
1974 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
1975 return false;
1976
1977 // We can convert pointers to integers and vice-versa. Same for vectors
1978 // of pointers and integers.
1979 OldTy = OldTy->getScalarType();
1980 NewTy = NewTy->getScalarType();
1981 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
1982 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
1983 unsigned OldAS = OldTy->getPointerAddressSpace();
1984 unsigned NewAS = NewTy->getPointerAddressSpace();
1985 // Convert pointers if they are pointers from the same address space or
1986 // different integral (not non-integral) address spaces with the same
1987 // pointer size.
1988 return OldAS == NewAS ||
1989 (!DL.isNonIntegralAddressSpace(AddrSpace: OldAS) &&
1990 !DL.isNonIntegralAddressSpace(AddrSpace: NewAS) &&
1991 DL.getPointerSize(AS: OldAS) == DL.getPointerSize(AS: NewAS));
1992 }
1993
1994 // We can convert integers to integral pointers, but not to non-integral
1995 // pointers.
1996 if (OldTy->isIntegerTy())
1997 return !DL.isNonIntegralPointerType(Ty: NewTy);
1998
1999 // We can convert integral pointers to integers, but non-integral pointers
2000 // need to remain pointers.
2001 if (!DL.isNonIntegralPointerType(Ty: OldTy))
2002 return NewTy->isIntegerTy();
2003
2004 return false;
2005 }
2006
2007 if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy())
2008 return false;
2009
2010 return true;
2011}
2012
2013/// Test whether the given slice use can be promoted to a vector.
2014///
2015/// This function is called to test each entry in a partition which is slated
2016/// for a single slice.
2017static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2018 VectorType *Ty,
2019 uint64_t ElementSize,
2020 const DataLayout &DL,
2021 unsigned VScale) {
2022 // First validate the slice offsets.
2023 uint64_t BeginOffset =
2024 std::max(a: S.beginOffset(), b: P.beginOffset()) - P.beginOffset();
2025 uint64_t BeginIndex = BeginOffset / ElementSize;
2026 if (BeginIndex * ElementSize != BeginOffset ||
2027 BeginIndex >= cast<FixedVectorType>(Val: Ty)->getNumElements())
2028 return false;
2029 uint64_t EndOffset = std::min(a: S.endOffset(), b: P.endOffset()) - P.beginOffset();
2030 uint64_t EndIndex = EndOffset / ElementSize;
2031 if (EndIndex * ElementSize != EndOffset ||
2032 EndIndex > cast<FixedVectorType>(Val: Ty)->getNumElements())
2033 return false;
2034
2035 assert(EndIndex > BeginIndex && "Empty vector!");
2036 uint64_t NumElements = EndIndex - BeginIndex;
2037 Type *SliceTy = (NumElements == 1)
2038 ? Ty->getElementType()
2039 : FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElements);
2040
2041 Type *SplitIntTy =
2042 Type::getIntNTy(C&: Ty->getContext(), N: NumElements * ElementSize * 8);
2043
2044 Use *U = S.getUse();
2045
2046 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2047 if (MI->isVolatile())
2048 return false;
2049 if (!S.isSplittable())
2050 return false; // Skip any unsplittable intrinsics.
2051 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2052 if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2053 return false;
2054 } else if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2055 if (LI->isVolatile())
2056 return false;
2057 Type *LTy = LI->getType();
2058 // Disable vector promotion when there are loads or stores of an FCA.
2059 if (LTy->isStructTy())
2060 return false;
2061 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2062 assert(LTy->isIntegerTy());
2063 LTy = SplitIntTy;
2064 }
2065 if (!canConvertValue(DL, OldTy: SliceTy, NewTy: LTy, VScale))
2066 return false;
2067 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2068 if (SI->isVolatile())
2069 return false;
2070 Type *STy = SI->getValueOperand()->getType();
2071 // Disable vector promotion when there are loads or stores of an FCA.
2072 if (STy->isStructTy())
2073 return false;
2074 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2075 assert(STy->isIntegerTy());
2076 STy = SplitIntTy;
2077 }
2078 if (!canConvertValue(DL, OldTy: STy, NewTy: SliceTy, VScale))
2079 return false;
2080 } else {
2081 return false;
2082 }
2083
2084 return true;
2085}
2086
2087/// Test whether any vector type in \p CandidateTys is viable for promotion.
2088///
2089/// This implements the necessary checking for \c isVectorPromotionViable over
2090/// all slices of the alloca for the given VectorType.
2091static VectorType *
2092checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
2093 SmallVectorImpl<VectorType *> &CandidateTys,
2094 bool HaveCommonEltTy, Type *CommonEltTy,
2095 bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2096 VectorType *CommonVecPtrTy, unsigned VScale) {
2097 // If we didn't find a vector type, nothing to do here.
2098 if (CandidateTys.empty())
2099 return nullptr;
2100
2101 // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2102 // then we should choose it, not some other alternative.
2103 // But, we can't perform a no-op pointer address space change via bitcast,
2104 // so if we didn't have a common pointer element type, bail.
2105 if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2106 return nullptr;
2107
2108 // Try to pick the "best" element type out of the choices.
2109 if (!HaveCommonEltTy && HaveVecPtrTy) {
2110 // If there was a pointer element type, there's really only one choice.
2111 CandidateTys.clear();
2112 CandidateTys.push_back(Elt: CommonVecPtrTy);
2113 } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2114 // Integer-ify vector types.
2115 for (VectorType *&VTy : CandidateTys) {
2116 if (!VTy->getElementType()->isIntegerTy())
2117 VTy = cast<VectorType>(Val: VTy->getWithNewType(EltTy: IntegerType::getIntNTy(
2118 C&: VTy->getContext(), N: VTy->getScalarSizeInBits())));
2119 }
2120
2121 // Rank the remaining candidate vector types. This is easy because we know
2122 // they're all integer vectors. We sort by ascending number of elements.
2123 auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2124 (void)DL;
2125 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2126 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2127 "Cannot have vector types of different sizes!");
2128 assert(RHSTy->getElementType()->isIntegerTy() &&
2129 "All non-integer types eliminated!");
2130 assert(LHSTy->getElementType()->isIntegerTy() &&
2131 "All non-integer types eliminated!");
2132 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() <
2133 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2134 };
2135 auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2136 (void)DL;
2137 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2138 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2139 "Cannot have vector types of different sizes!");
2140 assert(RHSTy->getElementType()->isIntegerTy() &&
2141 "All non-integer types eliminated!");
2142 assert(LHSTy->getElementType()->isIntegerTy() &&
2143 "All non-integer types eliminated!");
2144 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() ==
2145 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2146 };
2147 llvm::sort(C&: CandidateTys, Comp: RankVectorTypesComp);
2148 CandidateTys.erase(CS: llvm::unique(R&: CandidateTys, P: RankVectorTypesEq),
2149 CE: CandidateTys.end());
2150 } else {
2151// The only way to have the same element type in every vector type is to
2152// have the same vector type. Check that and remove all but one.
2153#ifndef NDEBUG
2154 for (VectorType *VTy : CandidateTys) {
2155 assert(VTy->getElementType() == CommonEltTy &&
2156 "Unaccounted for element type!");
2157 assert(VTy == CandidateTys[0] &&
2158 "Different vector types with the same element type!");
2159 }
2160#endif
2161 CandidateTys.resize(N: 1);
2162 }
2163
2164 // FIXME: hack. Do we have a named constant for this?
2165 // SDAG SDNode can't have more than 65535 operands.
2166 llvm::erase_if(C&: CandidateTys, P: [](VectorType *VTy) {
2167 return cast<FixedVectorType>(Val: VTy)->getNumElements() >
2168 std::numeric_limits<unsigned short>::max();
2169 });
2170
2171 // Find a vector type viable for promotion by iterating over all slices.
2172 auto *VTy = llvm::find_if(Range&: CandidateTys, P: [&](VectorType *VTy) -> bool {
2173 uint64_t ElementSize =
2174 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2175
2176 // While the definition of LLVM vectors is bitpacked, we don't support sizes
2177 // that aren't byte sized.
2178 if (ElementSize % 8)
2179 return false;
2180 assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&
2181 "vector size not a multiple of element size?");
2182 ElementSize /= 8;
2183
2184 for (const Slice &S : P)
2185 if (!isVectorPromotionViableForSlice(P, S, Ty: VTy, ElementSize, DL, VScale))
2186 return false;
2187
2188 for (const Slice *S : P.splitSliceTails())
2189 if (!isVectorPromotionViableForSlice(P, S: *S, Ty: VTy, ElementSize, DL, VScale))
2190 return false;
2191
2192 return true;
2193 });
2194 return VTy != CandidateTys.end() ? *VTy : nullptr;
2195}
2196
2197static VectorType *createAndCheckVectorTypesForPromotion(
2198 SetVector<Type *> &OtherTys, ArrayRef<VectorType *> CandidateTysCopy,
2199 function_ref<void(Type *)> CheckCandidateType, Partition &P,
2200 const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2201 bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
2202 bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
2203 [[maybe_unused]] VectorType *OriginalElt =
2204 CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
2205 // Consider additional vector types where the element type size is a
2206 // multiple of load/store element size.
2207 for (Type *Ty : OtherTys) {
2208 if (!VectorType::isValidElementType(ElemTy: Ty))
2209 continue;
2210 unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2211 // Make a copy of CandidateTys and iterate through it, because we
2212 // might append to CandidateTys in the loop.
2213 for (VectorType *const VTy : CandidateTysCopy) {
2214 // The elements in the copy should remain invariant throughout the loop
2215 assert(CandidateTysCopy[0] == OriginalElt && "Different Element");
2216 unsigned VectorSize = DL.getTypeSizeInBits(Ty: VTy).getFixedValue();
2217 unsigned ElementSize =
2218 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2219 if (TypeSize != VectorSize && TypeSize != ElementSize &&
2220 VectorSize % TypeSize == 0) {
2221 VectorType *NewVTy = VectorType::get(ElementType: Ty, NumElements: VectorSize / TypeSize, Scalable: false);
2222 CheckCandidateType(NewVTy);
2223 }
2224 }
2225 }
2226
2227 return checkVectorTypesForPromotion(
2228 P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2229 HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2230}
2231
2232/// Test whether the given alloca partitioning and range of slices can be
2233/// promoted to a vector.
2234///
2235/// This is a quick test to check whether we can rewrite a particular alloca
2236/// partition (and its newly formed alloca) into a vector alloca with only
2237/// whole-vector loads and stores such that it could be promoted to a vector
2238/// SSA value. We only can ensure this for a limited set of operations, and we
2239/// don't want to do the rewrites unless we are confident that the result will
2240/// be promotable, so we have an early test here.
2241static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL,
2242 unsigned VScale) {
2243 // Collect the candidate types for vector-based promotion. Also track whether
2244 // we have different element types.
2245 SmallVector<VectorType *, 4> CandidateTys;
2246 SetVector<Type *> LoadStoreTys;
2247 SetVector<Type *> DeferredTys;
2248 Type *CommonEltTy = nullptr;
2249 VectorType *CommonVecPtrTy = nullptr;
2250 bool HaveVecPtrTy = false;
2251 bool HaveCommonEltTy = true;
2252 bool HaveCommonVecPtrTy = true;
2253 auto CheckCandidateType = [&](Type *Ty) {
2254 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2255 // Return if bitcast to vectors is different for total size in bits.
2256 if (!CandidateTys.empty()) {
2257 VectorType *V = CandidateTys[0];
2258 if (DL.getTypeSizeInBits(Ty: VTy).getFixedValue() !=
2259 DL.getTypeSizeInBits(Ty: V).getFixedValue()) {
2260 CandidateTys.clear();
2261 return;
2262 }
2263 }
2264 CandidateTys.push_back(Elt: VTy);
2265 Type *EltTy = VTy->getElementType();
2266
2267 if (!CommonEltTy)
2268 CommonEltTy = EltTy;
2269 else if (CommonEltTy != EltTy)
2270 HaveCommonEltTy = false;
2271
2272 if (EltTy->isPointerTy()) {
2273 HaveVecPtrTy = true;
2274 if (!CommonVecPtrTy)
2275 CommonVecPtrTy = VTy;
2276 else if (CommonVecPtrTy != VTy)
2277 HaveCommonVecPtrTy = false;
2278 }
2279 }
2280 };
2281
2282 // Put load and store types into a set for de-duplication.
2283 for (const Slice &S : P) {
2284 Type *Ty;
2285 if (auto *LI = dyn_cast<LoadInst>(Val: S.getUse()->getUser()))
2286 Ty = LI->getType();
2287 else if (auto *SI = dyn_cast<StoreInst>(Val: S.getUse()->getUser()))
2288 Ty = SI->getValueOperand()->getType();
2289 else
2290 continue;
2291
2292 auto CandTy = Ty->getScalarType();
2293 if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() ||
2294 S.endOffset() != P.endOffset())) {
2295 DeferredTys.insert(X: Ty);
2296 continue;
2297 }
2298
2299 LoadStoreTys.insert(X: Ty);
2300 // Consider any loads or stores that are the exact size of the slice.
2301 if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2302 CheckCandidateType(Ty);
2303 }
2304
2305 SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
2306 if (auto *VTy = createAndCheckVectorTypesForPromotion(
2307 OtherTys&: LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2308 CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2309 HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2310 return VTy;
2311
2312 CandidateTys.clear();
2313 return createAndCheckVectorTypesForPromotion(
2314 OtherTys&: DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2315 HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2316 CommonVecPtrTy, VScale);
2317}
2318
2319/// Test whether a slice of an alloca is valid for integer widening.
2320///
2321/// This implements the necessary checking for the \c isIntegerWideningViable
2322/// test below on a single slice of the alloca.
2323static bool isIntegerWideningViableForSlice(const Slice &S,
2324 uint64_t AllocBeginOffset,
2325 Type *AllocaTy,
2326 const DataLayout &DL,
2327 bool &WholeAllocaOp) {
2328 uint64_t Size = DL.getTypeStoreSize(Ty: AllocaTy).getFixedValue();
2329
2330 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2331 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2332
2333 Use *U = S.getUse();
2334
2335 // Lifetime intrinsics operate over the whole alloca whose sizes are usually
2336 // larger than other load/store slices (RelEnd > Size). But lifetime are
2337 // always promotable and should not impact other slices' promotability of the
2338 // partition.
2339 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2340 if (II->isLifetimeStartOrEnd() || II->isDroppable())
2341 return true;
2342 }
2343
2344 // We can't reasonably handle cases where the load or store extends past
2345 // the end of the alloca's type and into its padding.
2346 if (RelEnd > Size)
2347 return false;
2348
2349 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2350 if (LI->isVolatile())
2351 return false;
2352 // We can't handle loads that extend past the allocated memory.
2353 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
2354 if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
2355 return false;
2356 // So far, AllocaSliceRewriter does not support widening split slice tails
2357 // in rewriteIntegerLoad.
2358 if (S.beginOffset() < AllocBeginOffset)
2359 return false;
2360 // Note that we don't count vector loads or stores as whole-alloca
2361 // operations which enable integer widening because we would prefer to use
2362 // vector widening instead.
2363 if (!isa<VectorType>(Val: LI->getType()) && RelBegin == 0 && RelEnd == Size)
2364 WholeAllocaOp = true;
2365 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: LI->getType())) {
2366 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2367 return false;
2368 } else if (RelBegin != 0 || RelEnd != Size ||
2369 !canConvertValue(DL, OldTy: AllocaTy, NewTy: LI->getType())) {
2370 // Non-integer loads need to be convertible from the alloca type so that
2371 // they are promotable.
2372 return false;
2373 }
2374 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2375 Type *ValueTy = SI->getValueOperand()->getType();
2376 if (SI->isVolatile())
2377 return false;
2378 // We can't handle stores that extend past the allocated memory.
2379 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValueTy);
2380 if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
2381 return false;
2382 // So far, AllocaSliceRewriter does not support widening split slice tails
2383 // in rewriteIntegerStore.
2384 if (S.beginOffset() < AllocBeginOffset)
2385 return false;
2386 // Note that we don't count vector loads or stores as whole-alloca
2387 // operations which enable integer widening because we would prefer to use
2388 // vector widening instead.
2389 if (!isa<VectorType>(Val: ValueTy) && RelBegin == 0 && RelEnd == Size)
2390 WholeAllocaOp = true;
2391 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: ValueTy)) {
2392 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2393 return false;
2394 } else if (RelBegin != 0 || RelEnd != Size ||
2395 !canConvertValue(DL, OldTy: ValueTy, NewTy: AllocaTy)) {
2396 // Non-integer stores need to be convertible to the alloca type so that
2397 // they are promotable.
2398 return false;
2399 }
2400 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2401 if (MI->isVolatile() || !isa<Constant>(Val: MI->getLength()))
2402 return false;
2403 if (!S.isSplittable())
2404 return false; // Skip any unsplittable intrinsics.
2405 } else {
2406 return false;
2407 }
2408
2409 return true;
2410}
2411
2412/// Test whether the given alloca partition's integer operations can be
2413/// widened to promotable ones.
2414///
2415/// This is a quick test to check whether we can rewrite the integer loads and
2416/// stores to a particular alloca into wider loads and stores and be able to
2417/// promote the resulting alloca.
2418static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2419 const DataLayout &DL) {
2420 uint64_t SizeInBits = DL.getTypeSizeInBits(Ty: AllocaTy).getFixedValue();
2421 // Don't create integer types larger than the maximum bitwidth.
2422 if (SizeInBits > IntegerType::MAX_INT_BITS)
2423 return false;
2424
2425 // Don't try to handle allocas with bit-padding.
2426 if (SizeInBits != DL.getTypeStoreSizeInBits(Ty: AllocaTy).getFixedValue())
2427 return false;
2428
2429 // We need to ensure that an integer type with the appropriate bitwidth can
2430 // be converted to the alloca type, whatever that is. We don't want to force
2431 // the alloca itself to have an integer type if there is a more suitable one.
2432 Type *IntTy = Type::getIntNTy(C&: AllocaTy->getContext(), N: SizeInBits);
2433 if (!canConvertValue(DL, OldTy: AllocaTy, NewTy: IntTy) ||
2434 !canConvertValue(DL, OldTy: IntTy, NewTy: AllocaTy))
2435 return false;
2436
2437 // While examining uses, we ensure that the alloca has a covering load or
2438 // store. We don't want to widen the integer operations only to fail to
2439 // promote due to some other unsplittable entry (which we may make splittable
2440 // later). However, if there are only splittable uses, go ahead and assume
2441 // that we cover the alloca.
2442 // FIXME: We shouldn't consider split slices that happen to start in the
2443 // partition here...
2444 bool WholeAllocaOp = P.empty() && DL.isLegalInteger(Width: SizeInBits);
2445
2446 for (const Slice &S : P)
2447 if (!isIntegerWideningViableForSlice(S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2448 WholeAllocaOp))
2449 return false;
2450
2451 for (const Slice *S : P.splitSliceTails())
2452 if (!isIntegerWideningViableForSlice(S: *S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2453 WholeAllocaOp))
2454 return false;
2455
2456 return WholeAllocaOp;
2457}
2458
2459static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
2460 IntegerType *Ty, uint64_t Offset,
2461 const Twine &Name) {
2462 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2463 IntegerType *IntTy = cast<IntegerType>(Val: V->getType());
2464 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2465 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2466 "Element extends past full value");
2467 uint64_t ShAmt = 8 * Offset;
2468 if (DL.isBigEndian())
2469 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2470 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2471 if (ShAmt) {
2472 V = IRB.CreateLShr(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2473 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2474 }
2475 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2476 "Cannot extract to a larger integer!");
2477 if (Ty != IntTy) {
2478 V = IRB.CreateTrunc(V, DestTy: Ty, Name: Name + ".trunc");
2479 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2480 }
2481 return V;
2482}
2483
2484static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2485 Value *V, uint64_t Offset, const Twine &Name) {
2486 IntegerType *IntTy = cast<IntegerType>(Val: Old->getType());
2487 IntegerType *Ty = cast<IntegerType>(Val: V->getType());
2488 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2489 "Cannot insert a larger integer!");
2490 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2491 if (Ty != IntTy) {
2492 V = IRB.CreateZExt(V, DestTy: IntTy, Name: Name + ".ext");
2493 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2494 }
2495 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2496 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2497 "Element store outside of alloca store");
2498 uint64_t ShAmt = 8 * Offset;
2499 if (DL.isBigEndian())
2500 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2501 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2502 if (ShAmt) {
2503 V = IRB.CreateShl(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2504 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2505 }
2506
2507 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2508 APInt Mask = ~Ty->getMask().zext(width: IntTy->getBitWidth()).shl(shiftAmt: ShAmt);
2509 Old = IRB.CreateAnd(LHS: Old, RHS: Mask, Name: Name + ".mask");
2510 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2511 V = IRB.CreateOr(LHS: Old, RHS: V, Name: Name + ".insert");
2512 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2513 }
2514 return V;
2515}
2516
2517static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2518 unsigned EndIndex, const Twine &Name) {
2519 auto *VecTy = cast<FixedVectorType>(Val: V->getType());
2520 unsigned NumElements = EndIndex - BeginIndex;
2521 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2522
2523 if (NumElements == VecTy->getNumElements())
2524 return V;
2525
2526 if (NumElements == 1) {
2527 V = IRB.CreateExtractElement(Vec: V, Idx: BeginIndex, Name: Name + ".extract");
2528 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2529 return V;
2530 }
2531
2532 auto Mask = llvm::to_vector<8>(Range: llvm::seq<int>(Begin: BeginIndex, End: EndIndex));
2533 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".extract");
2534 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2535 return V;
2536}
2537
2538static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
2539 unsigned BeginIndex, const Twine &Name) {
2540 VectorType *VecTy = cast<VectorType>(Val: Old->getType());
2541 assert(VecTy && "Can only insert a vector into a vector");
2542
2543 VectorType *Ty = dyn_cast<VectorType>(Val: V->getType());
2544 if (!Ty) {
2545 // Single element to insert.
2546 V = IRB.CreateInsertElement(Vec: Old, NewElt: V, Idx: BeginIndex, Name: Name + ".insert");
2547 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2548 return V;
2549 }
2550
2551 unsigned NumSubElements = cast<FixedVectorType>(Val: Ty)->getNumElements();
2552 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2553
2554 assert(NumSubElements <= NumElements && "Too many elements!");
2555 if (NumSubElements == NumElements) {
2556 assert(V->getType() == VecTy && "Vector type mismatch");
2557 return V;
2558 }
2559 unsigned EndIndex = BeginIndex + NumSubElements;
2560
2561 // When inserting a smaller vector into the larger to store, we first
2562 // use a shuffle vector to widen it with undef elements, and then
2563 // a second shuffle vector to select between the loaded vector and the
2564 // incoming vector.
2565 SmallVector<int, 8> Mask;
2566 Mask.reserve(N: NumElements);
2567 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2568 if (Idx >= BeginIndex && Idx < EndIndex)
2569 Mask.push_back(Elt: Idx - BeginIndex);
2570 else
2571 Mask.push_back(Elt: -1);
2572 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".expand");
2573 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2574
2575 Mask.clear();
2576 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2577 if (Idx >= BeginIndex && Idx < EndIndex)
2578 Mask.push_back(Elt: Idx);
2579 else
2580 Mask.push_back(Elt: Idx + NumElements);
2581 V = IRB.CreateShuffleVector(V1: V, V2: Old, Mask, Name: Name + "blend");
2582 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2583 return V;
2584}
2585
2586/// This function takes two vector values and combines them into a single vector
2587/// by concatenating their elements. The function handles:
2588///
2589/// 1. Element type mismatch: If either vector's element type differs from
2590/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2591/// preserving the total bit width (adjusting the number of elements
2592/// accordingly).
2593///
2594/// 2. Size mismatch: After transforming the vectors to have the desired element
2595/// type, if the two vectors have different numbers of elements, the smaller
2596/// vector is extended with poison values to match the size of the larger
2597/// vector before concatenation.
2598///
2599/// 3. Concatenation: The vectors are merged using a shuffle operation that
2600/// places all elements of V0 first, followed by all elements of V1.
2601///
2602/// \param V0 The first vector to merge (must be a vector type)
2603/// \param V1 The second vector to merge (must be a vector type)
2604/// \param DL The data layout for size calculations
2605/// \param NewAIEltTy The desired element type for the result vector
2606/// \param Builder IRBuilder for creating new instructions
2607/// \return A new vector containing all elements from V0 followed by all
2608/// elements from V1
2609static Value *mergeTwoVectors(Value *V0, Value *V1, const DataLayout &DL,
2610 Type *NewAIEltTy, IRBuilder<> &Builder) {
2611 // V0 and V1 are vectors
2612 // Create a new vector type with combined elements
2613 // Use ShuffleVector to concatenate the vectors
2614 auto *VecType0 = cast<FixedVectorType>(Val: V0->getType());
2615 auto *VecType1 = cast<FixedVectorType>(Val: V1->getType());
2616
2617 // If V0/V1 element types are different from NewAllocaElementType,
2618 // we need to introduce bitcasts before merging them
2619 auto BitcastIfNeeded = [&](Value *&V, FixedVectorType *&VecType,
2620 const char *DebugName) {
2621 Type *EltType = VecType->getElementType();
2622 if (EltType != NewAIEltTy) {
2623 // Calculate new number of elements to maintain same bit width
2624 unsigned TotalBits =
2625 VecType->getNumElements() * DL.getTypeSizeInBits(Ty: EltType);
2626 unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(Ty: NewAIEltTy);
2627
2628 auto *NewVecType = FixedVectorType::get(ElementType: NewAIEltTy, NumElts: NewNumElts);
2629 V = Builder.CreateBitCast(V, DestTy: NewVecType);
2630 VecType = NewVecType;
2631 LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2632 }
2633 };
2634
2635 BitcastIfNeeded(V0, VecType0, "V0");
2636 BitcastIfNeeded(V1, VecType1, "V1");
2637
2638 unsigned NumElts0 = VecType0->getNumElements();
2639 unsigned NumElts1 = VecType1->getNumElements();
2640
2641 SmallVector<int, 16> ShuffleMask;
2642
2643 if (NumElts0 == NumElts1) {
2644 for (unsigned i = 0; i < NumElts0 + NumElts1; ++i)
2645 ShuffleMask.push_back(Elt: i);
2646 } else {
2647 // If two vectors have different sizes, we need to extend
2648 // the smaller vector to the size of the larger vector.
2649 unsigned SmallSize = std::min(a: NumElts0, b: NumElts1);
2650 unsigned LargeSize = std::max(a: NumElts0, b: NumElts1);
2651 bool IsV0Smaller = NumElts0 < NumElts1;
2652 Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2653 SmallVector<int, 16> ExtendMask;
2654 for (unsigned i = 0; i < SmallSize; ++i)
2655 ExtendMask.push_back(Elt: i);
2656 for (unsigned i = SmallSize; i < LargeSize; ++i)
2657 ExtendMask.push_back(Elt: PoisonMaskElem);
2658 ExtendedVec = Builder.CreateShuffleVector(
2659 V1: ExtendedVec, V2: PoisonValue::get(T: ExtendedVec->getType()), Mask: ExtendMask);
2660 LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2661 for (unsigned i = 0; i < NumElts0; ++i)
2662 ShuffleMask.push_back(Elt: i);
2663 for (unsigned i = 0; i < NumElts1; ++i)
2664 ShuffleMask.push_back(Elt: LargeSize + i);
2665 }
2666
2667 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
2668}
2669
2670namespace {
2671
2672/// Visitor to rewrite instructions using p particular slice of an alloca
2673/// to use a new alloca.
2674///
2675/// Also implements the rewriting to vector-based accesses when the partition
2676/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2677/// lives here.
2678class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2679 // Befriend the base class so it can delegate to private visit methods.
2680 friend class InstVisitor<AllocaSliceRewriter, bool>;
2681
2682 using Base = InstVisitor<AllocaSliceRewriter, bool>;
2683
2684 const DataLayout &DL;
2685 AllocaSlices &AS;
2686 SROA &Pass;
2687 AllocaInst &OldAI, &NewAI;
2688 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2689 Type *NewAllocaTy;
2690
2691 // This is a convenience and flag variable that will be null unless the new
2692 // alloca's integer operations should be widened to this integer type due to
2693 // passing isIntegerWideningViable above. If it is non-null, the desired
2694 // integer type will be stored here for easy access during rewriting.
2695 IntegerType *IntTy;
2696
2697 // If we are rewriting an alloca partition which can be written as pure
2698 // vector operations, we stash extra information here. When VecTy is
2699 // non-null, we have some strict guarantees about the rewritten alloca:
2700 // - The new alloca is exactly the size of the vector type here.
2701 // - The accesses all either map to the entire vector or to a single
2702 // element.
2703 // - The set of accessing instructions is only one of those handled above
2704 // in isVectorPromotionViable. Generally these are the same access kinds
2705 // which are promotable via mem2reg.
2706 VectorType *VecTy;
2707 Type *ElementTy;
2708 uint64_t ElementSize;
2709
2710 // The original offset of the slice currently being rewritten relative to
2711 // the original alloca.
2712 uint64_t BeginOffset = 0;
2713 uint64_t EndOffset = 0;
2714
2715 // The new offsets of the slice currently being rewritten relative to the
2716 // original alloca.
2717 uint64_t NewBeginOffset = 0, NewEndOffset = 0;
2718
2719 uint64_t SliceSize = 0;
2720 bool IsSplittable = false;
2721 bool IsSplit = false;
2722 Use *OldUse = nullptr;
2723 Instruction *OldPtr = nullptr;
2724
2725 // Track post-rewrite users which are PHI nodes and Selects.
2726 SmallSetVector<PHINode *, 8> &PHIUsers;
2727 SmallSetVector<SelectInst *, 8> &SelectUsers;
2728
2729 // Utility IR builder, whose name prefix is setup for each visited use, and
2730 // the insertion point is set to point to the user.
2731 IRBuilderTy IRB;
2732
2733 // Return the new alloca, addrspacecasted if required to avoid changing the
2734 // addrspace of a volatile access.
2735 Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) {
2736 if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace())
2737 return &NewAI;
2738
2739 Type *AccessTy = IRB.getPtrTy(AddrSpace);
2740 return IRB.CreateAddrSpaceCast(V: &NewAI, DestTy: AccessTy);
2741 }
2742
2743public:
2744 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2745 AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2746 uint64_t NewAllocaBeginOffset,
2747 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2748 VectorType *PromotableVecTy,
2749 SmallSetVector<PHINode *, 8> &PHIUsers,
2750 SmallSetVector<SelectInst *, 8> &SelectUsers)
2751 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2752 NewAllocaBeginOffset(NewAllocaBeginOffset),
2753 NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2754 IntTy(IsIntegerPromotable
2755 ? Type::getIntNTy(
2756 C&: NewAI.getContext(),
2757 N: DL.getTypeSizeInBits(Ty: NewAllocaTy).getFixedValue())
2758 : nullptr),
2759 VecTy(PromotableVecTy),
2760 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2761 ElementSize(VecTy ? DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8
2762 : 0),
2763 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2764 IRB(NewAI.getContext(), ConstantFolder()) {
2765 if (VecTy) {
2766 assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
2767 "Only multiple-of-8 sized vector elements are viable");
2768 ++NumVectorized;
2769 }
2770 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
2771 }
2772
2773 bool visit(AllocaSlices::const_iterator I) {
2774 bool CanSROA = true;
2775 BeginOffset = I->beginOffset();
2776 EndOffset = I->endOffset();
2777 IsSplittable = I->isSplittable();
2778 IsSplit =
2779 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
2780 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2781 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2782 LLVM_DEBUG(dbgs() << "\n");
2783
2784 // Compute the intersecting offset range.
2785 assert(BeginOffset < NewAllocaEndOffset);
2786 assert(EndOffset > NewAllocaBeginOffset);
2787 NewBeginOffset = std::max(a: BeginOffset, b: NewAllocaBeginOffset);
2788 NewEndOffset = std::min(a: EndOffset, b: NewAllocaEndOffset);
2789
2790 SliceSize = NewEndOffset - NewBeginOffset;
2791 LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2792 << ") NewBegin:(" << NewBeginOffset << ", "
2793 << NewEndOffset << ") NewAllocaBegin:("
2794 << NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2795 << ")\n");
2796 assert(IsSplit || NewBeginOffset == BeginOffset);
2797 OldUse = I->getUse();
2798 OldPtr = cast<Instruction>(Val: OldUse->get());
2799
2800 Instruction *OldUserI = cast<Instruction>(Val: OldUse->getUser());
2801 IRB.SetInsertPoint(OldUserI);
2802 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2803 // Avoid materializing the name prefix when it is discarded anyway.
2804 if (!IRB.getContext().shouldDiscardValueNames())
2805 IRB.getInserter().SetNamePrefix(Twine(NewAI.getName()) + "." +
2806 Twine(BeginOffset) + ".");
2807
2808 CanSROA &= visit(I: cast<Instruction>(Val: OldUse->getUser()));
2809 if (VecTy || IntTy)
2810 assert(CanSROA);
2811 return CanSROA;
2812 }
2813
2814 /// Attempts to rewrite a partition using tree-structured merge optimization.
2815 ///
2816 /// This function handles two patterns. Both produce an O(log n) tree of
2817 /// shufflevectors in place of the linear expand+blend chain that SROA would
2818 /// otherwise emit for each partial store.
2819 ///
2820 /// Pattern 1 (stores-only):
2821 /// Multiple non-overlapping partial stores completely fill the alloca
2822 /// and there is exactly one full-width load coming after the stores.
2823 /// The stores are tree-merged into a single vector and stored once.
2824 ///
2825 /// Example transformation:
2826 /// Before: (stores do not have to be in order)
2827 /// %alloca = alloca <8 x float>
2828 /// store <2 x float> %val0, ptr %alloca ; offset 0-1
2829 /// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2830 /// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2831 /// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2832 /// %r = load <8 x float>, ptr %alloca
2833 ///
2834 /// After: tree of shufflevectors producing <8 x float> directly.
2835 ///
2836 /// Pattern 2 (init + RMW, possibly multi-round):
2837 /// A single full-width init store, followed by partial loads and
2838 /// partial stores that read-modify-write the alloca one or more
2839 /// times, optionally followed by a full-width load. The only
2840 /// structural requirement is that the distinct [begin, end) ranges
2841 /// touched by the partial loads and stores, taken together, tile
2842 /// the alloca disjointly.
2843 ///
2844 /// We keep a map from each slice range to the SSA value that
2845 /// currently lives there, `SliceValues[r] -> Value*`:
2846 /// - initialize each entry to the corresponding piece of the
2847 /// init store's value (via a shufflevector picking the
2848 /// range's elements out of the init value),
2849 /// - walk partial loads and stores in block order,
2850 /// - for a partial load at range r: RAUW with `SliceValues[r]`,
2851 /// - for a partial store at range r: update `SliceValues[r]` to
2852 /// the stored value and drop the store.
2853 /// At the end, the final `SliceValues[r]` entries are tree-merged
2854 /// (in range order) into a single store to the alloca, and the
2855 /// optional full-width load is replaced by a load of the alloca.
2856 ///
2857 /// Because the ranges are disjoint by construction, a store at one
2858 /// range cannot affect another range's tracked value, so a single
2859 /// block-order walk correctly tracks the memory state at each
2860 /// range. The algorithm handles multi-round RMW, partial loads
2861 /// and stores interleaved in any order, read-only slices (the
2862 /// tracked value stays at the init extract), and write-only
2863 /// slices (the tracked value never flows into a load).
2864 ///
2865 /// \param P The partition to analyze and potentially rewrite
2866 /// \return An optional vector of values that were deleted during the
2867 /// rewrite, or std::nullopt if the partition cannot be optimized.
2868 std::optional<SmallVector<Value *, 4>>
2869 rewriteTreeStructuredMerge(Partition &P) {
2870 // No tail slices that overlap with the partition
2871 if (P.splitSliceTails().size() > 0)
2872 return std::nullopt;
2873
2874 // Structure to hold store information
2875 struct StoreInfo {
2876 StoreInst *Store;
2877 uint64_t BeginOffset;
2878 uint64_t EndOffset;
2879 Value *StoredValue;
2880 StoreInfo(StoreInst *SI, uint64_t Begin, uint64_t End, Value *Val)
2881 : Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
2882 };
2883 struct LoadInfo {
2884 LoadInst *Load;
2885 uint64_t BeginOffset;
2886 uint64_t EndOffset;
2887 };
2888
2889 SmallVector<StoreInfo, 4> StoreInfos; // partial stores only
2890 SmallVector<LoadInfo, 4> LoadInfos; // partial loads only
2891 LoadInst *FullLoad = nullptr; // optional full-width load
2892 StoreInst *InitStore = nullptr; // optional full-width init store
2893
2894 // If the new alloca is a fixed vector type, we use its element type as the
2895 // allocated element type, otherwise we use i8 as the allocated element
2896 Type *AllocatedEltTy =
2897 isa<FixedVectorType>(Val: NewAllocaTy)
2898 ? cast<FixedVectorType>(Val: NewAllocaTy)->getElementType()
2899 : Type::getInt8Ty(C&: NewAI.getContext());
2900 unsigned AllocatedEltTySize = DL.getTypeSizeInBits(Ty: AllocatedEltTy);
2901
2902 // Helper to check if a type is
2903 // 1. A fixed vector type
2904 // 2. The element type is not a pointer
2905 // 3. The element type size is byte-aligned
2906 // We only handle the cases that the ld/st meet these conditions
2907 auto IsTypeValidForTreeStructuredMerge = [&](Type *Ty) -> bool {
2908 auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: Ty);
2909 return FixedVecTy &&
2910 DL.getTypeSizeInBits(Ty: FixedVecTy->getElementType()) % 8 == 0 &&
2911 !FixedVecTy->getElementType()->isPointerTy();
2912 };
2913
2914 for (Slice &S : P) {
2915 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
2916 // A "full-width" slice spans the entire alloca; it's either the single
2917 // init store (Pattern 2) or the single final load (both patterns).
2918 bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
2919 S.endOffset() == NewAllocaEndOffset);
2920 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
2921 // Only handle simple (non-volatile, non-atomic) loads.
2922 if (!LI->isSimple() ||
2923 !IsTypeValidForTreeStructuredMerge(LI->getType()))
2924 return std::nullopt;
2925 if (IsFullWidth) {
2926 // We accept at most one full-width load (the "final" load, after
2927 // all the partial stores).
2928 if (FullLoad)
2929 return std::nullopt;
2930 FullLoad = LI;
2931 } else {
2932 // Partial load (RMW pattern only).
2933 LoadInfos.push_back(Elt: {.Load: LI, .BeginOffset: S.beginOffset(), .EndOffset: S.endOffset()});
2934 }
2935 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
2936 // Do not handle the case if
2937 // 1. The store does not meet the conditions in the helper function
2938 // 2. The store is not simple — we drop stores as part of the
2939 // rewrite, so volatile stores (which must be kept) and atomic
2940 // stores (which carry memory-ordering semantics) are unsound
2941 // to replace with SSA bookkeeping.
2942 // 3. The total store size is not a multiple of the allocated
2943 // element type size (required so the tree merge can produce a
2944 // vector whose element type matches the alloca).
2945 if (!SI->isSimple() || !IsTypeValidForTreeStructuredMerge(
2946 SI->getValueOperand()->getType()))
2947 return std::nullopt;
2948 auto *StVecTy = cast<FixedVectorType>(Val: SI->getValueOperand()->getType());
2949 unsigned NumElts = StVecTy->getNumElements();
2950 unsigned EltSize = DL.getTypeSizeInBits(Ty: StVecTy->getElementType());
2951 if (NumElts * EltSize % AllocatedEltTySize != 0)
2952 return std::nullopt;
2953 if (IsFullWidth) {
2954 // At most one full-width store is allowed — it's the init store
2955 // for the RMW pattern.
2956 if (InitStore)
2957 return std::nullopt;
2958 InitStore = SI;
2959 } else {
2960 StoreInfos.emplace_back(Args&: SI, Args: S.beginOffset(), Args: S.endOffset(),
2961 Args: SI->getValueOperand());
2962 }
2963 } else {
2964 // If we have instructions other than load and store, we cannot do
2965 // the tree structured merge.
2966 return std::nullopt;
2967 }
2968 }
2969
2970 // Need at least two partial stores to benefit from tree-merging; a
2971 // single store is already optimal as-is. This applies to both patterns
2972 // below, so check it before classifying.
2973 if (StoreInfos.size() < 2)
2974 return std::nullopt;
2975
2976 // Classify the pattern by looking at what we collected:
2977 // Pattern 1 (stores-only): only partial stores + exactly one full load.
2978 // Pattern 2 (RMW): one full init store + partial loads + partial stores
2979 // (+ optional full final load). RMW also needs VecTy to be set
2980 // because we use getIndex() to convert byte offsets to element
2981 // indices, which requires a promoted vector alloca.
2982 bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
2983 bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
2984 if (!IsRMWPattern && !IsStoresOnlyPattern)
2985 return std::nullopt;
2986
2987 // All partial stores must live in the same basic block — the tree merge
2988 // is built in a single BB using block-order ordering (comesBefore).
2989 BasicBlock *StoreBB = StoreInfos[0].Store->getParent();
2990 for (auto &Info : StoreInfos)
2991 if (Info.Store->getParent() != StoreBB)
2992 return std::nullopt;
2993
2994 SmallVector<Value *, 4> DeletedValues;
2995
2996 // Helper: pairwise tree-merge a list of vectors into a single vector.
2997 // At each iteration we merge each adjacent pair via mergeTwoVectors,
2998 // collect the merged values into Next, and (if Vals had odd length)
2999 // carry the trailing element through unchanged. Loop until one value
3000 // remains — the fully-merged vector.
3001 auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3002 IRBuilder<> &B) -> Value * {
3003 LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3004 while (Vals.size() > 1) {
3005 SmallVector<Value *, 8> Next;
3006 for (unsigned I = 0, E = Vals.size(); I + 1 < E; I += 2) {
3007 Value *M =
3008 mergeTwoVectors(V0: Vals[I], V1: Vals[I + 1], DL, NewAIEltTy: AllocatedEltTy, Builder&: B);
3009 LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3010 Next.push_back(Elt: M);
3011 }
3012 if (Vals.size() % 2 == 1)
3013 Next.push_back(Elt: Vals.back());
3014 Vals = std::move(Next);
3015 }
3016 return Vals[0];
3017 };
3018
3019 // Replace a full-width load with a load of the freshly-merged alloca.
3020 // The merge stored a value of type Merged->getType() into NewAI; we load
3021 // that same type back so every access to NewAI stays consistently typed
3022 // (otherwise the alloca is no longer promotable).
3023 auto ReplaceFullLoad = [&](LoadInst *LoadToReplace, Value *Merged) {
3024 IRBuilder<> LoadBuilder(LoadToReplace);
3025 Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3026 Ty: Merged->getType(), Ptr: &NewAI, Align: getSliceAlign(),
3027 isVolatile: LoadToReplace->isVolatile(),
3028 Name: LoadToReplace->getName() + ".sroa.new.load");
3029 if (NewLoad->getType() != LoadToReplace->getType())
3030 NewLoad = LoadBuilder.CreateBitCast(V: NewLoad, DestTy: LoadToReplace->getType());
3031 LoadToReplace->replaceAllUsesWith(V: NewLoad);
3032 DeletedValues.push_back(Elt: LoadToReplace);
3033 };
3034
3035 if (IsStoresOnlyPattern) {
3036 // Stores should not overlap and should cover the whole alloca.
3037 // Sort by begin offset to verify this with a single linear scan.
3038 llvm::sort(C&: StoreInfos, Comp: [](const StoreInfo &A, const StoreInfo &B) {
3039 return A.BeginOffset < B.BeginOffset;
3040 });
3041 // Check for gap or overlap: each begin offset must equal the previous
3042 // end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3043 // NewAllocaEndOffset) exactly.
3044 uint64_t Expected = NewAllocaBeginOffset;
3045 for (auto &Info : StoreInfos) {
3046 if (Info.BeginOffset != Expected)
3047 return std::nullopt;
3048 Expected = Info.EndOffset;
3049 }
3050 // Stores cover the entire alloca (no trailing gap either).
3051 if (Expected != NewAllocaEndOffset)
3052 return std::nullopt;
3053
3054 // The load should not be in the middle of the stores.
3055 // Note:
3056 // If the load is in a different basic block from the stores, we can
3057 // still do the tree-structured merge. We don't have store->load
3058 // forwarding here — the merged vector is stored back to NewAI and
3059 // the new load loads from NewAI. The forwarding will be handled
3060 // later when NewAI is promoted.
3061 BasicBlock *LoadBB = FullLoad->getParent();
3062 if (LoadBB == StoreBB) {
3063 for (auto &Info : StoreInfos)
3064 if (!Info.Store->comesBefore(Other: FullLoad))
3065 return std::nullopt;
3066 }
3067
3068 LLVM_DEBUG({
3069 dbgs() << "Tree structured merge rewrite (stores-only):\n";
3070 dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3071 for (auto [I, Info] : enumerate(StoreInfos)) {
3072 dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3073 << Info.EndOffset << ") \tStore: " << *Info.Store
3074 << "\tValue: " << *Info.StoredValue << "\n";
3075 }
3076 });
3077
3078 // StoreInfos is sorted by offset, not by block order. Anchoring to
3079 // StoreInfos.back().Store (last by offset) can place shuffles before
3080 // operands that appear later in the block (invalid SSA). Insert before
3081 // FullLoad when it shares the store block (after all stores, before
3082 // any later IR in that block). Otherwise insert before the store
3083 // block's terminator so the merge runs after every store and any
3084 // trailing instructions in that block.
3085 IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(Val: FullLoad)
3086 : StoreBB->getTerminator());
3087 SmallVector<Value *, 8> Vals;
3088 for (const auto &Info : StoreInfos) {
3089 DeletedValues.push_back(Elt: Info.Store);
3090 Vals.push_back(Elt: Info.StoredValue);
3091 }
3092 // Merge all stored values and store the merged value into the alloca.
3093 Value *Merged = TreeMerge(Vals, Builder);
3094 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3095
3096 // Replace the original load with a load of the newly-merged alloca.
3097 ReplaceFullLoad(FullLoad, Merged);
3098 return DeletedValues;
3099 }
3100
3101 // RMW pattern handling starts from here.
3102 // Like StoreBB above: keep the init store, all partial loads and all
3103 // partial stores in one basic block so we can reason about ordering
3104 // with comesBefore and build SSA without PHIs.
3105 if (InitStore->getParent() != StoreBB)
3106 return std::nullopt;
3107 if (any_of(Range&: LoadInfos, P: [&](const LoadInfo &I) {
3108 return I.Load->getParent() != StoreBB;
3109 }))
3110 return std::nullopt;
3111 // FullLoad (if any) is allowed to live in a different basic block. See
3112 // the note on the stores-only path: we don't do store->load forwarding
3113 // directly — the merged vector is stored to NewAI and the new load
3114 // loads from NewAI, so cross-BB ordering is resolved later when NewAI
3115 // is promoted.
3116
3117 // Collect the combined partial-load/partial-store accesses sorted
3118 // by block order. Used both for ordering checks and for the rewrite
3119 // walk below.
3120 struct Access {
3121 Instruction *Inst;
3122 uint64_t BeginOffset, EndOffset;
3123 bool IsStore;
3124 };
3125 SmallVector<Access, 16> Accesses;
3126 Accesses.reserve(N: LoadInfos.size() + StoreInfos.size());
3127 for (const auto &L : LoadInfos)
3128 Accesses.push_back(Elt: {.Inst: L.Load, .BeginOffset: L.BeginOffset, .EndOffset: L.EndOffset, .IsStore: false});
3129 for (const auto &S : StoreInfos)
3130 Accesses.push_back(Elt: {.Inst: S.Store, .BeginOffset: S.BeginOffset, .EndOffset: S.EndOffset, .IsStore: true});
3131 llvm::sort(C&: Accesses, Comp: [](const Access &A, const Access &B) {
3132 return A.Inst->comesBefore(Other: B.Inst);
3133 });
3134
3135 // Ordering constraint 1: InitStore must come before every partial
3136 // access — they read/write the RMW state initialised by InitStore.
3137 // Accesses is sorted by block order, so the first element is the
3138 // earliest; checking it is enough.
3139 if (!InitStore->comesBefore(Other: Accesses.front().Inst))
3140 return std::nullopt;
3141 // Ordering constraint 2: when FullLoad shares the block with the
3142 // partial accesses, it must come after every one of them — otherwise
3143 // it could read a stale value. Accesses is sorted, so the last
3144 // element is the latest; checking it is enough. If FullLoad is in
3145 // another block, mem2reg forwards the merged store to it.
3146 if (FullLoad && FullLoad->getParent() == StoreBB &&
3147 !Accesses.back().Inst->comesBefore(Other: FullLoad))
3148 return std::nullopt;
3149
3150 // Coverage check: the distinct [begin, end) ranges touched by the
3151 // partial loads and stores must tile the alloca disjointly. That is
3152 // the only precondition the per-range SliceValues tracking below
3153 // needs — a disjoint tile guarantees the entries don't alias each
3154 // other. We don't check per-range load/store counts: a range with
3155 // only loads ends with SliceValues[r] = the init extract
3156 // (contributed to the final tree-merge), and a range with only
3157 // stores ends with SliceValues[r] = its last stored value. Both are
3158 // correct.
3159 using SliceRange = std::pair<uint64_t, uint64_t>;
3160 SmallVector<SliceRange, 8> SortedRanges;
3161 SortedRanges.reserve(N: Accesses.size());
3162 for (auto &Acc : Accesses)
3163 SortedRanges.emplace_back(Args&: Acc.BeginOffset, Args&: Acc.EndOffset);
3164 llvm::sort(C&: SortedRanges);
3165 SortedRanges.erase(CS: llvm::unique(R&: SortedRanges), CE: SortedRanges.end());
3166 // Disjoint + contiguous tile of the whole alloca.
3167 uint64_t Expected = NewAllocaBeginOffset;
3168 for (auto &Range : SortedRanges) {
3169 if (Range.first != Expected)
3170 return std::nullopt;
3171 Expected = Range.second;
3172 }
3173 if (Expected != NewAllocaEndOffset)
3174 return std::nullopt;
3175
3176 LLVM_DEBUG({
3177 dbgs() << "Tree structured merge rewrite (RMW):\n";
3178 dbgs() << " Init store: " << *InitStore << "\n";
3179 if (FullLoad)
3180 dbgs() << " Final load: " << *FullLoad << "\n";
3181 dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3182 for (auto &Range : SortedRanges)
3183 dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3184 });
3185
3186 // Initialize SliceValues: one SSA value per slice range, tracking
3187 // the value the alloca currently holds at that range. Each entry
3188 // starts at the corresponding piece of the init store, obtained by
3189 // bitcasting the init value to the alloca's vector type (if needed)
3190 // and extracting the slice's sub-range.
3191 IRB.SetInsertPoint(InitStore->getNextNode());
3192 Value *InitVec = InitStore->getValueOperand();
3193 if (InitVec->getType() != NewAllocaTy)
3194 InitVec = IRB.CreateBitCast(V: InitVec, DestTy: NewAllocaTy, Name: "init.cast");
3195 DenseMap<SliceRange, Value *> SliceValues;
3196 for (auto &Range : SortedRanges) {
3197 unsigned BeginIdx = getIndex(Offset: Range.first);
3198 unsigned EndIdx = getIndex(Offset: Range.second);
3199 SliceValues[Range] = IRB.CreateShuffleVector(
3200 V: InitVec, Mask: createSequentialMask(Start: BeginIdx, NumInts: EndIdx - BeginIdx, NumUndefs: 0),
3201 Name: "init.extract");
3202 }
3203 // The init store itself becomes dead — its value is consumed via the
3204 // extracts above.
3205 DeletedValues.push_back(Elt: InitStore);
3206
3207 // Walk accesses in block order:
3208 // - partial load at range r: replace with SliceValues[r] (bitcast
3209 // if the load's type differs from the current tracked value's
3210 // type, e.g. because a previous store wrote a vector with a
3211 // different element type);
3212 // - partial store at range r: update SliceValues[r] to the stored
3213 // value and drop the store.
3214 for (auto &Acc : Accesses) {
3215 SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3216 if (!Acc.IsStore) {
3217 Value *V = SliceValues[Range];
3218 if (V->getType() != Acc.Inst->getType()) {
3219 IRB.SetInsertPoint(cast<LoadInst>(Val: Acc.Inst));
3220 V = IRB.CreateBitCast(V, DestTy: Acc.Inst->getType());
3221 }
3222 Acc.Inst->replaceAllUsesWith(V);
3223 } else {
3224 SliceValues[Range] = cast<StoreInst>(Val: Acc.Inst)->getValueOperand();
3225 }
3226 DeletedValues.push_back(Elt: Acc.Inst);
3227 }
3228
3229 // Tree-merge the final per-range values (in range order) into the
3230 // alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3231 // shares the partial-access block) or otherwise to the block's
3232 // terminator — never to a partial access, since those are queued for
3233 // deletion. Both anchors are guaranteed to dominate every SliceValues
3234 // entry: each one is either an init extract (before any access) or a
3235 // stored value defined before its (now-deleted) store.
3236 IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3237 ? cast<Instruction>(Val: FullLoad)
3238 : StoreBB->getTerminator());
3239 SmallVector<Value *, 8> Vals;
3240 for (auto &Range : SortedRanges)
3241 Vals.push_back(Elt: SliceValues[Range]);
3242 Value *Merged = TreeMerge(Vals, Builder);
3243 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3244
3245 // Replace the optional final full-width load with a load of the newly
3246 // merged alloca. Later promotion will forward the store above to it.
3247 if (FullLoad)
3248 ReplaceFullLoad(FullLoad, Merged);
3249
3250 return DeletedValues;
3251 }
3252
3253private:
3254 // Make sure the other visit overloads are visible.
3255 using Base::visit;
3256
3257 // Every instruction which can end up as a user must have a rewrite rule.
3258 bool visitInstruction(Instruction &I) {
3259 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3260 llvm_unreachable("No rewrite rule for this instruction!");
3261 }
3262
3263 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
3264 // Note that the offset computation can use BeginOffset or NewBeginOffset
3265 // interchangeably for unsplit slices.
3266 assert(IsSplit || BeginOffset == NewBeginOffset);
3267 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3268
3269 StringRef OldName = OldPtr->getName();
3270 // Skip through the last '.sroa.' component of the name.
3271 size_t LastSROAPrefix = OldName.rfind(Str: ".sroa.");
3272 if (LastSROAPrefix != StringRef::npos) {
3273 OldName = OldName.substr(Start: LastSROAPrefix + strlen(s: ".sroa."));
3274 // Look for an SROA slice index.
3275 size_t IndexEnd = OldName.find_first_not_of(Chars: "0123456789");
3276 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
3277 // Strip the index and look for the offset.
3278 OldName = OldName.substr(Start: IndexEnd + 1);
3279 size_t OffsetEnd = OldName.find_first_not_of(Chars: "0123456789");
3280 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
3281 // Strip the offset.
3282 OldName = OldName.substr(Start: OffsetEnd + 1);
3283 }
3284 }
3285 // Strip any SROA suffixes as well.
3286 OldName = OldName.substr(Start: 0, N: OldName.find(Str: ".sroa_"));
3287
3288 return getAdjustedPtr(IRB, DL, Ptr: &NewAI,
3289 Offset: APInt(DL.getIndexTypeSizeInBits(Ty: PointerTy), Offset),
3290 PointerTy, NamePrefix: Twine(OldName) + ".");
3291 }
3292
3293 /// Compute suitable alignment to access this slice of the *new*
3294 /// alloca.
3295 ///
3296 /// You can optionally pass a type to this routine and if that type's ABI
3297 /// alignment is itself suitable, this will return zero.
3298 Align getSliceAlign() {
3299 return commonAlignment(A: NewAI.getAlign(),
3300 Offset: NewBeginOffset - NewAllocaBeginOffset);
3301 }
3302
3303 unsigned getIndex(uint64_t Offset) {
3304 assert(VecTy && "Can only call getIndex when rewriting a vector");
3305 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3306 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3307 uint32_t Index = RelOffset / ElementSize;
3308 assert(Index * ElementSize == RelOffset);
3309 return Index;
3310 }
3311
3312 void deleteIfTriviallyDead(Value *V) {
3313 Instruction *I = cast<Instruction>(Val: V);
3314 if (isInstructionTriviallyDead(I))
3315 Pass.DeadInsts.push_back(Elt: I);
3316 }
3317
3318 Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3319 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3320 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3321 assert(EndIndex > BeginIndex && "Empty vector!");
3322
3323 LoadInst *Load =
3324 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3325
3326 Load->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3327 LLVMContext::MD_access_group});
3328 return extractVector(IRB, V: Load, BeginIndex, EndIndex, Name: "vec");
3329 }
3330
3331 Value *rewriteIntegerLoad(LoadInst &LI) {
3332 assert(IntTy && "We cannot insert an integer to the alloca");
3333 assert(!LI.isVolatile());
3334 Value *V =
3335 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3336 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: IntTy);
3337 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3338 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3339 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
3340 IntegerType *ExtractTy = Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8);
3341 V = extractInteger(DL, IRB, V, Ty: ExtractTy, Offset, Name: "extract");
3342 }
3343 // It is possible that the extracted type is not the load type. This
3344 // happens if there is a load past the end of the alloca, and as
3345 // a consequence the slice is narrower but still a candidate for integer
3346 // lowering. To handle this case, we just zero extend the extracted
3347 // integer.
3348 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
3349 "Can only handle an extract for an overly wide load");
3350 if (cast<IntegerType>(Val: LI.getType())->getBitWidth() > SliceSize * 8)
3351 V = IRB.CreateZExt(V, DestTy: LI.getType());
3352 return V;
3353 }
3354
3355 bool visitLoadInst(LoadInst &LI) {
3356 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3357 Value *OldOp = LI.getOperand(i_nocapture: 0);
3358 assert(OldOp == OldPtr);
3359
3360 AAMDNodes AATags = LI.getAAMetadata();
3361
3362 unsigned AS = LI.getPointerAddressSpace();
3363
3364 Type *TargetTy = IsSplit ? Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8)
3365 : LI.getType();
3366 bool IsPtrAdjusted = false;
3367 Value *V;
3368 if (VecTy) {
3369 V = rewriteVectorizedLoadInst(LI);
3370 } else if (IntTy && LI.getType()->isIntegerTy()) {
3371 V = rewriteIntegerLoad(LI);
3372 } else if (NewBeginOffset == NewAllocaBeginOffset &&
3373 NewEndOffset == NewAllocaEndOffset &&
3374 (canConvertValue(DL, OldTy: NewAllocaTy, NewTy: TargetTy) ||
3375 (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3376 DL.getTypeStoreSize(Ty: TargetTy).getFixedValue() > SliceSize &&
3377 !LI.isVolatile()))) {
3378 Value *NewPtr =
3379 getPtrToNewAI(AddrSpace: LI.getPointerAddressSpace(), IsVolatile: LI.isVolatile());
3380 LoadInst *NewLI = IRB.CreateAlignedLoad(
3381 Ty: NewAllocaTy, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3382 if (LI.isVolatile())
3383 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3384 if (NewLI->isAtomic())
3385 NewLI->setAlignment(LI.getAlign());
3386
3387 // Copy any metadata that is valid for the new load. This may require
3388 // conversion to a different kind of metadata, e.g. !nonnull might change
3389 // to !range or vice versa.
3390 copyMetadataForLoad(Dest&: *NewLI, Source: LI);
3391
3392 // Do this after copyMetadataForLoad() to preserve the TBAA shift.
3393 if (AATags)
3394 NewLI->setAAMetadata(AATags.adjustForAccess(
3395 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3396
3397 // Try to preserve nonnull metadata
3398 V = NewLI;
3399
3400 // If this is an integer load past the end of the slice (which means the
3401 // bytes outside the slice are undef or this load is dead) just forcibly
3402 // fix the integer size with correct handling of endianness.
3403 if (auto *AITy = dyn_cast<IntegerType>(Val: NewAllocaTy))
3404 if (auto *TITy = dyn_cast<IntegerType>(Val: TargetTy))
3405 if (AITy->getBitWidth() < TITy->getBitWidth()) {
3406 V = IRB.CreateZExt(V, DestTy: TITy, Name: "load.ext");
3407 if (DL.isBigEndian())
3408 V = IRB.CreateShl(LHS: V, RHS: TITy->getBitWidth() - AITy->getBitWidth(),
3409 Name: "endian_shift");
3410 }
3411 } else {
3412 Type *LTy = IRB.getPtrTy(AddrSpace: AS);
3413 LoadInst *NewLI =
3414 IRB.CreateAlignedLoad(Ty: TargetTy, Ptr: getNewAllocaSlicePtr(IRB, PointerTy: LTy),
3415 Align: getSliceAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3416
3417 if (AATags)
3418 NewLI->setAAMetadata(AATags.adjustForAccess(
3419 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3420
3421 if (LI.isVolatile())
3422 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3423 NewLI->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3424 LLVMContext::MD_access_group});
3425
3426 V = NewLI;
3427 IsPtrAdjusted = true;
3428 }
3429 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: TargetTy);
3430
3431 if (IsSplit) {
3432 assert(!LI.isVolatile());
3433 assert(LI.getType()->isIntegerTy() &&
3434 "Only integer type loads and stores are split");
3435 assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3436 "Split load isn't smaller than original load");
3437 assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3438 "Non-byte-multiple bit width");
3439 // Move the insertion point just past the load so that we can refer to it.
3440 BasicBlock::iterator LIIt = std::next(x: LI.getIterator());
3441 // Ensure the insertion point comes before any debug-info immediately
3442 // after the load, so that variable values referring to the load are
3443 // dominated by it.
3444 LIIt.setHeadBit(true);
3445 IRB.SetInsertPoint(TheBB: LI.getParent(), IP: LIIt);
3446 // Create a placeholder value with the same type as LI to use as the
3447 // basis for the new value. This allows us to replace the uses of LI with
3448 // the computed value, and then replace the placeholder with LI, leaving
3449 // LI only used for this computation.
3450 Value *Placeholder =
3451 new LoadInst(LI.getType(), PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AS)), "",
3452 false, Align(1));
3453 V = insertInteger(DL, IRB, Old: Placeholder, V, Offset: NewBeginOffset - BeginOffset,
3454 Name: "insert");
3455 LI.replaceAllUsesWith(V);
3456 Placeholder->replaceAllUsesWith(V: &LI);
3457 Placeholder->deleteValue();
3458 } else {
3459 LI.replaceAllUsesWith(V);
3460 }
3461
3462 Pass.DeadInsts.push_back(Elt: &LI);
3463 deleteIfTriviallyDead(V: OldOp);
3464 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3465 return !LI.isVolatile() && !IsPtrAdjusted;
3466 }
3467
3468 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
3469 AAMDNodes AATags) {
3470 // Capture V for the purpose of debug-info accounting once it's converted
3471 // to a vector store.
3472 Value *OrigV = V;
3473 if (V->getType() != VecTy) {
3474 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3475 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3476 assert(EndIndex > BeginIndex && "Empty vector!");
3477 unsigned NumElements = EndIndex - BeginIndex;
3478 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3479 "Too many elements!");
3480 Type *SliceTy = (NumElements == 1)
3481 ? ElementTy
3482 : FixedVectorType::get(ElementType: ElementTy, NumElts: NumElements);
3483 if (V->getType() != SliceTy)
3484 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: SliceTy);
3485
3486 // Mix in the existing elements.
3487 Value *Old =
3488 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3489 V = insertVector(IRB, Old, V, BeginIndex, Name: "vec");
3490 }
3491 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3492 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3493 LLVMContext::MD_access_group});
3494 if (AATags)
3495 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3496 AccessTy: V->getType(), DL));
3497 Pass.DeadInsts.push_back(Elt: &SI);
3498
3499 // NOTE: Careful to use OrigV rather than V.
3500 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3501 Inst: Store, Dest: Store->getPointerOperand(), Value: OrigV, DL);
3502 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3503 return true;
3504 }
3505
3506 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3507 assert(IntTy && "We cannot extract an integer from the alloca");
3508 assert(!SI.isVolatile());
3509 if (DL.getTypeSizeInBits(Ty: V->getType()).getFixedValue() !=
3510 IntTy->getBitWidth()) {
3511 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3512 Name: "oldload");
3513 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3514 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3515 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3516 V = insertInteger(DL, IRB, Old, V: SI.getValueOperand(), Offset, Name: "insert");
3517 }
3518 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3519 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3520 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3521 LLVMContext::MD_access_group});
3522 if (AATags)
3523 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3524 AccessTy: V->getType(), DL));
3525
3526 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3527 Inst: Store, Dest: Store->getPointerOperand(),
3528 Value: Store->getValueOperand(), DL);
3529
3530 Pass.DeadInsts.push_back(Elt: &SI);
3531 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3532 return true;
3533 }
3534
3535 bool visitStoreInst(StoreInst &SI) {
3536 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3537 Value *OldOp = SI.getOperand(i_nocapture: 1);
3538 assert(OldOp == OldPtr);
3539
3540 AAMDNodes AATags = SI.getAAMetadata();
3541 Value *V = SI.getValueOperand();
3542
3543 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3544 // alloca that should be re-examined after promoting this alloca.
3545 if (V->getType()->isPointerTy())
3546 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V->stripInBoundsOffsets()))
3547 Pass.PostPromotionWorklist.insert(X: AI);
3548
3549 TypeSize StoreSize = DL.getTypeStoreSize(Ty: V->getType());
3550 if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3551 assert(!SI.isVolatile());
3552 assert(V->getType()->isIntegerTy() &&
3553 "Only integer type loads and stores are split");
3554 assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3555 "Non-byte-multiple bit width");
3556 IntegerType *NarrowTy = Type::getIntNTy(C&: SI.getContext(), N: SliceSize * 8);
3557 V = extractInteger(DL, IRB, V, Ty: NarrowTy, Offset: NewBeginOffset - BeginOffset,
3558 Name: "extract");
3559 }
3560
3561 if (VecTy)
3562 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3563 if (IntTy && V->getType()->isIntegerTy())
3564 return rewriteIntegerStore(V, SI, AATags);
3565
3566 StoreInst *NewSI;
3567 if (NewBeginOffset == NewAllocaBeginOffset &&
3568 NewEndOffset == NewAllocaEndOffset &&
3569 canConvertValue(DL, OldTy: V->getType(), NewTy: NewAllocaTy)) {
3570 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3571 Value *NewPtr =
3572 getPtrToNewAI(AddrSpace: SI.getPointerAddressSpace(), IsVolatile: SI.isVolatile());
3573
3574 NewSI =
3575 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: SI.isVolatile());
3576 } else {
3577 unsigned AS = SI.getPointerAddressSpace();
3578 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: IRB.getPtrTy(AddrSpace: AS));
3579 NewSI =
3580 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: getSliceAlign(), isVolatile: SI.isVolatile());
3581 }
3582 NewSI->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3583 LLVMContext::MD_access_group});
3584 if (AATags)
3585 NewSI->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3586 AccessTy: V->getType(), DL));
3587 if (SI.isVolatile())
3588 NewSI->setAtomic(Ordering: SI.getOrdering(), SSID: SI.getSyncScopeID());
3589 if (NewSI->isAtomic())
3590 NewSI->setAlignment(SI.getAlign());
3591
3592 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3593 Inst: NewSI, Dest: NewSI->getPointerOperand(),
3594 Value: NewSI->getValueOperand(), DL);
3595
3596 Pass.DeadInsts.push_back(Elt: &SI);
3597 deleteIfTriviallyDead(V: OldOp);
3598
3599 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3600 return NewSI->getPointerOperand() == &NewAI &&
3601 NewSI->getValueOperand()->getType() == NewAllocaTy &&
3602 !SI.isVolatile();
3603 }
3604
3605 /// Compute an integer value from splatting an i8 across the given
3606 /// number of bytes.
3607 ///
3608 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3609 /// call this routine.
3610 /// FIXME: Heed the advice above.
3611 ///
3612 /// \param V The i8 value to splat.
3613 /// \param Size The number of bytes in the output (assuming i8 is one byte)
3614 Value *getIntegerSplat(Value *V, unsigned Size) {
3615 assert(Size > 0 && "Expected a positive number of bytes.");
3616 IntegerType *VTy = cast<IntegerType>(Val: V->getType());
3617 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
3618 if (Size == 1)
3619 return V;
3620
3621 Type *SplatIntTy = Type::getIntNTy(C&: VTy->getContext(), N: Size * 8);
3622 V = IRB.CreateMul(
3623 LHS: IRB.CreateZExt(V, DestTy: SplatIntTy, Name: "zext"),
3624 RHS: IRB.CreateUDiv(LHS: Constant::getAllOnesValue(Ty: SplatIntTy),
3625 RHS: IRB.CreateZExt(V: Constant::getAllOnesValue(Ty: V->getType()),
3626 DestTy: SplatIntTy)),
3627 Name: "isplat");
3628 return V;
3629 }
3630
3631 /// Compute a vector splat for a given element value.
3632 Value *getVectorSplat(Value *V, unsigned NumElements) {
3633 V = IRB.CreateVectorSplat(NumElts: NumElements, V, Name: "vsplat");
3634 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3635 return V;
3636 }
3637
3638 bool visitMemSetInst(MemSetInst &II) {
3639 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3640 assert(II.getRawDest() == OldPtr);
3641
3642 AAMDNodes AATags = II.getAAMetadata();
3643
3644 // If the memset has a variable size, it cannot be split, just adjust the
3645 // pointer to the new alloca.
3646 if (!isa<ConstantInt>(Val: II.getLength())) {
3647 assert(!IsSplit);
3648 assert(NewBeginOffset == BeginOffset);
3649 II.setDest(getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()));
3650 II.setDestAlignment(getSliceAlign());
3651 // In theory we should call migrateDebugInfo here. However, we do not
3652 // emit dbg.assign intrinsics for mem intrinsics storing through non-
3653 // constant geps, or storing a variable number of bytes.
3654 assert(at::getDVRAssignmentMarkers(&II).empty() &&
3655 "AT: Unexpected link to non-const GEP");
3656 deleteIfTriviallyDead(V: OldPtr);
3657 return false;
3658 }
3659
3660 // Record this instruction for deletion.
3661 Pass.DeadInsts.push_back(Elt: &II);
3662
3663 Type *ScalarTy = NewAllocaTy->getScalarType();
3664
3665 const bool CanContinue = [&]() {
3666 if (VecTy || IntTy)
3667 return true;
3668 if (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset)
3669 return false;
3670 // Length must be in range for FixedVectorType.
3671 auto *C = cast<ConstantInt>(Val: II.getLength());
3672 const uint64_t Len = C->getLimitedValue();
3673 if (Len > std::numeric_limits<unsigned>::max())
3674 return false;
3675 auto *Int8Ty = IntegerType::getInt8Ty(C&: NewAI.getContext());
3676 auto *SrcTy = FixedVectorType::get(ElementType: Int8Ty, NumElts: Len);
3677 return canConvertValue(DL, OldTy: SrcTy, NewTy: NewAllocaTy) &&
3678 DL.isLegalInteger(Width: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue());
3679 }();
3680
3681 // If this doesn't map cleanly onto the alloca type, and that type isn't
3682 // a single value type, just emit a memset.
3683 if (!CanContinue) {
3684 Type *SizeTy = II.getLength()->getType();
3685 unsigned Sz = NewEndOffset - NewBeginOffset;
3686 Constant *Size = ConstantInt::get(Ty: SizeTy, V: Sz);
3687 MemIntrinsic *New = cast<MemIntrinsic>(Val: IRB.CreateMemSet(
3688 Ptr: getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()), Val: II.getValue(), Size,
3689 Align: MaybeAlign(getSliceAlign()), isVolatile: II.isVolatile()));
3690 if (AATags)
3691 New->setAAMetadata(
3692 AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset, AccessSize: Sz));
3693
3694 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3695 Inst: New, Dest: New->getRawDest(), Value: nullptr, DL);
3696
3697 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3698 return false;
3699 }
3700
3701 // If we can represent this as a simple value, we have to build the actual
3702 // value to store, which requires expanding the byte present in memset to
3703 // a sensible representation for the alloca type. This is essentially
3704 // splatting the byte to a sufficiently wide integer, splatting it across
3705 // any desired vector width, and bitcasting to the final type.
3706 Value *V;
3707
3708 if (VecTy) {
3709 // If this is a memset of a vectorized alloca, insert it.
3710 assert(ElementTy == ScalarTy);
3711
3712 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3713 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3714 assert(EndIndex > BeginIndex && "Empty vector!");
3715 unsigned NumElements = EndIndex - BeginIndex;
3716 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3717 "Too many elements!");
3718
3719 Value *Splat = getIntegerSplat(
3720 V: II.getValue(), Size: DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8);
3721 Splat = IRB.CreateBitPreservingCastChain(DL, V: Splat, NewTy: ElementTy);
3722 if (NumElements > 1)
3723 Splat = getVectorSplat(V: Splat, NumElements);
3724
3725 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3726 Name: "oldload");
3727 V = insertVector(IRB, Old, V: Splat, BeginIndex, Name: "vec");
3728 } else if (IntTy) {
3729 // If this is a memset on an alloca where we can widen stores, insert the
3730 // set integer.
3731 assert(!II.isVolatile());
3732
3733 uint64_t Size = NewEndOffset - NewBeginOffset;
3734 V = getIntegerSplat(V: II.getValue(), Size);
3735
3736 if (IntTy && (NewBeginOffset != NewAllocaBeginOffset ||
3737 NewEndOffset != NewAllocaEndOffset)) {
3738 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI,
3739 Align: NewAI.getAlign(), Name: "oldload");
3740 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3741 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3742 V = insertInteger(DL, IRB, Old, V, Offset, Name: "insert");
3743 } else {
3744 assert(V->getType() == IntTy &&
3745 "Wrong type for an alloca wide integer!");
3746 }
3747 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3748 } else {
3749 // Established these invariants above.
3750 assert(NewBeginOffset == NewAllocaBeginOffset);
3751 assert(NewEndOffset == NewAllocaEndOffset);
3752
3753 V = getIntegerSplat(V: II.getValue(),
3754 Size: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue() / 8);
3755 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(Val: NewAllocaTy))
3756 V = getVectorSplat(
3757 V, NumElements: cast<FixedVectorType>(Val: AllocaVecTy)->getNumElements());
3758
3759 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3760 }
3761
3762 Value *NewPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3763 StoreInst *New =
3764 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: II.isVolatile());
3765 New->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3766 LLVMContext::MD_access_group});
3767 if (AATags)
3768 New->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3769 AccessTy: V->getType(), DL));
3770
3771 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3772 Inst: New, Dest: New->getPointerOperand(), Value: V, DL);
3773
3774 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3775 return !II.isVolatile();
3776 }
3777
3778 bool visitMemTransferInst(MemTransferInst &II) {
3779 // Rewriting of memory transfer instructions can be a bit tricky. We break
3780 // them into two categories: split intrinsics and unsplit intrinsics.
3781
3782 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3783
3784 AAMDNodes AATags = II.getAAMetadata();
3785
3786 bool IsDest = &II.getRawDestUse() == OldUse;
3787 assert((IsDest && II.getRawDest() == OldPtr) ||
3788 (!IsDest && II.getRawSource() == OldPtr));
3789
3790 Align SliceAlign = getSliceAlign();
3791 // For unsplit intrinsics, we simply modify the source and destination
3792 // pointers in place. This isn't just an optimization, it is a matter of
3793 // correctness. With unsplit intrinsics we may be dealing with transfers
3794 // within a single alloca before SROA ran, or with transfers that have
3795 // a variable length. We may also be dealing with memmove instead of
3796 // memcpy, and so simply updating the pointers is the necessary for us to
3797 // update both source and dest of a single call.
3798 if (!IsSplittable) {
3799 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3800 if (IsDest) {
3801 // Update the address component of linked dbg.assigns.
3802 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(Inst: &II)) {
3803 if (llvm::is_contained(Range: DbgAssign->location_ops(), Element: II.getDest()) ||
3804 DbgAssign->getAddress() == II.getDest())
3805 DbgAssign->replaceVariableLocationOp(OldValue: II.getDest(), NewValue: AdjustedPtr);
3806 }
3807 II.setDest(AdjustedPtr);
3808 II.setDestAlignment(SliceAlign);
3809 } else {
3810 II.setSource(AdjustedPtr);
3811 II.setSourceAlignment(SliceAlign);
3812 }
3813
3814 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3815 deleteIfTriviallyDead(V: OldPtr);
3816 return false;
3817 }
3818 // For split transfer intrinsics we have an incredibly useful assurance:
3819 // the source and destination do not reside within the same alloca, and at
3820 // least one of them does not escape. This means that we can replace
3821 // memmove with memcpy, and we don't need to worry about all manner of
3822 // downsides to splitting and transforming the operations.
3823
3824 // If this doesn't map cleanly onto the alloca type, and that type isn't
3825 // a single value type, just emit a memcpy.
3826 bool EmitMemCpy =
3827 !VecTy && !IntTy &&
3828 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
3829 SliceSize != DL.getTypeStoreSize(Ty: NewAllocaTy).getFixedValue() ||
3830 !DL.typeSizeEqualsStoreSize(Ty: NewAllocaTy) ||
3831 !NewAllocaTy->isSingleValueType());
3832
3833 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
3834 // size hasn't been shrunk based on analysis of the viable range, this is
3835 // a no-op.
3836 if (EmitMemCpy && &OldAI == &NewAI) {
3837 // Ensure the start lines up.
3838 assert(NewBeginOffset == BeginOffset);
3839
3840 // Rewrite the size as needed.
3841 if (NewEndOffset != EndOffset)
3842 II.setLength(NewEndOffset - NewBeginOffset);
3843 return false;
3844 }
3845 // Record this instruction for deletion.
3846 Pass.DeadInsts.push_back(Elt: &II);
3847
3848 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3849 // alloca that should be re-examined after rewriting this instruction.
3850 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
3851 if (AllocaInst *AI =
3852 dyn_cast<AllocaInst>(Val: OtherPtr->stripInBoundsOffsets())) {
3853 assert(AI != &OldAI && AI != &NewAI &&
3854 "Splittable transfers cannot reach the same alloca on both ends.");
3855 Pass.Worklist.insert(X: AI);
3856 }
3857
3858 Type *OtherPtrTy = OtherPtr->getType();
3859 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
3860
3861 // Compute the relative offset for the other pointer within the transfer.
3862 unsigned OffsetWidth = DL.getIndexSizeInBits(AS: OtherAS);
3863 APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
3864 Align OtherAlign =
3865 (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
3866 OtherAlign =
3867 commonAlignment(A: OtherAlign, Offset: OtherOffset.zextOrTrunc(width: 64).getZExtValue());
3868
3869 if (EmitMemCpy) {
3870 // Compute the other pointer, folding as much as possible to produce
3871 // a single, simple GEP in most cases.
3872 OtherPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
3873 NamePrefix: OtherPtr->getName() + ".");
3874
3875 Value *OurPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3876 Type *SizeTy = II.getLength()->getType();
3877 Constant *Size = ConstantInt::get(Ty: SizeTy, V: NewEndOffset - NewBeginOffset);
3878
3879 Value *DestPtr, *SrcPtr;
3880 MaybeAlign DestAlign, SrcAlign;
3881 // Note: IsDest is true iff we're copying into the new alloca slice
3882 if (IsDest) {
3883 DestPtr = OurPtr;
3884 DestAlign = SliceAlign;
3885 SrcPtr = OtherPtr;
3886 SrcAlign = OtherAlign;
3887 } else {
3888 DestPtr = OtherPtr;
3889 DestAlign = OtherAlign;
3890 SrcPtr = OurPtr;
3891 SrcAlign = SliceAlign;
3892 }
3893 CallInst *New = IRB.CreateMemCpy(Dst: DestPtr, DstAlign: DestAlign, Src: SrcPtr, SrcAlign,
3894 Size, isVolatile: II.isVolatile());
3895 if (AATags)
3896 New->setAAMetadata(AATags.shift(Offset: NewBeginOffset - BeginOffset));
3897
3898 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DestPtr->getType()), 0);
3899 if (IsDest) {
3900 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8,
3901 OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
3902 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
3903 Val: DestPtr->stripAndAccumulateConstantOffsets(
3904 DL, Offset, /*AllowNonInbounds*/ true))) {
3905 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
3906 SliceSizeInBits: SliceSize * 8, OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
3907 }
3908 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3909 return false;
3910 }
3911
3912 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
3913 NewEndOffset == NewAllocaEndOffset;
3914 uint64_t Size = NewEndOffset - NewBeginOffset;
3915 unsigned BeginIndex = VecTy ? getIndex(Offset: NewBeginOffset) : 0;
3916 unsigned EndIndex = VecTy ? getIndex(Offset: NewEndOffset) : 0;
3917 unsigned NumElements = EndIndex - BeginIndex;
3918 IntegerType *SubIntTy =
3919 IntTy ? Type::getIntNTy(C&: IntTy->getContext(), N: Size * 8) : nullptr;
3920
3921 // Reset the other pointer type to match the register type we're going to
3922 // use, but using the address space of the original other pointer.
3923 Type *OtherTy;
3924 if (VecTy && !IsWholeAlloca) {
3925 if (NumElements == 1)
3926 OtherTy = VecTy->getElementType();
3927 else
3928 OtherTy = FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: NumElements);
3929 } else if (IntTy && !IsWholeAlloca) {
3930 OtherTy = SubIntTy;
3931 } else {
3932 OtherTy = NewAllocaTy;
3933 }
3934
3935 Value *AdjPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
3936 NamePrefix: OtherPtr->getName() + ".");
3937 MaybeAlign SrcAlign = OtherAlign;
3938 MaybeAlign DstAlign = SliceAlign;
3939 if (!IsDest)
3940 std::swap(a&: SrcAlign, b&: DstAlign);
3941
3942 Value *SrcPtr;
3943 Value *DstPtr;
3944
3945 if (IsDest) {
3946 DstPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3947 SrcPtr = AdjPtr;
3948 } else {
3949 DstPtr = AdjPtr;
3950 SrcPtr = getPtrToNewAI(AddrSpace: II.getSourceAddressSpace(), IsVolatile: II.isVolatile());
3951 }
3952
3953 Value *Src;
3954 if (VecTy && !IsWholeAlloca && !IsDest) {
3955 Src =
3956 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3957 Src = extractVector(IRB, V: Src, BeginIndex, EndIndex, Name: "vec");
3958 } else if (IntTy && !IsWholeAlloca && !IsDest) {
3959 Src =
3960 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3961 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: IntTy);
3962 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3963 Src = extractInteger(DL, IRB, V: Src, Ty: SubIntTy, Offset, Name: "extract");
3964 } else {
3965 LoadInst *Load = IRB.CreateAlignedLoad(Ty: OtherTy, Ptr: SrcPtr, Align: SrcAlign,
3966 isVolatile: II.isVolatile(), Name: "copyload");
3967 Load->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3968 LLVMContext::MD_access_group});
3969 if (AATags)
3970 Load->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3971 AccessTy: Load->getType(), DL));
3972 Src = Load;
3973 }
3974
3975 if (VecTy && !IsWholeAlloca && IsDest) {
3976 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3977 Name: "oldload");
3978 Src = insertVector(IRB, Old, V: Src, BeginIndex, Name: "vec");
3979 } else if (IntTy && !IsWholeAlloca && IsDest) {
3980 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3981 Name: "oldload");
3982 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3983 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3984 Src = insertInteger(DL, IRB, Old, V: Src, Offset, Name: "insert");
3985 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: NewAllocaTy);
3986 }
3987
3988 StoreInst *Store = cast<StoreInst>(
3989 Val: IRB.CreateAlignedStore(Val: Src, Ptr: DstPtr, Align: DstAlign, isVolatile: II.isVolatile()));
3990 Store->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3991 LLVMContext::MD_access_group});
3992 if (AATags)
3993 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3994 AccessTy: Src->getType(), DL));
3995
3996 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DstPtr->getType()), 0);
3997 if (IsDest) {
3998
3999 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
4000 Inst: Store, Dest: DstPtr, Value: Src, DL);
4001 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4002 Val: DstPtr->stripAndAccumulateConstantOffsets(
4003 DL, Offset, /*AllowNonInbounds*/ true))) {
4004 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8, SliceSizeInBits: SliceSize * 8,
4005 OldInst: &II, Inst: Store, Dest: DstPtr, Value: Src, DL);
4006 }
4007
4008 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4009 return !II.isVolatile();
4010 }
4011
4012 bool visitIntrinsicInst(IntrinsicInst &II) {
4013 assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
4014 "Unexpected intrinsic!");
4015 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4016
4017 // Record this instruction for deletion.
4018 Pass.DeadInsts.push_back(Elt: &II);
4019
4020 if (II.isDroppable()) {
4021 assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4022 // TODO For now we forget assumed information, this can be improved.
4023 OldPtr->dropDroppableUsesIn(Usr&: II);
4024 return true;
4025 }
4026
4027 assert(II.getArgOperand(0) == OldPtr);
4028 Type *PointerTy = IRB.getPtrTy(AddrSpace: OldPtr->getType()->getPointerAddressSpace());
4029 Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4030 Value *New;
4031 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4032 New = IRB.CreateLifetimeStart(Ptr);
4033 else
4034 New = IRB.CreateLifetimeEnd(Ptr);
4035
4036 (void)New;
4037 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4038
4039 return true;
4040 }
4041
4042 void fixLoadStoreAlign(Instruction &Root) {
4043 // This algorithm implements the same visitor loop as
4044 // hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4045 // or store found.
4046 SmallPtrSet<Instruction *, 4> Visited;
4047 SmallVector<Instruction *, 4> Uses;
4048 Visited.insert(Ptr: &Root);
4049 Uses.push_back(Elt: &Root);
4050 do {
4051 Instruction *I = Uses.pop_back_val();
4052
4053 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
4054 LI->setAlignment(std::min(a: LI->getAlign(), b: getSliceAlign()));
4055 continue;
4056 }
4057 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
4058 SI->setAlignment(std::min(a: SI->getAlign(), b: getSliceAlign()));
4059 continue;
4060 }
4061
4062 assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||
4063 isa<PHINode>(I) || isa<SelectInst>(I) ||
4064 isa<GetElementPtrInst>(I));
4065 for (User *U : I->users())
4066 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
4067 Uses.push_back(Elt: cast<Instruction>(Val: U));
4068 } while (!Uses.empty());
4069 }
4070
4071 bool visitPHINode(PHINode &PN) {
4072 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4073 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4074 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4075
4076 // We would like to compute a new pointer in only one place, but have it be
4077 // as local as possible to the PHI. To do that, we re-use the location of
4078 // the old pointer, which necessarily must be in the right position to
4079 // dominate the PHI.
4080 IRBuilderBase::InsertPointGuard Guard(IRB);
4081 if (isa<PHINode>(Val: OldPtr))
4082 IRB.SetInsertPoint(TheBB: OldPtr->getParent(),
4083 IP: OldPtr->getParent()->getFirstInsertionPt());
4084 else
4085 IRB.SetInsertPoint(OldPtr);
4086 IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4087
4088 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4089 // Replace the operands which were using the old pointer.
4090 std::replace(first: PN.op_begin(), last: PN.op_end(), old_value: cast<Value>(Val: OldPtr), new_value: NewPtr);
4091
4092 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4093 deleteIfTriviallyDead(V: OldPtr);
4094
4095 // Fix the alignment of any loads or stores using this PHI node.
4096 fixLoadStoreAlign(Root&: PN);
4097
4098 // PHIs can't be promoted on their own, but often can be speculated. We
4099 // check the speculation outside of the rewriter so that we see the
4100 // fully-rewritten alloca.
4101 PHIUsers.insert(X: &PN);
4102 return true;
4103 }
4104
4105 bool visitSelectInst(SelectInst &SI) {
4106 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4107 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
4108 "Pointer isn't an operand!");
4109 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4110 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4111
4112 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4113 // Replace the operands which were using the old pointer.
4114 if (SI.getOperand(i_nocapture: 1) == OldPtr)
4115 SI.setOperand(i_nocapture: 1, Val_nocapture: NewPtr);
4116 if (SI.getOperand(i_nocapture: 2) == OldPtr)
4117 SI.setOperand(i_nocapture: 2, Val_nocapture: NewPtr);
4118
4119 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4120 deleteIfTriviallyDead(V: OldPtr);
4121
4122 // Fix the alignment of any loads or stores using this select.
4123 fixLoadStoreAlign(Root&: SI);
4124
4125 // Selects can't be promoted on their own, but often can be speculated. We
4126 // check the speculation outside of the rewriter so that we see the
4127 // fully-rewritten alloca.
4128 SelectUsers.insert(X: &SI);
4129 return true;
4130 }
4131};
4132
4133/// Visitor to rewrite aggregate loads and stores as scalar.
4134///
4135/// This pass aggressively rewrites all aggregate loads and stores on
4136/// a particular pointer (or any pointer derived from it which we can identify)
4137/// with scalar loads and stores.
4138class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4139 // Befriend the base class so it can delegate to private visit methods.
4140 friend class InstVisitor<AggLoadStoreRewriter, bool>;
4141
4142 /// Queue of pointer uses to analyze and potentially rewrite.
4143 SmallVector<Use *, 8> Queue;
4144
4145 /// Set to prevent us from cycling with phi nodes and loops.
4146 SmallPtrSet<User *, 8> Visited;
4147
4148 /// The current pointer use being rewritten. This is used to dig up the used
4149 /// value (as opposed to the user).
4150 Use *U = nullptr;
4151
4152 /// Used to calculate offsets, and hence alignment, of subobjects.
4153 const DataLayout &DL;
4154
4155 IRBuilderTy &IRB;
4156
4157public:
4158 AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4159 : DL(DL), IRB(IRB) {}
4160
4161 /// Rewrite loads and stores through a pointer and all pointers derived from
4162 /// it.
4163 bool rewrite(Instruction &I) {
4164 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4165 enqueueUsers(I);
4166 bool Changed = false;
4167 while (!Queue.empty()) {
4168 U = Queue.pop_back_val();
4169 Changed |= visit(I: cast<Instruction>(Val: U->getUser()));
4170 }
4171 return Changed;
4172 }
4173
4174private:
4175 /// Enqueue all the users of the given instruction for further processing.
4176 /// This uses a set to de-duplicate users.
4177 void enqueueUsers(Instruction &I) {
4178 for (Use &U : I.uses())
4179 if (Visited.insert(Ptr: U.getUser()).second)
4180 Queue.push_back(Elt: &U);
4181 }
4182
4183 // Conservative default is to not rewrite anything.
4184 bool visitInstruction(Instruction &I) { return false; }
4185
4186 /// Generic recursive split emission class.
4187 template <typename Derived> class OpSplitter {
4188 protected:
4189 /// The builder used to form new instructions.
4190 IRBuilderTy &IRB;
4191
4192 /// The indices which to be used with insert- or extractvalue to select the
4193 /// appropriate value within the aggregate.
4194 SmallVector<unsigned, 4> Indices;
4195
4196 /// The indices to a GEP instruction which will move Ptr to the correct slot
4197 /// within the aggregate.
4198 SmallVector<Value *, 4> GEPIndices;
4199
4200 /// The base pointer of the original op, used as a base for GEPing the
4201 /// split operations.
4202 Value *Ptr;
4203
4204 /// The base pointee type being GEPed into.
4205 Type *BaseTy;
4206
4207 /// Known alignment of the base pointer.
4208 Align BaseAlign;
4209
4210 /// To calculate offset of each component so we can correctly deduce
4211 /// alignments.
4212 const DataLayout &DL;
4213
4214 /// Initialize the splitter with an insertion point, Ptr and start with a
4215 /// single zero GEP index.
4216 OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4217 Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4218 : IRB(IRB), GEPIndices(1, IRB.getInt32(C: 0)), Ptr(Ptr), BaseTy(BaseTy),
4219 BaseAlign(BaseAlign), DL(DL) {
4220 IRB.SetInsertPoint(InsertionPoint);
4221 }
4222
4223 public:
4224 /// Generic recursive split emission routine.
4225 ///
4226 /// This method recursively splits an aggregate op (load or store) into
4227 /// scalar or vector ops. It splits recursively until it hits a single value
4228 /// and emits that single value operation via the template argument.
4229 ///
4230 /// The logic of this routine relies on GEPs and insertvalue and
4231 /// extractvalue all operating with the same fundamental index list, merely
4232 /// formatted differently (GEPs need actual values).
4233 ///
4234 /// \param Ty The type being split recursively into smaller ops.
4235 /// \param Agg The aggregate value being built up or stored, depending on
4236 /// whether this is splitting a load or a store respectively.
4237 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
4238 if (Ty->isSingleValueType()) {
4239 unsigned Offset = DL.getIndexedOffsetInType(ElemTy: BaseTy, Indices: GEPIndices);
4240 return static_cast<Derived *>(this)->emitFunc(
4241 Ty, Agg, commonAlignment(A: BaseAlign, Offset), Name);
4242 }
4243
4244 if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
4245 unsigned OldSize = Indices.size();
4246 (void)OldSize;
4247 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
4248 ++Idx) {
4249 assert(Indices.size() == OldSize && "Did not return to the old size");
4250 Indices.push_back(Elt: Idx);
4251 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4252 emitSplitOps(Ty: ATy->getElementType(), Agg, Name: Name + "." + Twine(Idx));
4253 GEPIndices.pop_back();
4254 Indices.pop_back();
4255 }
4256 return;
4257 }
4258
4259 if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4260 unsigned OldSize = Indices.size();
4261 (void)OldSize;
4262 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
4263 ++Idx) {
4264 assert(Indices.size() == OldSize && "Did not return to the old size");
4265 Indices.push_back(Elt: Idx);
4266 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4267 emitSplitOps(Ty: STy->getElementType(N: Idx), Agg, Name: Name + "." + Twine(Idx));
4268 GEPIndices.pop_back();
4269 Indices.pop_back();
4270 }
4271 return;
4272 }
4273
4274 llvm_unreachable("Only arrays and structs are aggregate loadable types");
4275 }
4276 };
4277
4278 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4279 AAMDNodes AATags;
4280 // A vector to hold the split components that we want to emit
4281 // separate fake uses for.
4282 SmallVector<Value *, 4> Components;
4283 // A vector to hold all the fake uses of the struct that we are splitting.
4284 // Usually there should only be one, but we are handling the general case.
4285 SmallVector<Instruction *, 1> FakeUses;
4286
4287 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4288 AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4289 IRBuilderTy &IRB)
4290 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4291 IRB),
4292 AATags(AATags) {}
4293
4294 /// Emit a leaf load of a single value. This is called at the leaves of the
4295 /// recursive emission to actually load values.
4296 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4297 assert(Ty->isSingleValueType());
4298 // Load the single value and insert it using the indices.
4299 Value *GEP =
4300 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4301 LoadInst *Load =
4302 IRB.CreateAlignedLoad(Ty, Ptr: GEP, Align: Alignment, Name: Name + ".load");
4303
4304 APInt Offset(
4305 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4306 if (AATags &&
4307 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset))
4308 Load->setAAMetadata(
4309 AATags.adjustForAccess(Offset: Offset.getZExtValue(), AccessTy: Load->getType(), DL));
4310 // Record the load so we can generate a fake use for this aggregate
4311 // component.
4312 Components.push_back(Elt: Load);
4313
4314 Agg = IRB.CreateInsertValue(Agg, Val: Load, Idxs: Indices, Name: Name + ".insert");
4315 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4316 }
4317
4318 // Stash the fake uses that use the value generated by this instruction.
4319 void recordFakeUses(LoadInst &LI) {
4320 for (Use &U : LI.uses())
4321 if (auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser()))
4322 if (II->getIntrinsicID() == Intrinsic::fake_use)
4323 FakeUses.push_back(Elt: II);
4324 }
4325
4326 // Replace all fake uses of the aggregate with a series of fake uses, one
4327 // for each split component.
4328 void emitFakeUses() {
4329 for (Instruction *I : FakeUses) {
4330 IRB.SetInsertPoint(I);
4331 for (auto *V : Components)
4332 IRB.CreateIntrinsic(ID: Intrinsic::fake_use, Args: {V});
4333 I->eraseFromParent();
4334 }
4335 }
4336 };
4337
4338 bool visitLoadInst(LoadInst &LI) {
4339 assert(LI.getPointerOperand() == *U);
4340 if (!LI.isSimple() || LI.getType()->isSingleValueType())
4341 return false;
4342
4343 // We have an aggregate being loaded, split it apart.
4344 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4345 LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4346 getAdjustedAlignment(I: &LI, Offset: 0), DL, IRB);
4347 Splitter.recordFakeUses(LI);
4348 Value *V = PoisonValue::get(T: LI.getType());
4349 Splitter.emitSplitOps(Ty: LI.getType(), Agg&: V, Name: LI.getName() + ".fca");
4350 Splitter.emitFakeUses();
4351 Visited.erase(Ptr: &LI);
4352 LI.replaceAllUsesWith(V);
4353 LI.eraseFromParent();
4354 return true;
4355 }
4356
4357 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4358 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4359 AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4360 const DataLayout &DL, IRBuilderTy &IRB)
4361 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4362 DL, IRB),
4363 AATags(AATags), AggStore(AggStore) {}
4364 AAMDNodes AATags;
4365 StoreInst *AggStore;
4366 /// Emit a leaf store of a single value. This is called at the leaves of the
4367 /// recursive emission to actually produce stores.
4368 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4369 assert(Ty->isSingleValueType());
4370 // Extract the single value and store it using the indices.
4371 //
4372 // The gep and extractvalue values are factored out of the CreateStore
4373 // call to make the output independent of the argument evaluation order.
4374 Value *ExtractValue =
4375 IRB.CreateExtractValue(Agg, Idxs: Indices, Name: Name + ".extract");
4376 Value *InBoundsGEP =
4377 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4378 StoreInst *Store =
4379 IRB.CreateAlignedStore(Val: ExtractValue, Ptr: InBoundsGEP, Align: Alignment);
4380
4381 APInt Offset(
4382 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4383 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset);
4384 if (AATags) {
4385 Store->setAAMetadata(AATags.adjustForAccess(
4386 Offset: Offset.getZExtValue(), AccessTy: ExtractValue->getType(), DL));
4387 }
4388
4389 // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4390 // If we cannot (because there's an intervening non-const or unbounded
4391 // gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4392 // this instruction.
4393 Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets();
4394 if (auto *OldAI = dyn_cast<AllocaInst>(Val: Base)) {
4395 uint64_t SizeInBits =
4396 DL.getTypeSizeInBits(Ty: Store->getValueOperand()->getType());
4397 migrateDebugInfo(OldAlloca: OldAI, /*IsSplit*/ true, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
4398 SliceSizeInBits: SizeInBits, OldInst: AggStore, Inst: Store,
4399 Dest: Store->getPointerOperand(), Value: Store->getValueOperand(),
4400 DL);
4401 } else {
4402 assert(at::getDVRAssignmentMarkers(Store).empty() &&
4403 "AT: unexpected debug.assign linked to store through "
4404 "unbounded GEP");
4405 }
4406 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4407 }
4408 };
4409
4410 bool visitStoreInst(StoreInst &SI) {
4411 if (!SI.isSimple() || SI.getPointerOperand() != *U)
4412 return false;
4413 Value *V = SI.getValueOperand();
4414 if (V->getType()->isSingleValueType())
4415 return false;
4416
4417 // We have an aggregate being stored, split it apart.
4418 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4419 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4420 getAdjustedAlignment(I: &SI, Offset: 0), DL, IRB);
4421 Splitter.emitSplitOps(Ty: V->getType(), Agg&: V, Name: V->getName() + ".fca");
4422 Visited.erase(Ptr: &SI);
4423 // The stores replacing SI each have markers describing fragments of the
4424 // assignment so delete the assignment markers linked to SI.
4425 at::deleteAssignmentMarkers(Inst: &SI);
4426 SI.eraseFromParent();
4427 return true;
4428 }
4429
4430 bool visitBitCastInst(BitCastInst &BC) {
4431 enqueueUsers(I&: BC);
4432 return false;
4433 }
4434
4435 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4436 enqueueUsers(I&: ASC);
4437 return false;
4438 }
4439
4440 // Unfold gep (select cond, ptr1, ptr2), idx
4441 // => select cond, gep(ptr1, idx), gep(ptr2, idx)
4442 // and gep ptr, (select cond, idx1, idx2)
4443 // => select cond, gep(ptr, idx1), gep(ptr, idx2)
4444 // We also allow for i1 zext indices, which are equivalent to selects.
4445 bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4446 // Check whether the GEP has exactly one select operand and all indices
4447 // will become constant after the transform.
4448 Instruction *Sel = dyn_cast<SelectInst>(Val: GEPI.getPointerOperand());
4449 for (Value *Op : GEPI.indices()) {
4450 if (auto *SI = dyn_cast<SelectInst>(Val: Op)) {
4451 if (Sel)
4452 return false;
4453
4454 Sel = SI;
4455 if (!isa<ConstantInt>(Val: SI->getTrueValue()) ||
4456 !isa<ConstantInt>(Val: SI->getFalseValue()))
4457 return false;
4458 continue;
4459 }
4460 if (auto *ZI = dyn_cast<ZExtInst>(Val: Op)) {
4461 if (Sel)
4462 return false;
4463 Sel = ZI;
4464 if (!ZI->getSrcTy()->isIntegerTy(BitWidth: 1))
4465 return false;
4466 continue;
4467 }
4468
4469 if (!isa<ConstantInt>(Val: Op))
4470 return false;
4471 }
4472
4473 if (!Sel)
4474 return false;
4475
4476 LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4477 dbgs() << " original: " << *Sel << "\n";
4478 dbgs() << " " << GEPI << "\n";);
4479
4480 auto GetNewOps = [&](Value *SelOp) {
4481 SmallVector<Value *> NewOps;
4482 for (Value *Op : GEPI.operands())
4483 if (Op == Sel)
4484 NewOps.push_back(Elt: SelOp);
4485 else
4486 NewOps.push_back(Elt: Op);
4487 return NewOps;
4488 };
4489
4490 Value *Cond, *True, *False;
4491 Instruction *MDFrom = nullptr;
4492 if (auto *SI = dyn_cast<SelectInst>(Val: Sel)) {
4493 Cond = SI->getCondition();
4494 True = SI->getTrueValue();
4495 False = SI->getFalseValue();
4496 if (!ProfcheckDisableMetadataFixes)
4497 MDFrom = SI;
4498 } else {
4499 Cond = Sel->getOperand(i: 0);
4500 True = ConstantInt::get(Ty: Sel->getType(), V: 1);
4501 False = ConstantInt::get(Ty: Sel->getType(), V: 0);
4502 }
4503 SmallVector<Value *> TrueOps = GetNewOps(True);
4504 SmallVector<Value *> FalseOps = GetNewOps(False);
4505
4506 IRB.SetInsertPoint(&GEPI);
4507 GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4508
4509 Type *Ty = GEPI.getSourceElementType();
4510 Value *NTrue = IRB.CreateGEP(Ty, Ptr: TrueOps[0], IdxList: ArrayRef(TrueOps).drop_front(),
4511 Name: True->getName() + ".sroa.gep", NW);
4512
4513 Value *NFalse =
4514 IRB.CreateGEP(Ty, Ptr: FalseOps[0], IdxList: ArrayRef(FalseOps).drop_front(),
4515 Name: False->getName() + ".sroa.gep", NW);
4516
4517 Value *NSel = MDFrom
4518 ? IRB.CreateSelect(C: Cond, True: NTrue, False: NFalse,
4519 Name: Sel->getName() + ".sroa.sel", MDFrom)
4520 : IRB.CreateSelectWithUnknownProfile(
4521 C: Cond, True: NTrue, False: NFalse, DEBUG_TYPE,
4522 Name: Sel->getName() + ".sroa.sel");
4523 Visited.erase(Ptr: &GEPI);
4524 GEPI.replaceAllUsesWith(V: NSel);
4525 GEPI.eraseFromParent();
4526 Instruction *NSelI = cast<Instruction>(Val: NSel);
4527 Visited.insert(Ptr: NSelI);
4528 enqueueUsers(I&: *NSelI);
4529
4530 LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4531 dbgs() << " " << *NFalse << "\n";
4532 dbgs() << " " << *NSel << "\n";);
4533
4534 return true;
4535 }
4536
4537 // Unfold gep (phi ptr1, ptr2), idx
4538 // => phi ((gep ptr1, idx), (gep ptr2, idx))
4539 // and gep ptr, (phi idx1, idx2)
4540 // => phi ((gep ptr, idx1), (gep ptr, idx2))
4541 bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4542 // To prevent infinitely expanding recursive phis, bail if the GEP pointer
4543 // operand (looking through the phi if it is the phi we want to unfold) is
4544 // an instruction besides a static alloca.
4545 PHINode *Phi = dyn_cast<PHINode>(Val: GEPI.getPointerOperand());
4546 auto IsInvalidPointerOperand = [](Value *V) {
4547 if (!isa<Instruction>(Val: V))
4548 return false;
4549 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
4550 return !AI->isStaticAlloca();
4551 return true;
4552 };
4553 if (Phi) {
4554 if (any_of(Range: Phi->operands(), P: IsInvalidPointerOperand))
4555 return false;
4556 } else {
4557 if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4558 return false;
4559 }
4560 // Check whether the GEP has exactly one phi operand (including the pointer
4561 // operand) and all indices will become constant after the transform.
4562 for (Value *Op : GEPI.indices()) {
4563 if (auto *SI = dyn_cast<PHINode>(Val: Op)) {
4564 if (Phi)
4565 return false;
4566
4567 Phi = SI;
4568 if (!all_of(Range: Phi->incoming_values(),
4569 P: [](Value *V) { return isa<ConstantInt>(Val: V); }))
4570 return false;
4571 continue;
4572 }
4573
4574 if (!isa<ConstantInt>(Val: Op))
4575 return false;
4576 }
4577
4578 if (!Phi)
4579 return false;
4580
4581 LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4582 dbgs() << " original: " << *Phi << "\n";
4583 dbgs() << " " << GEPI << "\n";);
4584
4585 auto GetNewOps = [&](Value *PhiOp) {
4586 SmallVector<Value *> NewOps;
4587 for (Value *Op : GEPI.operands())
4588 if (Op == Phi)
4589 NewOps.push_back(Elt: PhiOp);
4590 else
4591 NewOps.push_back(Elt: Op);
4592 return NewOps;
4593 };
4594
4595 IRB.SetInsertPoint(Phi);
4596 PHINode *NewPhi = IRB.CreatePHI(Ty: GEPI.getType(), NumReservedValues: Phi->getNumIncomingValues(),
4597 Name: Phi->getName() + ".sroa.phi");
4598
4599 Type *SourceTy = GEPI.getSourceElementType();
4600 // We only handle arguments, constants, and static allocas here, so we can
4601 // insert GEPs at the end of the entry block.
4602 IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4603 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4604 Value *Op = Phi->getIncomingValue(i: I);
4605 BasicBlock *BB = Phi->getIncomingBlock(i: I);
4606 Value *NewGEP;
4607 if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) {
4608 NewGEP = NewPhi->getIncomingValue(i: NI);
4609 } else {
4610 SmallVector<Value *> NewOps = GetNewOps(Op);
4611 NewGEP =
4612 IRB.CreateGEP(Ty: SourceTy, Ptr: NewOps[0], IdxList: ArrayRef(NewOps).drop_front(),
4613 Name: Phi->getName() + ".sroa.gep", NW: GEPI.getNoWrapFlags());
4614 }
4615 NewPhi->addIncoming(V: NewGEP, BB);
4616 }
4617
4618 Visited.erase(Ptr: &GEPI);
4619 GEPI.replaceAllUsesWith(V: NewPhi);
4620 GEPI.eraseFromParent();
4621 Visited.insert(Ptr: NewPhi);
4622 enqueueUsers(I&: *NewPhi);
4623
4624 LLVM_DEBUG(dbgs() << " to: ";
4625 for (Value *In
4626 : NewPhi->incoming_values()) dbgs()
4627 << "\n " << *In;
4628 dbgs() << "\n " << *NewPhi << '\n');
4629
4630 return true;
4631 }
4632
4633 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4634 if (unfoldGEPSelect(GEPI))
4635 return true;
4636
4637 if (unfoldGEPPhi(GEPI))
4638 return true;
4639
4640 enqueueUsers(I&: GEPI);
4641 return false;
4642 }
4643
4644 bool visitPHINode(PHINode &PN) {
4645 enqueueUsers(I&: PN);
4646 return false;
4647 }
4648
4649 bool visitSelectInst(SelectInst &SI) {
4650 enqueueUsers(I&: SI);
4651 return false;
4652 }
4653};
4654
4655} // end anonymous namespace
4656
4657/// Strip aggregate type wrapping.
4658///
4659/// This removes no-op aggregate types wrapping an underlying type. It will
4660/// strip as many layers of types as it can without changing either the type
4661/// size or the allocated size.
4662static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
4663 if (Ty->isSingleValueType())
4664 return Ty;
4665
4666 uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4667 uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4668
4669 Type *InnerTy;
4670 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Val: Ty)) {
4671 InnerTy = ArrTy->getElementType();
4672 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4673 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4674 unsigned Index = SL->getElementContainingOffset(FixedOffset: 0);
4675 InnerTy = STy->getElementType(N: Index);
4676 } else {
4677 return Ty;
4678 }
4679
4680 if (AllocSize > DL.getTypeAllocSize(Ty: InnerTy).getFixedValue() ||
4681 TypeSize > DL.getTypeSizeInBits(Ty: InnerTy).getFixedValue())
4682 return Ty;
4683
4684 return stripAggregateTypeWrapping(DL, Ty: InnerTy);
4685}
4686
4687/// Try to find a partition of the aggregate type passed in for a given
4688/// offset and size.
4689///
4690/// This recurses through the aggregate type and tries to compute a subtype
4691/// based on the offset and size. When the offset and size span a sub-section
4692/// of an array, it will even compute a new array type for that sub-section,
4693/// and the same for structs.
4694///
4695/// Note that this routine is very strict and tries to find a partition of the
4696/// type which produces the *exact* right offset and size. It is not forgiving
4697/// when the size or offset cause either end of type-based partition to be off.
4698/// Also, this is a best-effort routine. It is reasonable to give up and not
4699/// return a type if necessary.
4700static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
4701 uint64_t Size) {
4702 if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4703 return stripAggregateTypeWrapping(DL, Ty);
4704 if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() ||
4705 (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4706 return nullptr;
4707
4708 if (isa<ArrayType>(Val: Ty) || isa<VectorType>(Val: Ty)) {
4709 Type *ElementTy;
4710 uint64_t TyNumElements;
4711 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
4712 ElementTy = AT->getElementType();
4713 TyNumElements = AT->getNumElements();
4714 } else {
4715 // FIXME: This isn't right for vectors with non-byte-sized or
4716 // non-power-of-two sized elements.
4717 auto *VT = cast<FixedVectorType>(Val: Ty);
4718 ElementTy = VT->getElementType();
4719 TyNumElements = VT->getNumElements();
4720 }
4721 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4722 uint64_t NumSkippedElements = Offset / ElementSize;
4723 if (NumSkippedElements >= TyNumElements)
4724 return nullptr;
4725 Offset -= NumSkippedElements * ElementSize;
4726
4727 // First check if we need to recurse.
4728 if (Offset > 0 || Size < ElementSize) {
4729 // Bail if the partition ends in a different array element.
4730 if ((Offset + Size) > ElementSize)
4731 return nullptr;
4732 // Recurse through the element type trying to peel off offset bytes.
4733 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4734 }
4735 assert(Offset == 0);
4736
4737 if (Size == ElementSize)
4738 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4739 assert(Size > ElementSize);
4740 uint64_t NumElements = Size / ElementSize;
4741 if (NumElements * ElementSize != Size)
4742 return nullptr;
4743 return ArrayType::get(ElementType: ElementTy, NumElements);
4744 }
4745
4746 StructType *STy = dyn_cast<StructType>(Val: Ty);
4747 if (!STy)
4748 return nullptr;
4749
4750 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4751
4752 if (SL->getSizeInBits().isScalable())
4753 return nullptr;
4754
4755 if (Offset >= SL->getSizeInBytes())
4756 return nullptr;
4757 uint64_t EndOffset = Offset + Size;
4758 if (EndOffset > SL->getSizeInBytes())
4759 return nullptr;
4760
4761 unsigned Index = SL->getElementContainingOffset(FixedOffset: Offset);
4762 Offset -= SL->getElementOffset(Idx: Index);
4763
4764 Type *ElementTy = STy->getElementType(N: Index);
4765 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4766 if (Offset >= ElementSize)
4767 return nullptr; // The offset points into alignment padding.
4768
4769 // See if any partition must be contained by the element.
4770 if (Offset > 0 || Size < ElementSize) {
4771 if ((Offset + Size) > ElementSize)
4772 return nullptr;
4773 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4774 }
4775 assert(Offset == 0);
4776
4777 if (Size == ElementSize)
4778 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4779
4780 StructType::element_iterator EI = STy->element_begin() + Index,
4781 EE = STy->element_end();
4782 if (EndOffset < SL->getSizeInBytes()) {
4783 unsigned EndIndex = SL->getElementContainingOffset(FixedOffset: EndOffset);
4784 if (Index == EndIndex)
4785 return nullptr; // Within a single element and its padding.
4786
4787 // Don't try to form "natural" types if the elements don't line up with the
4788 // expected size.
4789 // FIXME: We could potentially recurse down through the last element in the
4790 // sub-struct to find a natural end point.
4791 if (SL->getElementOffset(Idx: EndIndex) != EndOffset)
4792 return nullptr;
4793
4794 assert(Index < EndIndex);
4795 EE = STy->element_begin() + EndIndex;
4796 }
4797
4798 // Try to build up a sub-structure.
4799 StructType *SubTy =
4800 StructType::get(Context&: STy->getContext(), Elements: ArrayRef(EI, EE), isPacked: STy->isPacked());
4801 const StructLayout *SubSL = DL.getStructLayout(Ty: SubTy);
4802 if (Size != SubSL->getSizeInBytes())
4803 return nullptr; // The sub-struct doesn't have quite the size needed.
4804
4805 return SubTy;
4806}
4807
4808/// Pre-split loads and stores to simplify rewriting.
4809///
4810/// We want to break up the splittable load+store pairs as much as
4811/// possible. This is important to do as a preprocessing step, as once we
4812/// start rewriting the accesses to partitions of the alloca we lose the
4813/// necessary information to correctly split apart paired loads and stores
4814/// which both point into this alloca. The case to consider is something like
4815/// the following:
4816///
4817/// %a = alloca [12 x i8]
4818/// %gep1 = getelementptr i8, ptr %a, i32 0
4819/// %gep2 = getelementptr i8, ptr %a, i32 4
4820/// %gep3 = getelementptr i8, ptr %a, i32 8
4821/// store float 0.0, ptr %gep1
4822/// store float 1.0, ptr %gep2
4823/// %v = load i64, ptr %gep1
4824/// store i64 %v, ptr %gep2
4825/// %f1 = load float, ptr %gep2
4826/// %f2 = load float, ptr %gep3
4827///
4828/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
4829/// promote everything so we recover the 2 SSA values that should have been
4830/// there all along.
4831///
4832/// \returns true if any changes are made.
4833bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
4834 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
4835
4836 // Track the loads and stores which are candidates for pre-splitting here, in
4837 // the order they first appear during the partition scan. These give stable
4838 // iteration order and a basis for tracking which loads and stores we
4839 // actually split.
4840 SmallVector<LoadInst *, 4> Loads;
4841 SmallVector<StoreInst *, 4> Stores;
4842
4843 // We need to accumulate the splits required of each load or store where we
4844 // can find them via a direct lookup. This is important to cross-check loads
4845 // and stores against each other. We also track the slice so that we can kill
4846 // all the slices that end up split.
4847 struct SplitOffsets {
4848 Slice *S;
4849 std::vector<uint64_t> Splits;
4850 };
4851 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
4852
4853 // Track loads out of this alloca which cannot, for any reason, be pre-split.
4854 // This is important as we also cannot pre-split stores of those loads!
4855 // FIXME: This is all pretty gross. It means that we can be more aggressive
4856 // in pre-splitting when the load feeding the store happens to come from
4857 // a separate alloca. Put another way, the effectiveness of SROA would be
4858 // decreased by a frontend which just concatenated all of its local allocas
4859 // into one big flat alloca. But defeating such patterns is exactly the job
4860 // SROA is tasked with! Sadly, to not have this discrepancy we would have
4861 // change store pre-splitting to actually force pre-splitting of the load
4862 // that feeds it *and all stores*. That makes pre-splitting much harder, but
4863 // maybe it would make it more principled?
4864 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
4865
4866 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
4867 for (auto &P : AS.partitions()) {
4868 for (Slice &S : P) {
4869 Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
4870 if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
4871 // If this is a load we have to track that it can't participate in any
4872 // pre-splitting. If this is a store of a load we have to track that
4873 // that load also can't participate in any pre-splitting.
4874 if (auto *LI = dyn_cast<LoadInst>(Val: I))
4875 UnsplittableLoads.insert(Ptr: LI);
4876 else if (auto *SI = dyn_cast<StoreInst>(Val: I))
4877 if (auto *LI = dyn_cast<LoadInst>(Val: SI->getValueOperand()))
4878 UnsplittableLoads.insert(Ptr: LI);
4879 continue;
4880 }
4881 assert(P.endOffset() > S.beginOffset() &&
4882 "Empty or backwards partition!");
4883
4884 // Determine if this is a pre-splittable slice.
4885 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
4886 assert(!LI->isVolatile() && "Cannot split volatile loads!");
4887
4888 // The load must be used exclusively to store into other pointers for
4889 // us to be able to arbitrarily pre-split it. The stores must also be
4890 // simple to avoid changing semantics.
4891 auto IsLoadSimplyStored = [](LoadInst *LI) {
4892 for (User *LU : LI->users()) {
4893 auto *SI = dyn_cast<StoreInst>(Val: LU);
4894 if (!SI || !SI->isSimple())
4895 return false;
4896 }
4897 return true;
4898 };
4899 if (!IsLoadSimplyStored(LI)) {
4900 UnsplittableLoads.insert(Ptr: LI);
4901 continue;
4902 }
4903
4904 Loads.push_back(Elt: LI);
4905 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4906 if (S.getUse() != &SI->getOperandUse(i: SI->getPointerOperandIndex()))
4907 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
4908 continue;
4909 auto *StoredLoad = dyn_cast<LoadInst>(Val: SI->getValueOperand());
4910 if (!StoredLoad || !StoredLoad->isSimple())
4911 continue;
4912 assert(!SI->isVolatile() && "Cannot split volatile stores!");
4913
4914 Stores.push_back(Elt: SI);
4915 } else {
4916 // Other uses cannot be pre-split.
4917 continue;
4918 }
4919
4920 // Record the initial split.
4921 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
4922 auto &Offsets = SplitOffsetsMap[I];
4923 assert(Offsets.Splits.empty() &&
4924 "Should not have splits the first time we see an instruction!");
4925 Offsets.S = &S;
4926 Offsets.Splits.push_back(x: P.endOffset() - S.beginOffset());
4927 }
4928
4929 // Now scan the already split slices, and add a split for any of them which
4930 // we're going to pre-split.
4931 for (Slice *S : P.splitSliceTails()) {
4932 auto SplitOffsetsMapI =
4933 SplitOffsetsMap.find(Val: cast<Instruction>(Val: S->getUse()->getUser()));
4934 if (SplitOffsetsMapI == SplitOffsetsMap.end())
4935 continue;
4936 auto &Offsets = SplitOffsetsMapI->second;
4937
4938 assert(Offsets.S == S && "Found a mismatched slice!");
4939 assert(!Offsets.Splits.empty() &&
4940 "Cannot have an empty set of splits on the second partition!");
4941 assert(Offsets.Splits.back() ==
4942 P.beginOffset() - Offsets.S->beginOffset() &&
4943 "Previous split does not end where this one begins!");
4944
4945 // Record each split. The last partition's end isn't needed as the size
4946 // of the slice dictates that.
4947 if (S->endOffset() > P.endOffset())
4948 Offsets.Splits.push_back(x: P.endOffset() - Offsets.S->beginOffset());
4949 }
4950 }
4951
4952 // We may have split loads where some of their stores are split stores. For
4953 // such loads and stores, we can only pre-split them if their splits exactly
4954 // match relative to their starting offset. We have to verify this prior to
4955 // any rewriting.
4956 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
4957 // Lookup the load we are storing in our map of split
4958 // offsets.
4959 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
4960 // If it was completely unsplittable, then we're done,
4961 // and this store can't be pre-split.
4962 if (UnsplittableLoads.count(Ptr: LI))
4963 return true;
4964
4965 auto LoadOffsetsI = SplitOffsetsMap.find(Val: LI);
4966 if (LoadOffsetsI == SplitOffsetsMap.end())
4967 return false; // Unrelated loads are definitely safe.
4968 auto &LoadOffsets = LoadOffsetsI->second;
4969
4970 // Now lookup the store's offsets.
4971 auto &StoreOffsets = SplitOffsetsMap[SI];
4972
4973 // If the relative offsets of each split in the load and
4974 // store match exactly, then we can split them and we
4975 // don't need to remove them here.
4976 if (LoadOffsets.Splits == StoreOffsets.Splits)
4977 return false;
4978
4979 LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
4980 << " " << *LI << "\n"
4981 << " " << *SI << "\n");
4982
4983 // We've found a store and load that we need to split
4984 // with mismatched relative splits. Just give up on them
4985 // and remove both instructions from our list of
4986 // candidates.
4987 UnsplittableLoads.insert(Ptr: LI);
4988 return true;
4989 });
4990 // Now we have to go *back* through all the stores, because a later store may
4991 // have caused an earlier store's load to become unsplittable and if it is
4992 // unsplittable for the later store, then we can't rely on it being split in
4993 // the earlier store either.
4994 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads](StoreInst *SI) {
4995 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
4996 return UnsplittableLoads.count(Ptr: LI);
4997 });
4998 // Once we've established all the loads that can't be split for some reason,
4999 // filter any that made it into our list out.
5000 llvm::erase_if(C&: Loads, P: [&UnsplittableLoads](LoadInst *LI) {
5001 return UnsplittableLoads.count(Ptr: LI);
5002 });
5003
5004 // If no loads or stores are left, there is no pre-splitting to be done for
5005 // this alloca.
5006 if (Loads.empty() && Stores.empty())
5007 return false;
5008
5009 // From here on, we can't fail and will be building new accesses, so rig up
5010 // an IR builder.
5011 IRBuilderTy IRB(&AI);
5012
5013 // Collect the new slices which we will merge into the alloca slices.
5014 SmallVector<Slice, 4> NewSlices;
5015
5016 // Track any allocas we end up splitting loads and stores for so we iterate
5017 // on them.
5018 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
5019
5020 // At this point, we have collected all of the loads and stores we can
5021 // pre-split, and the specific splits needed for them. We actually do the
5022 // splitting in a specific order in order to handle when one of the loads in
5023 // the value operand to one of the stores.
5024 //
5025 // First, we rewrite all of the split loads, and just accumulate each split
5026 // load in a parallel structure. We also build the slices for them and append
5027 // them to the alloca slices.
5028 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
5029 std::vector<LoadInst *> SplitLoads;
5030 const DataLayout &DL = AI.getDataLayout();
5031 for (LoadInst *LI : Loads) {
5032 SplitLoads.clear();
5033
5034 auto &Offsets = SplitOffsetsMap[LI];
5035 unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5036 assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&
5037 "Load must have type size equal to store size");
5038 assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&
5039 "Load must be >= slice size");
5040
5041 uint64_t BaseOffset = Offsets.S->beginOffset();
5042 assert(BaseOffset + SliceSize > BaseOffset &&
5043 "Cannot represent alloca access size using 64-bit integers!");
5044
5045 Instruction *BasePtr = cast<Instruction>(Val: LI->getPointerOperand());
5046 IRB.SetInsertPoint(LI);
5047
5048 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5049
5050 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5051 int Idx = 0, Size = Offsets.Splits.size();
5052 for (;;) {
5053 auto *PartTy = Type::getIntNTy(C&: LI->getContext(), N: PartSize * 8);
5054 auto AS = LI->getPointerAddressSpace();
5055 auto *PartPtrTy = LI->getPointerOperandType();
5056 LoadInst *PLoad = IRB.CreateAlignedLoad(
5057 Ty: PartTy,
5058 Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5059 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5060 PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5061 Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5062 /*IsVolatile*/ isVolatile: false, Name: LI->getName());
5063 PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5064 LLVMContext::MD_access_group});
5065
5066 // Append this load onto the list of split loads so we can find it later
5067 // to rewrite the stores.
5068 SplitLoads.push_back(x: PLoad);
5069
5070 // Now build a new slice for the alloca.
5071 NewSlices.push_back(
5072 Elt: Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5073 &PLoad->getOperandUse(i: PLoad->getPointerOperandIndex()),
5074 /*IsSplittable*/ false));
5075 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5076 << ", " << NewSlices.back().endOffset()
5077 << "): " << *PLoad << "\n");
5078
5079 // See if we've handled all the splits.
5080 if (Idx >= Size)
5081 break;
5082
5083 // Setup the next partition.
5084 PartOffset = Offsets.Splits[Idx];
5085 ++Idx;
5086 PartSize = (Idx < Size ? Offsets.Splits[Idx] : SliceSize) - PartOffset;
5087 }
5088
5089 // Now that we have the split loads, do the slow walk over all uses of the
5090 // load and rewrite them as split stores, or save the split loads to use
5091 // below if the store is going to be split there anyways.
5092 bool DeferredStores = false;
5093 for (User *LU : LI->users()) {
5094 StoreInst *SI = cast<StoreInst>(Val: LU);
5095 if (!Stores.empty() && SplitOffsetsMap.count(Val: SI)) {
5096 DeferredStores = true;
5097 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5098 << "\n");
5099 continue;
5100 }
5101
5102 Value *StoreBasePtr = SI->getPointerOperand();
5103 IRB.SetInsertPoint(SI);
5104 AAMDNodes AATags = SI->getAAMetadata();
5105
5106 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5107
5108 for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5109 LoadInst *PLoad = SplitLoads[Idx];
5110 uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1];
5111 auto *PartPtrTy = SI->getPointerOperandType();
5112
5113 auto AS = SI->getPointerAddressSpace();
5114 StoreInst *PStore = IRB.CreateAlignedStore(
5115 Val: PLoad,
5116 Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5117 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5118 PointerTy: PartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5119 Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5120 /*IsVolatile*/ isVolatile: false);
5121 PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5122 LLVMContext::MD_access_group,
5123 LLVMContext::MD_DIAssignID});
5124
5125 if (AATags)
5126 PStore->setAAMetadata(
5127 AATags.adjustForAccess(Offset: PartOffset, AccessTy: PLoad->getType(), DL));
5128 LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5129 }
5130
5131 // We want to immediately iterate on any allocas impacted by splitting
5132 // this store, and we have to track any promotable alloca (indicated by
5133 // a direct store) as needing to be resplit because it is no longer
5134 // promotable.
5135 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: StoreBasePtr)) {
5136 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5137 Worklist.insert(X: OtherAI);
5138 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5139 Val: StoreBasePtr->stripInBoundsOffsets())) {
5140 Worklist.insert(X: OtherAI);
5141 }
5142
5143 // Mark the original store as dead.
5144 DeadInsts.push_back(Elt: SI);
5145 }
5146
5147 // Save the split loads if there are deferred stores among the users.
5148 if (DeferredStores)
5149 SplitLoadsMap.insert(KV: std::make_pair(x&: LI, y: std::move(SplitLoads)));
5150
5151 // Mark the original load as dead and kill the original slice.
5152 DeadInsts.push_back(Elt: LI);
5153 Offsets.S->kill();
5154 }
5155
5156 // Second, we rewrite all of the split stores. At this point, we know that
5157 // all loads from this alloca have been split already. For stores of such
5158 // loads, we can simply look up the pre-existing split loads. For stores of
5159 // other loads, we split those loads first and then write split stores of
5160 // them.
5161 for (StoreInst *SI : Stores) {
5162 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5163 IntegerType *Ty = cast<IntegerType>(Val: LI->getType());
5164 assert(Ty->getBitWidth() % 8 == 0);
5165 uint64_t StoreSize = Ty->getBitWidth() / 8;
5166 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
5167
5168 auto &Offsets = SplitOffsetsMap[SI];
5169 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5170 "Slice size should always match load size exactly!");
5171 uint64_t BaseOffset = Offsets.S->beginOffset();
5172 assert(BaseOffset + StoreSize > BaseOffset &&
5173 "Cannot represent alloca access size using 64-bit integers!");
5174
5175 Value *LoadBasePtr = LI->getPointerOperand();
5176 Instruction *StoreBasePtr = cast<Instruction>(Val: SI->getPointerOperand());
5177
5178 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5179
5180 // Check whether we have an already split load.
5181 auto SplitLoadsMapI = SplitLoadsMap.find(Val: LI);
5182 std::vector<LoadInst *> *SplitLoads = nullptr;
5183 if (SplitLoadsMapI != SplitLoadsMap.end()) {
5184 SplitLoads = &SplitLoadsMapI->second;
5185 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
5186 "Too few split loads for the number of splits in the store!");
5187 } else {
5188 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5189 }
5190
5191 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5192 int Idx = 0, Size = Offsets.Splits.size();
5193 for (;;) {
5194 auto *PartTy = Type::getIntNTy(C&: Ty->getContext(), N: PartSize * 8);
5195 auto *LoadPartPtrTy = LI->getPointerOperandType();
5196 auto *StorePartPtrTy = SI->getPointerOperandType();
5197
5198 // Either lookup a split load or create one.
5199 LoadInst *PLoad;
5200 if (SplitLoads) {
5201 PLoad = (*SplitLoads)[Idx];
5202 } else {
5203 IRB.SetInsertPoint(LI);
5204 auto AS = LI->getPointerAddressSpace();
5205 PLoad = IRB.CreateAlignedLoad(
5206 Ty: PartTy,
5207 Ptr: getAdjustedPtr(IRB, DL, Ptr: LoadBasePtr,
5208 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5209 PointerTy: LoadPartPtrTy, NamePrefix: LoadBasePtr->getName() + "."),
5210 Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5211 /*IsVolatile*/ isVolatile: false, Name: LI->getName());
5212 PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5213 LLVMContext::MD_access_group});
5214 }
5215
5216 // And store this partition.
5217 IRB.SetInsertPoint(SI);
5218 auto AS = SI->getPointerAddressSpace();
5219 StoreInst *PStore = IRB.CreateAlignedStore(
5220 Val: PLoad,
5221 Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5222 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5223 PointerTy: StorePartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5224 Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5225 /*IsVolatile*/ isVolatile: false);
5226 PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5227 LLVMContext::MD_access_group});
5228
5229 // Now build a new slice for the alloca.
5230 NewSlices.push_back(
5231 Elt: Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5232 &PStore->getOperandUse(i: PStore->getPointerOperandIndex()),
5233 /*IsSplittable*/ false));
5234 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5235 << ", " << NewSlices.back().endOffset()
5236 << "): " << *PStore << "\n");
5237 if (!SplitLoads) {
5238 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5239 }
5240
5241 // See if we've finished all the splits.
5242 if (Idx >= Size)
5243 break;
5244
5245 // Setup the next partition.
5246 PartOffset = Offsets.Splits[Idx];
5247 ++Idx;
5248 PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset;
5249 }
5250
5251 // We want to immediately iterate on any allocas impacted by splitting
5252 // this load, which is only relevant if it isn't a load of this alloca and
5253 // thus we didn't already split the loads above. We also have to keep track
5254 // of any promotable allocas we split loads on as they can no longer be
5255 // promoted.
5256 if (!SplitLoads) {
5257 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: LoadBasePtr)) {
5258 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5259 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5260 Worklist.insert(X: OtherAI);
5261 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5262 Val: LoadBasePtr->stripInBoundsOffsets())) {
5263 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5264 Worklist.insert(X: OtherAI);
5265 }
5266 }
5267
5268 // Mark the original store as dead now that we've split it up and kill its
5269 // slice. Note that we leave the original load in place unless this store
5270 // was its only use. It may in turn be split up if it is an alloca load
5271 // for some other alloca, but it may be a normal load. This may introduce
5272 // redundant loads, but where those can be merged the rest of the optimizer
5273 // should handle the merging, and this uncovers SSA splits which is more
5274 // important. In practice, the original loads will almost always be fully
5275 // split and removed eventually, and the splits will be merged by any
5276 // trivial CSE, including instcombine.
5277 if (LI->hasOneUse()) {
5278 assert(*LI->user_begin() == SI && "Single use isn't this store!");
5279 DeadInsts.push_back(Elt: LI);
5280 }
5281 DeadInsts.push_back(Elt: SI);
5282 Offsets.S->kill();
5283 }
5284
5285 // Remove the killed slices that have ben pre-split.
5286 llvm::erase_if(C&: AS, P: [](const Slice &S) { return S.isDead(); });
5287
5288 // Insert our new slices. This will sort and merge them into the sorted
5289 // sequence.
5290 AS.insert(NewSlices);
5291
5292 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5293#ifndef NDEBUG
5294 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5295 LLVM_DEBUG(AS.print(dbgs(), I, " "));
5296#endif
5297
5298 // Finally, don't try to promote any allocas that new require re-splitting.
5299 // They have already been added to the worklist above.
5300 PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5301
5302 return true;
5303}
5304
5305/// Try to canonicalize a homogeneous struct partition to a vector type.
5306///
5307/// We can do this if all the elements of the struct are the same and the
5308/// corresponding vector has the same byte-level layout. This can sometimes
5309/// eliminate allocas because structs cannot get promoted to LLVM values, but
5310/// vectors can.
5311///
5312/// We only apply this transformation when all users of the partition are memory
5313/// intrinsics. Otherwise, if there is a load or store of some other type to the
5314/// partition, SROA would select that type.
5315///
5316/// Applying this transformation too early may hinder memcpyopt, which may
5317/// generate better code when eliminating allocas. For example, see
5318/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5319/// this before memcpyopt can initialize previously uninitialized memory when
5320/// the alloca gets promoted to an SSA value. For another example, see
5321/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5322/// this before memcpyopt can result in promoting an alloca so that we load a
5323/// temporary value instead of copying the temporary value into memory, whereas
5324/// memcpyopt eliminates the temporary altogether.
5325///
5326/// As such, we only apply this transformation after memcpyopt has run. We gate
5327/// this transformation by the "AggregateToVector" pass option.
5328static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
5329 Partition &P,
5330 const DataLayout &DL) {
5331 unsigned NumElts = STy->getNumElements();
5332
5333 Type *EltTy = STy->getElementType(N: 0);
5334 if (!llvm::all_equal(Range: STy->elements()))
5335 return nullptr;
5336
5337 bool IsIntegralPointerTy =
5338 EltTy->isPointerTy() && !DL.isNonIntegralPointerType(Ty: EltTy);
5339 if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5340 !IsIntegralPointerTy)
5341 return nullptr;
5342
5343 // Ensure the struct is tightly packed so that the bit-layout is the same as
5344 // the corresponding vector. For example, this prevents a miscompile for
5345 // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5346 // tightly packed elements and trailing padding.
5347 if (DL.getTypeSizeInBits(Ty: EltTy) != DL.getTypeAllocSizeInBits(Ty: EltTy))
5348 return nullptr;
5349
5350 auto *VTy = FixedVectorType::get(ElementType: EltTy, NumElts);
5351 TypeSize StructSize = DL.getStructLayout(Ty: STy)->getSizeInBytes();
5352 TypeSize VectorSize = DL.getTypeStoreSize(Ty: VTy);
5353 // After ruling out per-element padding, make sure a vector load/store
5354 // covers the same number of bytes as the struct layout.
5355 if (StructSize != VectorSize)
5356 return nullptr;
5357
5358 auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5359 if (S.isDead())
5360 return true;
5361 auto *U = S.getUse();
5362 if (!U)
5363 return true;
5364
5365 User *Usr = U->getUser();
5366 if (isa<LifetimeIntrinsic>(Val: Usr) || isa<DbgInfoIntrinsic>(Val: Usr))
5367 return true;
5368
5369 return isa<MemIntrinsic>(Val: Usr);
5370 };
5371
5372 for (const Slice &S : P)
5373 if (!IsIgnorableOrMemIntrinsicSlice(S))
5374 return nullptr;
5375
5376 for (const Slice *S : P.splitSliceTails())
5377 if (!IsIgnorableOrMemIntrinsicSlice(*S))
5378 return nullptr;
5379
5380 return VTy;
5381}
5382
5383/// Select a partition type for an alloca partition.
5384///
5385/// Try to compute a friendly type for this partition of the alloca. This
5386/// won't always succeed, in which case we fall back to a legal integer type
5387/// or an i8 array of an appropriate size.
5388///
5389/// \returns A tuple with the following elements:
5390/// - PartitionType: The computed type for this partition.
5391/// - IsIntegerWideningViable: True if integer widening promotion is used.
5392/// - VectorType: The vector type if vector promotion is used, otherwise
5393/// nullptr.
5394static std::tuple<Type *, bool, VectorType *>
5395selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
5396 LLVMContext &C, bool AggregateToVector) {
5397 auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5398 VectorType *SelectedVecTy, bool SelectedIntWidening) {
5399 LLVM_DEBUG({
5400 dbgs() << "selectPartitionType path=" << Path
5401 << " func=" << AI.getFunction()->getName() << " alloca=";
5402 if (AI.hasName())
5403 dbgs() << AI.getName();
5404 else
5405 dbgs() << "<unnamed>";
5406 dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5407 << ") size=" << P.size();
5408 if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5409 dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5410 if (SelectedTy)
5411 dbgs() << " chosen=" << *SelectedTy;
5412 if (SelectedVecTy)
5413 dbgs() << " vec=" << *SelectedVecTy;
5414 dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5415 });
5416 };
5417 // First check if the partition is viable for vector promotion.
5418 //
5419 // We prefer vector promotion over integer widening promotion when:
5420 // - The vector element type is a floating-point type.
5421 // - All the loads/stores to the alloca are vector loads/stores to the
5422 // entire alloca or load/store a single element of the vector.
5423 //
5424 // Otherwise when there is an integer vector with mixed type loads/stores we
5425 // prefer integer widening promotion because it's more likely the user is
5426 // doing bitwise arithmetic and we generate better code.
5427 VectorType *VecTy =
5428 isVectorPromotionViable(P, DL, VScale: AI.getFunction()->getVScaleValue());
5429 // If the vector element type is a floating-point type, we prefer vector
5430 // promotion. If the vector has one element, let the below code select
5431 // whether we promote with the vector or scalar.
5432 if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5433 VecTy->getElementCount().getFixedValue() > 1) {
5434 LogSelection("direct-fp-vecty", VecTy, VecTy, false);
5435 return {VecTy, false, VecTy};
5436 }
5437
5438 // Check if there is a common type that all slices of the partition use that
5439 // spans the partition.
5440 auto [CommonUseTy, LargestIntTy] =
5441 findCommonType(B: P.begin(), E: P.end(), EndOffset: P.endOffset());
5442 if (CommonUseTy) {
5443 TypeSize CommonUseSize = DL.getTypeAllocSize(Ty: CommonUseTy);
5444 if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5445 // We prefer vector promotion here because if vector promotion is viable
5446 // and there is a common type used, then it implies the second listed
5447 // condition for preferring vector promotion is true.
5448 if (VecTy) {
5449 LogSelection("common-type-vecty", VecTy, VecTy, false);
5450 return {VecTy, false, VecTy};
5451 }
5452 bool IntWiden = isIntegerWideningViable(P, AllocaTy: CommonUseTy, DL);
5453 LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
5454 return {CommonUseTy, IntWiden, nullptr};
5455 }
5456 }
5457
5458 // Can we find an appropriate subtype in the original allocated
5459 // type?
5460 if (Type *TypePartitionTy = getTypePartition(DL, Ty: AI.getAllocatedType(),
5461 Offset: P.beginOffset(), Size: P.size())) {
5462 // If the partition is an integer array that can be spanned by a legal
5463 // integer type, prefer to represent it as a legal integer type because
5464 // it's more likely to be promotable.
5465 if (TypePartitionTy->isArrayTy() &&
5466 TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5467 DL.isLegalInteger(Width: P.size() * 8))
5468 TypePartitionTy = Type::getIntNTy(C, N: P.size() * 8);
5469 // There was no common type used, so we prefer integer widening promotion.
5470 if (isIntegerWideningViable(P, AllocaTy: TypePartitionTy, DL)) {
5471 LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
5472 return {TypePartitionTy, true, nullptr};
5473 }
5474 if (VecTy) {
5475 LogSelection("type-partition-vecty", VecTy, VecTy, false);
5476 return {VecTy, false, VecTy};
5477 }
5478 // If we couldn't promote with TypePartitionTy, try with the largest
5479 // integer type used.
5480 if (LargestIntTy &&
5481 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size() &&
5482 isIntegerWideningViable(P, AllocaTy: LargestIntTy, DL)) {
5483 LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
5484 return {LargestIntTy, true, nullptr};
5485 }
5486
5487 // Try homogeneous struct to vector canonicalization when requested. Running
5488 // this too early can hide memcpy chains from MemCpyOpt.
5489 if (AggregateToVector) {
5490 if (auto *STy = dyn_cast<StructType>(Val: TypePartitionTy)) {
5491 if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5492 LogSelection("struct-fallback-vecty", VTy, nullptr, false);
5493 return {VTy, false, nullptr};
5494 }
5495 }
5496 }
5497
5498 // Fallback to TypePartitionTy and we probably won't promote.
5499 LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
5500 return {TypePartitionTy, false, nullptr};
5501 }
5502
5503 // Select the largest integer type used if it spans the partition.
5504 if (LargestIntTy &&
5505 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size()) {
5506 LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
5507 return {LargestIntTy, false, nullptr};
5508 }
5509
5510 // Select a legal integer type if it spans the partition.
5511 if (DL.isLegalInteger(Width: P.size() * 8)) {
5512 Type *IntTy = Type::getIntNTy(C, N: P.size() * 8);
5513 LogSelection("legal-int-fallback", IntTy, nullptr, false);
5514 return {IntTy, false, nullptr};
5515 }
5516
5517 // Fallback to an i8 array.
5518 Type *ArrayTy = ArrayType::get(ElementType: Type::getInt8Ty(C), NumElements: P.size());
5519 LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
5520 return {ArrayTy, false, nullptr};
5521}
5522
5523/// Rewrite an alloca partition's users.
5524///
5525/// This routine drives both of the rewriting goals of the SROA pass. It tries
5526/// to rewrite uses of an alloca partition to be conducive for SSA value
5527/// promotion. If the partition needs a new, more refined alloca, this will
5528/// build that new alloca, preserving as much type information as possible, and
5529/// rewrite the uses of the old alloca to point at the new one and have the
5530/// appropriate new offsets. It also evaluates how successful the rewrite was
5531/// at enabling promotion and if it was successful queues the alloca to be
5532/// promoted.
5533std::pair<AllocaInst *, uint64_t>
5534SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5535 const DataLayout &DL = AI.getDataLayout();
5536 // Select the type for the new alloca that spans the partition.
5537 auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5538 selectPartitionType(P, DL, AI, C&: *C, AggregateToVector);
5539
5540 // Check for the case where we're going to rewrite to a new alloca of the
5541 // exact same type as the original, and with the same access offsets. In that
5542 // case, re-use the existing alloca, but still run through the rewriter to
5543 // perform phi and select speculation.
5544 // P.beginOffset() can be non-zero even with the same type in a case with
5545 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5546 AllocaInst *NewAI;
5547 if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == 0) {
5548 NewAI = &AI;
5549 // FIXME: We should be able to bail at this point with "nothing changed".
5550 // FIXME: We might want to defer PHI speculation until after here.
5551 // FIXME: return nullptr;
5552 } else {
5553 // Make sure the alignment is compatible with P.beginOffset().
5554 const Align Alignment = commonAlignment(A: AI.getAlign(), Offset: P.beginOffset());
5555 // If we will get at least this much alignment from the type alone, leave
5556 // the alloca's alignment unconstrained.
5557 const bool IsUnconstrained = Alignment <= DL.getABITypeAlign(Ty: PartitionTy);
5558 NewAI = new AllocaInst(
5559 PartitionTy, AI.getAddressSpace(), nullptr,
5560 IsUnconstrained ? DL.getPrefTypeAlign(Ty: PartitionTy) : Alignment,
5561 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()),
5562 AI.getIterator());
5563 // Copy the old AI debug location over to the new one.
5564 NewAI->setDebugLoc(AI.getDebugLoc());
5565 ++NumNewAllocas;
5566 }
5567
5568 LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5569 << "," << P.endOffset() << ") to: " << *NewAI << "\n");
5570
5571 // Track the high watermark on the worklist as it is only relevant for
5572 // promoted allocas. We will reset it to this point if the alloca is not in
5573 // fact scheduled for promotion.
5574 unsigned PPWOldSize = PostPromotionWorklist.size();
5575 unsigned NumUses = 0;
5576 SmallSetVector<PHINode *, 8> PHIUsers;
5577 SmallSetVector<SelectInst *, 8> SelectUsers;
5578
5579 AllocaSliceRewriter Rewriter(
5580 DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5581 IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5582 bool Promotable = true;
5583 // Check whether we can have tree-structured merge.
5584 if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5585 NumUses += DeletedValues->size() + 1;
5586 for (Value *V : *DeletedValues)
5587 DeadInsts.push_back(Elt: V);
5588 } else {
5589 for (Slice *S : P.splitSliceTails()) {
5590 Promotable &= Rewriter.visit(I: S);
5591 ++NumUses;
5592 }
5593 for (Slice &S : P) {
5594 Promotable &= Rewriter.visit(I: &S);
5595 ++NumUses;
5596 }
5597 }
5598
5599 NumAllocaPartitionUses += NumUses;
5600 MaxUsesPerAllocaPartition.updateMax(V: NumUses);
5601
5602 // Now that we've processed all the slices in the new partition, check if any
5603 // PHIs or Selects would block promotion.
5604 for (PHINode *PHI : PHIUsers)
5605 if (!isSafePHIToSpeculate(PN&: *PHI)) {
5606 Promotable = false;
5607 PHIUsers.clear();
5608 SelectUsers.clear();
5609 break;
5610 }
5611
5612 SmallVector<std::pair<SelectInst *, RewriteableMemOps>, 2>
5613 NewSelectsToRewrite;
5614 NewSelectsToRewrite.reserve(N: SelectUsers.size());
5615 for (SelectInst *Sel : SelectUsers) {
5616 std::optional<RewriteableMemOps> Ops =
5617 isSafeSelectToSpeculate(SI&: *Sel, PreserveCFG);
5618 if (!Ops) {
5619 Promotable = false;
5620 PHIUsers.clear();
5621 SelectUsers.clear();
5622 NewSelectsToRewrite.clear();
5623 break;
5624 }
5625 NewSelectsToRewrite.emplace_back(Args: std::make_pair(x&: Sel, y&: *Ops));
5626 }
5627
5628 if (Promotable) {
5629 for (Use *U : AS.getDeadUsesIfPromotable()) {
5630 auto *OldInst = dyn_cast<Instruction>(Val: U->get());
5631 Value::dropDroppableUse(U&: *U);
5632 if (OldInst)
5633 if (isInstructionTriviallyDead(I: OldInst))
5634 DeadInsts.push_back(Elt: OldInst);
5635 }
5636 if (PHIUsers.empty() && SelectUsers.empty()) {
5637 // Promote the alloca.
5638 PromotableAllocas.insert(X: NewAI);
5639 } else {
5640 // If we have either PHIs or Selects to speculate, add them to those
5641 // worklists and re-queue the new alloca so that we promote in on the
5642 // next iteration.
5643 SpeculatablePHIs.insert_range(R&: PHIUsers);
5644 SelectsToRewrite.reserve(NumEntries: SelectsToRewrite.size() +
5645 NewSelectsToRewrite.size());
5646 for (auto &&KV : llvm::make_range(
5647 x: std::make_move_iterator(i: NewSelectsToRewrite.begin()),
5648 y: std::make_move_iterator(i: NewSelectsToRewrite.end())))
5649 SelectsToRewrite.insert(KV: std::move(KV));
5650 Worklist.insert(X: NewAI);
5651 }
5652 } else {
5653 // Drop any post-promotion work items if promotion didn't happen.
5654 while (PostPromotionWorklist.size() > PPWOldSize)
5655 PostPromotionWorklist.pop_back();
5656
5657 // We couldn't promote and we didn't create a new partition, nothing
5658 // happened.
5659 if (NewAI == &AI)
5660 return {nullptr, 0};
5661
5662 // If we can't promote the alloca, iterate on it to check for new
5663 // refinements exposed by splitting the current alloca. Don't iterate on an
5664 // alloca which didn't actually change and didn't get promoted.
5665 Worklist.insert(X: NewAI);
5666 }
5667
5668 return {NewAI, DL.getTypeSizeInBits(Ty: PartitionTy).getFixedValue()};
5669}
5670
5671// There isn't a shared interface to get the "address" parts out of a
5672// dbg.declare and dbg.assign, so provide some wrappers.
5673bool isKillAddress(const DbgVariableRecord *DVR) {
5674 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5675 return DVR->isKillAddress();
5676 return DVR->isKillLocation();
5677}
5678
5679const DIExpression *getAddressExpression(const DbgVariableRecord *DVR) {
5680 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5681 return DVR->getAddressExpression();
5682 return DVR->getExpression();
5683}
5684
5685/// Create or replace an existing fragment in a DIExpression with \p Frag.
5686/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5687/// operation, add \p BitExtractOffset to the offset part.
5688///
5689/// Returns the new expression, or nullptr if this fails (see details below).
5690///
5691/// This function is similar to DIExpression::createFragmentExpression except
5692/// for 3 important distinctions:
5693/// 1. The new fragment isn't relative to an existing fragment.
5694/// 2. It assumes the computed location is a memory location. This means we
5695/// don't need to perform checks that creating the fragment preserves the
5696/// expression semantics.
5697/// 3. Existing extract_bits are modified independently of fragment changes
5698/// using \p BitExtractOffset. A change to the fragment offset or size
5699/// may affect a bit extract. But a bit extract offset can change
5700/// independently of the fragment dimensions.
5701///
5702/// Returns the new expression, or nullptr if one couldn't be created.
5703/// Ideally this is only used to signal that a bit-extract has become
5704/// zero-sized (and thus the new debug record has no size and can be
5705/// dropped), however, it fails for other reasons too - see the FIXME below.
5706///
5707/// FIXME: To keep the change that introduces this function NFC it bails
5708/// in some situations unecessarily, e.g. when fragment and bit extract
5709/// sizes differ.
5710static DIExpression *createOrReplaceFragment(const DIExpression *Expr,
5711 DIExpression::FragmentInfo Frag,
5712 int64_t BitExtractOffset) {
5713 SmallVector<uint64_t, 8> Ops;
5714 bool HasFragment = false;
5715 bool HasBitExtract = false;
5716
5717 for (auto &Op : Expr->expr_ops()) {
5718 if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5719 HasFragment = true;
5720 continue;
5721 }
5722 if (Op.getOp() == dwarf::DW_OP_LLVM_extract_bits_zext ||
5723 Op.getOp() == dwarf::DW_OP_LLVM_extract_bits_sext) {
5724 HasBitExtract = true;
5725 int64_t ExtractOffsetInBits = Op.getArg(I: 0);
5726 int64_t ExtractSizeInBits = Op.getArg(I: 1);
5727
5728 // DIExpression::createFragmentExpression doesn't know how to handle
5729 // a fragment that is smaller than the extract. Copy the behaviour
5730 // (bail) to avoid non-NFC changes.
5731 // FIXME: Don't do this.
5732 if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5733 return nullptr;
5734
5735 assert(BitExtractOffset <= 0);
5736 int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5737
5738 // DIExpression::createFragmentExpression doesn't know what to do
5739 // if the new extract starts "outside" the existing one. Copy the
5740 // behaviour (bail) to avoid non-NFC changes.
5741 // FIXME: Don't do this.
5742 if (AdjustedOffset < 0)
5743 return nullptr;
5744
5745 Ops.push_back(Elt: Op.getOp());
5746 Ops.push_back(Elt: std::max<int64_t>(a: 0, b: AdjustedOffset));
5747 Ops.push_back(Elt: ExtractSizeInBits);
5748 continue;
5749 }
5750 Op.appendToVector(V&: Ops);
5751 }
5752
5753 // Unsupported by createFragmentExpression, so don't support it here yet to
5754 // preserve NFC-ness.
5755 if (HasFragment && HasBitExtract)
5756 return nullptr;
5757
5758 if (!HasBitExtract) {
5759 Ops.push_back(Elt: dwarf::DW_OP_LLVM_fragment);
5760 Ops.push_back(Elt: Frag.OffsetInBits);
5761 Ops.push_back(Elt: Frag.SizeInBits);
5762 }
5763 return DIExpression::get(Context&: Expr->getContext(), Elements: Ops);
5764}
5765
5766/// Insert a new DbgRecord.
5767/// \p Orig Original to copy record type, debug loc and variable from, and
5768/// additionally value and value expression for dbg_assign records.
5769/// \p NewAddr Location's new base address.
5770/// \p NewAddrExpr New expression to apply to address.
5771/// \p BeforeInst Insert position.
5772/// \p NewFragment New fragment (absolute, non-relative).
5773/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
5774static void
5775insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr,
5776 DIExpression *NewAddrExpr, Instruction *BeforeInst,
5777 std::optional<DIExpression::FragmentInfo> NewFragment,
5778 int64_t BitExtractAdjustment) {
5779 (void)DIB;
5780
5781 // A dbg_assign puts fragment info in the value expression only. The address
5782 // expression has already been built: NewAddrExpr. A dbg_declare puts the
5783 // new fragment info into NewAddrExpr (as it only has one expression).
5784 DIExpression *NewFragmentExpr =
5785 Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
5786 if (NewFragment)
5787 NewFragmentExpr = createOrReplaceFragment(Expr: NewFragmentExpr, Frag: *NewFragment,
5788 BitExtractOffset: BitExtractAdjustment);
5789 if (!NewFragmentExpr)
5790 return;
5791
5792 if (Orig->isDbgDeclare()) {
5793 DbgVariableRecord *DVR = DbgVariableRecord::createDVRDeclare(
5794 Address: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5795 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5796 Here: BeforeInst->getIterator());
5797 return;
5798 }
5799
5800 if (Orig->isDbgValue()) {
5801 DbgVariableRecord *DVR = DbgVariableRecord::createDbgVariableRecord(
5802 Location: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5803 // Drop debug information if the expression doesn't start with a
5804 // DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
5805 // describes the address of alloca rather than the value inside the alloca.
5806 if (!NewFragmentExpr->startsWithDeref())
5807 DVR->setKillAddress();
5808 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5809 Here: BeforeInst->getIterator());
5810 return;
5811 }
5812
5813 // Apply a DIAssignID to the store if it doesn't already have it.
5814 if (!NewAddr->hasMetadata(KindID: LLVMContext::MD_DIAssignID)) {
5815 NewAddr->setMetadata(KindID: LLVMContext::MD_DIAssignID,
5816 Node: DIAssignID::getDistinct(Context&: NewAddr->getContext()));
5817 }
5818
5819 DbgVariableRecord *NewAssign = DbgVariableRecord::createLinkedDVRAssign(
5820 LinkedInstr: NewAddr, Val: Orig->getValue(), Variable: Orig->getVariable(), Expression: NewFragmentExpr, Address: NewAddr,
5821 AddressExpression: NewAddrExpr, DI: Orig->getDebugLoc());
5822 LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
5823 (void)NewAssign;
5824}
5825
5826/// Walks the slices of an alloca and form partitions based on them,
5827/// rewriting each of their uses.
5828bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
5829 if (AS.begin() == AS.end())
5830 return false;
5831
5832 unsigned NumPartitions = 0;
5833 bool Changed = false;
5834 const DataLayout &DL = AI.getModule()->getDataLayout();
5835
5836 // First try to pre-split loads and stores.
5837 Changed |= presplitLoadsAndStores(AI, AS);
5838
5839 // Now that we have identified any pre-splitting opportunities,
5840 // mark loads and stores unsplittable except for the following case.
5841 // We leave a slice splittable if all other slices are disjoint or fully
5842 // included in the slice, such as whole-alloca loads and stores.
5843 // If we fail to split these during pre-splitting, we want to force them
5844 // to be rewritten into a partition.
5845 bool IsSorted = true;
5846
5847 uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
5848 const uint64_t MaxBitVectorSize = 1024;
5849 if (AllocaSize <= MaxBitVectorSize) {
5850 // If a byte boundary is included in any load or store, a slice starting or
5851 // ending at the boundary is not splittable.
5852 SmallBitVector SplittableOffset(AllocaSize + 1, true);
5853 for (Slice &S : AS)
5854 for (unsigned O = S.beginOffset() + 1;
5855 O < S.endOffset() && O < AllocaSize; O++)
5856 SplittableOffset.reset(Idx: O);
5857
5858 for (Slice &S : AS) {
5859 if (!S.isSplittable())
5860 continue;
5861
5862 if ((S.beginOffset() > AllocaSize || SplittableOffset[S.beginOffset()]) &&
5863 (S.endOffset() > AllocaSize || SplittableOffset[S.endOffset()]))
5864 continue;
5865
5866 if (isa<LoadInst>(Val: S.getUse()->getUser()) ||
5867 isa<StoreInst>(Val: S.getUse()->getUser())) {
5868 S.makeUnsplittable();
5869 IsSorted = false;
5870 }
5871 }
5872 } else {
5873 // We only allow whole-alloca splittable loads and stores
5874 // for a large alloca to avoid creating too large BitVector.
5875 for (Slice &S : AS) {
5876 if (!S.isSplittable())
5877 continue;
5878
5879 if (S.beginOffset() == 0 && S.endOffset() >= AllocaSize)
5880 continue;
5881
5882 if (isa<LoadInst>(Val: S.getUse()->getUser()) ||
5883 isa<StoreInst>(Val: S.getUse()->getUser())) {
5884 S.makeUnsplittable();
5885 IsSorted = false;
5886 }
5887 }
5888 }
5889
5890 if (!IsSorted)
5891 llvm::stable_sort(Range&: AS);
5892
5893 /// Describes the allocas introduced by rewritePartition in order to migrate
5894 /// the debug info.
5895 struct Fragment {
5896 AllocaInst *Alloca;
5897 uint64_t Offset;
5898 uint64_t Size;
5899 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
5900 : Alloca(AI), Offset(O), Size(S) {}
5901 };
5902 SmallVector<Fragment, 4> Fragments;
5903
5904 // Rewrite each partition.
5905 for (auto &P : AS.partitions()) {
5906 auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
5907 if (NewAI) {
5908 Changed = true;
5909 if (NewAI != &AI) {
5910 uint64_t SizeOfByte = 8;
5911 // Don't include any padding.
5912 uint64_t Size = std::min(a: ActiveBits, b: P.size() * SizeOfByte);
5913 Fragments.push_back(
5914 Elt: Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
5915 }
5916 }
5917 ++NumPartitions;
5918 }
5919
5920 NumAllocaPartitions += NumPartitions;
5921 MaxPartitionsPerAlloca.updateMax(V: NumPartitions);
5922
5923 // Migrate debug information from the old alloca to the new alloca(s)
5924 // and the individual partitions.
5925 auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
5926 // Can't overlap with undef memory.
5927 if (isKillAddress(DVR: DbgVariable))
5928 return;
5929
5930 const Value *DbgPtr = DbgVariable->getAddress();
5931 DIExpression::FragmentInfo VarFrag =
5932 DbgVariable->getFragmentOrEntireVariable();
5933 // Get the address expression constant offset if one exists and the ops
5934 // that come after it.
5935 int64_t CurrentExprOffsetInBytes = 0;
5936 SmallVector<uint64_t> PostOffsetOps;
5937 if (!getAddressExpression(DVR: DbgVariable)
5938 ->extractLeadingOffset(OffsetInBytes&: CurrentExprOffsetInBytes, RemainingOps&: PostOffsetOps))
5939 return; // Couldn't interpret this DIExpression - drop the var.
5940
5941 // Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
5942 int64_t ExtractOffsetInBits = 0;
5943 for (auto Op : getAddressExpression(DVR: DbgVariable)->expr_ops()) {
5944 if (Op.getOp() == dwarf::DW_OP_LLVM_extract_bits_zext ||
5945 Op.getOp() == dwarf::DW_OP_LLVM_extract_bits_sext) {
5946 ExtractOffsetInBits = Op.getArg(I: 0);
5947 break;
5948 }
5949 }
5950
5951 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
5952 for (auto Fragment : Fragments) {
5953 int64_t OffsetFromLocationInBits;
5954 std::optional<DIExpression::FragmentInfo> NewDbgFragment;
5955 // Find the variable fragment that the new alloca slice covers.
5956 // Drop debug info for this variable fragment if we can't compute an
5957 // intersect between it and the alloca slice.
5958 if (!DIExpression::calculateFragmentIntersect(
5959 DL, SliceStart: &AI, SliceOffsetInBits: Fragment.Offset, SliceSizeInBits: Fragment.Size, DbgPtr,
5960 DbgPtrOffsetInBits: CurrentExprOffsetInBytes * 8, DbgExtractOffsetInBits: ExtractOffsetInBits, VarFrag,
5961 Result&: NewDbgFragment, OffsetFromLocationInBits))
5962 continue; // Do not migrate this fragment to this slice.
5963
5964 // Zero sized fragment indicates there's no intersect between the variable
5965 // fragment and the alloca slice. Skip this slice for this variable
5966 // fragment.
5967 if (NewDbgFragment && !NewDbgFragment->SizeInBits)
5968 continue; // Do not migrate this fragment to this slice.
5969
5970 // No fragment indicates DbgVariable's variable or fragment exactly
5971 // overlaps the slice; copy its fragment (or nullopt if there isn't one).
5972 if (!NewDbgFragment)
5973 NewDbgFragment = DbgVariable->getFragment();
5974
5975 // Reduce the new expression offset by the bit-extract offset since
5976 // we'll be keeping that.
5977 int64_t OffestFromNewAllocaInBits =
5978 OffsetFromLocationInBits - ExtractOffsetInBits;
5979 // We need to adjust an existing bit extract if the offset expression
5980 // can't eat the slack (i.e., if the new offset would be negative).
5981 int64_t BitExtractOffset =
5982 std::min<int64_t>(a: 0, b: OffestFromNewAllocaInBits);
5983 // The magnitude of a negative value indicates the number of bits into
5984 // the existing variable fragment that the memory region begins. The new
5985 // variable fragment already excludes those bits - the new DbgPtr offset
5986 // only needs to be applied if it's positive.
5987 OffestFromNewAllocaInBits =
5988 std::max(a: int64_t(0), b: OffestFromNewAllocaInBits);
5989
5990 // Rebuild the expression:
5991 // {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
5992 // Add NewDbgFragment later, because dbg.assigns don't want it in the
5993 // address expression but the value expression instead.
5994 DIExpression *NewExpr = DIExpression::get(Context&: AI.getContext(), Elements: PostOffsetOps);
5995 if (OffestFromNewAllocaInBits > 0) {
5996 int64_t OffsetInBytes = (OffestFromNewAllocaInBits + 7) / 8;
5997 NewExpr = DIExpression::prepend(Expr: NewExpr, /*flags=*/Flags: 0, Offset: OffsetInBytes);
5998 }
5999
6000 // Remove any existing intrinsics on the new alloca describing
6001 // the variable fragment.
6002 auto RemoveOne = [DbgVariable](auto *OldDII) {
6003 auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
6004 return LHS->getVariable() == RHS->getVariable() &&
6005 LHS->getDebugLoc()->getInlinedAt() ==
6006 RHS->getDebugLoc()->getInlinedAt();
6007 };
6008 if (SameVariableFragment(OldDII, DbgVariable))
6009 OldDII->eraseFromParent();
6010 };
6011 for_each(Range: findDVRDeclares(V: Fragment.Alloca), F: RemoveOne);
6012 for_each(Range: findDVRValues(V: Fragment.Alloca), F: RemoveOne);
6013 insertNewDbgInst(DIB, Orig: DbgVariable, NewAddr: Fragment.Alloca, NewAddrExpr: NewExpr, BeforeInst: &AI,
6014 NewFragment: NewDbgFragment, BitExtractAdjustment: BitExtractOffset);
6015 }
6016 };
6017
6018 // Migrate debug information from the old alloca to the new alloca(s)
6019 // and the individual partitions.
6020 for_each(Range: findDVRDeclares(V: &AI), F: MigrateOne);
6021 for_each(Range: findDVRValues(V: &AI), F: MigrateOne);
6022 for_each(Range: at::getDVRAssignmentMarkers(Inst: &AI), F: MigrateOne);
6023
6024 return Changed;
6025}
6026
6027/// Clobber a use with poison, deleting the used value if it becomes dead.
6028void SROA::clobberUse(Use &U) {
6029 Value *OldV = U;
6030 // Replace the use with an poison value.
6031 U = PoisonValue::get(T: OldV->getType());
6032
6033 // Check for this making an instruction dead. We have to garbage collect
6034 // all the dead instructions to ensure the uses of any alloca end up being
6035 // minimal.
6036 if (Instruction *OldI = dyn_cast<Instruction>(Val: OldV))
6037 if (isInstructionTriviallyDead(I: OldI)) {
6038 DeadInsts.push_back(Elt: OldI);
6039 }
6040}
6041
6042/// A basic LoadAndStorePromoter that does not remove store nodes.
6043class BasicLoadAndStorePromoter : public LoadAndStorePromoter {
6044public:
6045 BasicLoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S,
6046 Type *ZeroType)
6047 : LoadAndStorePromoter(Insts, S), ZeroType(ZeroType) {}
6048 bool shouldDelete(Instruction *I) const override {
6049 return !isa<StoreInst>(Val: I) && !isa<AllocaInst>(Val: I);
6050 }
6051
6052 Value *getValueToUseForAlloca(Instruction *I) const override {
6053 return UndefValue::get(T: ZeroType);
6054 }
6055
6056private:
6057 Type *ZeroType;
6058};
6059
6060bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6061 // Look through each "partition", looking for slices with the same start/end
6062 // that do not overlap with any before them. The slices are sorted by
6063 // increasing beginOffset. We don't use AS.partitions(), as it will use a more
6064 // sophisticated algorithm that takes splittable slices into account.
6065 LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6066 bool AllSameAndValid = true;
6067 Type *PartitionType = nullptr;
6068 SmallVector<Instruction *> Insts;
6069 uint64_t BeginOffset = 0;
6070 uint64_t EndOffset = 0;
6071
6072 auto Flush = [&]() {
6073 if (AllSameAndValid && !Insts.empty()) {
6074 LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6075 << EndOffset << ")\n");
6076 SmallVector<PHINode *, 4> NewPHIs;
6077 SSAUpdater SSA(&NewPHIs);
6078 Insts.push_back(Elt: &AI);
6079 BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6080 Promoter.run(Insts);
6081 }
6082 AllSameAndValid = true;
6083 PartitionType = nullptr;
6084 Insts.clear();
6085 };
6086
6087 for (Slice &S : AS) {
6088 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
6089 if (isAssumeLikeIntrinsic(I: User)) {
6090 LLVM_DEBUG({
6091 dbgs() << "Ignoring slice: ";
6092 AS.print(dbgs(), &S);
6093 });
6094 continue;
6095 }
6096 if (S.beginOffset() >= EndOffset) {
6097 Flush();
6098 BeginOffset = S.beginOffset();
6099 EndOffset = S.endOffset();
6100 } else if (S.beginOffset() != BeginOffset || S.endOffset() != EndOffset) {
6101 if (AllSameAndValid) {
6102 LLVM_DEBUG({
6103 dbgs() << "Slice does not match range [" << BeginOffset << ", "
6104 << EndOffset << ")";
6105 AS.print(dbgs(), &S);
6106 });
6107 AllSameAndValid = false;
6108 }
6109 EndOffset = std::max(a: EndOffset, b: S.endOffset());
6110 continue;
6111 }
6112
6113 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
6114 Type *UserTy = LI->getType();
6115 // LoadAndStorePromoter requires all the types to be the same.
6116 if (!LI->isSimple() || (PartitionType && UserTy != PartitionType))
6117 AllSameAndValid = false;
6118 PartitionType = UserTy;
6119 Insts.push_back(Elt: User);
6120 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
6121 Type *UserTy = SI->getValueOperand()->getType();
6122 if (!SI->isSimple() || (PartitionType && UserTy != PartitionType))
6123 AllSameAndValid = false;
6124 PartitionType = UserTy;
6125 Insts.push_back(Elt: User);
6126 } else {
6127 AllSameAndValid = false;
6128 }
6129 }
6130
6131 Flush();
6132 return true;
6133}
6134
6135/// Analyze an alloca for SROA.
6136///
6137/// This analyzes the alloca to ensure we can reason about it, builds
6138/// the slices of the alloca, and then hands it off to be split and
6139/// rewritten as needed.
6140std::pair<bool /*Changed*/, bool /*CFGChanged*/>
6141SROA::runOnAlloca(AllocaInst &AI) {
6142 bool Changed = false;
6143 bool CFGChanged = false;
6144
6145 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6146 ++NumAllocasAnalyzed;
6147
6148 // Special case dead allocas, as they're trivial.
6149 if (AI.use_empty()) {
6150 AI.eraseFromParent();
6151 Changed = true;
6152 return {Changed, CFGChanged};
6153 }
6154 const DataLayout &DL = AI.getDataLayout();
6155
6156 // Skip alloca forms that this analysis can't handle.
6157 std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6158 if (AI.isArrayAllocation() || !Size || Size->isScalable() || Size->isZero())
6159 return {Changed, CFGChanged};
6160
6161 // First, split any FCA loads and stores touching this alloca to promote
6162 // better splitting and promotion opportunities.
6163 IRBuilderTy IRB(&AI);
6164 AggLoadStoreRewriter AggRewriter(DL, IRB);
6165 Changed |= AggRewriter.rewrite(I&: AI);
6166
6167 // Build the slices using a recursive instruction-visiting builder.
6168 AllocaSlices AS(DL, AI);
6169 LLVM_DEBUG(AS.print(dbgs()));
6170 if (AS.isEscaped())
6171 return {Changed, CFGChanged};
6172
6173 if (AS.isEscapedReadOnly()) {
6174 Changed |= propagateStoredValuesToLoads(AI, AS);
6175 return {Changed, CFGChanged};
6176 }
6177
6178 // Delete all the dead users of this alloca before splitting and rewriting it.
6179 for (Instruction *DeadUser : AS.getDeadUsers()) {
6180 // Free up everything used by this instruction.
6181 for (Use &DeadOp : DeadUser->operands())
6182 clobberUse(U&: DeadOp);
6183
6184 // Now replace the uses of this instruction.
6185 DeadUser->replaceAllUsesWith(V: PoisonValue::get(T: DeadUser->getType()));
6186
6187 // And mark it for deletion.
6188 DeadInsts.push_back(Elt: DeadUser);
6189 Changed = true;
6190 }
6191 for (Use *DeadOp : AS.getDeadOperands()) {
6192 clobberUse(U&: *DeadOp);
6193 Changed = true;
6194 }
6195
6196 // No slices to split. Leave the dead alloca for a later pass to clean up.
6197 if (AS.begin() == AS.end())
6198 return {Changed, CFGChanged};
6199
6200 Changed |= splitAlloca(AI, AS);
6201
6202 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6203 while (!SpeculatablePHIs.empty())
6204 speculatePHINodeLoads(IRB, PN&: *SpeculatablePHIs.pop_back_val());
6205
6206 LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6207 auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6208 while (!RemainingSelectsToRewrite.empty()) {
6209 const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6210 CFGChanged |=
6211 rewriteSelectInstMemOps(SI&: *K, Ops: V, IRB, DTU: PreserveCFG ? nullptr : DTU);
6212 }
6213
6214 return {Changed, CFGChanged};
6215}
6216
6217/// Delete the dead instructions accumulated in this run.
6218///
6219/// Recursively deletes the dead instructions we've accumulated. This is done
6220/// at the very end to maximize locality of the recursive delete and to
6221/// minimize the problems of invalidated instruction pointers as such pointers
6222/// are used heavily in the intermediate stages of the algorithm.
6223///
6224/// We also record the alloca instructions deleted here so that they aren't
6225/// subsequently handed to mem2reg to promote.
6226bool SROA::deleteDeadInstructions(
6227 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6228 bool Changed = false;
6229 while (!DeadInsts.empty()) {
6230 Instruction *I = dyn_cast_or_null<Instruction>(Val: DeadInsts.pop_back_val());
6231 if (!I)
6232 continue;
6233 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6234
6235 // If the instruction is an alloca, find the possible dbg.declare connected
6236 // to it, and remove it too. We must do this before calling RAUW or we will
6237 // not be able to find it.
6238 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: I)) {
6239 DeletedAllocas.insert(Ptr: AI);
6240 for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6241 OldDII->eraseFromParent();
6242 }
6243
6244 at::deleteAssignmentMarkers(Inst: I);
6245 I->replaceAllUsesWith(V: UndefValue::get(T: I->getType()));
6246
6247 for (Use &Operand : I->operands())
6248 if (Instruction *U = dyn_cast<Instruction>(Val&: Operand)) {
6249 // Zero out the operand and see if it becomes trivially dead.
6250 Operand = nullptr;
6251 if (isInstructionTriviallyDead(I: U))
6252 DeadInsts.push_back(Elt: U);
6253 }
6254
6255 ++NumDeleted;
6256 I->eraseFromParent();
6257 Changed = true;
6258 }
6259 return Changed;
6260}
6261/// Promote the allocas, using the best available technique.
6262///
6263/// This attempts to promote whatever allocas have been identified as viable in
6264/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6265/// This function returns whether any promotion occurred.
6266bool SROA::promoteAllocas() {
6267 if (PromotableAllocas.empty())
6268 return false;
6269
6270 if (SROASkipMem2Reg) {
6271 LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6272 } else {
6273 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6274 NumPromoted += PromotableAllocas.size();
6275 PromoteMemToReg(Allocas: PromotableAllocas.getArrayRef(), DT&: DTU->getDomTree(), AC);
6276 }
6277
6278 PromotableAllocas.clear();
6279 return true;
6280}
6281
6282std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
6283 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6284
6285 const DataLayout &DL = F.getDataLayout();
6286 BasicBlock &EntryBB = F.getEntryBlock();
6287 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(x: EntryBB.end());
6288 I != E; ++I) {
6289 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val&: I)) {
6290 std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6291 if (Size && Size->isScalable() && isAllocaPromotable(AI))
6292 PromotableAllocas.insert(X: AI);
6293 else
6294 Worklist.insert(X: AI);
6295 }
6296 }
6297
6298 bool Changed = false;
6299 bool CFGChanged = false;
6300 // A set of deleted alloca instruction pointers which should be removed from
6301 // the list of promotable allocas.
6302 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
6303
6304 do {
6305 while (!Worklist.empty()) {
6306 auto [IterationChanged, IterationCFGChanged] =
6307 runOnAlloca(AI&: *Worklist.pop_back_val());
6308 Changed |= IterationChanged;
6309 CFGChanged |= IterationCFGChanged;
6310
6311 Changed |= deleteDeadInstructions(DeletedAllocas);
6312
6313 // Remove the deleted allocas from various lists so that we don't try to
6314 // continue processing them.
6315 if (!DeletedAllocas.empty()) {
6316 Worklist.set_subtract(DeletedAllocas);
6317 PostPromotionWorklist.set_subtract(DeletedAllocas);
6318 PromotableAllocas.set_subtract(DeletedAllocas);
6319 DeletedAllocas.clear();
6320 }
6321 }
6322
6323 Changed |= promoteAllocas();
6324
6325 Worklist = PostPromotionWorklist;
6326 PostPromotionWorklist.clear();
6327 } while (!Worklist.empty());
6328
6329 assert((!CFGChanged || Changed) && "Can not only modify the CFG.");
6330 assert((!CFGChanged || !PreserveCFG) &&
6331 "Should not have modified the CFG when told to preserve it.");
6332
6333 if (Changed && isAssignmentTrackingEnabled(M: *F.getParent())) {
6334 for (auto &BB : F) {
6335 RemoveRedundantDbgInstrs(BB: &BB);
6336 }
6337 }
6338
6339 return {Changed, CFGChanged};
6340}
6341
6342PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
6343 DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
6344 AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
6345 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6346 auto [Changed, CFGChanged] =
6347 SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6348 if (!Changed)
6349 return PreservedAnalyses::all();
6350 PreservedAnalyses PA;
6351 if (!CFGChanged)
6352 PA.preserveSet<CFGAnalyses>();
6353 PA.preserve<DominatorTreeAnalysis>();
6354 return PA;
6355}
6356
6357void SROAPass::printPipeline(
6358 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6359 static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
6360 OS, MapClassName2PassName);
6361 OS << '<'
6362 << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6363 : "modify-cfg");
6364 if (Options.AggregateToVector)
6365 OS << ";aggregate-to-vector";
6366 OS << '>';
6367}
6368
6369SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
6370
6371namespace {
6372
6373/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6374class SROALegacyPass : public FunctionPass {
6375 SROAOptions Options;
6376
6377public:
6378 static char ID;
6379
6380 SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
6381 : FunctionPass(ID), Options(Options) {
6382 initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
6383 }
6384
6385 bool runOnFunction(Function &F) override {
6386 if (skipFunction(F))
6387 return false;
6388
6389 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6390 AssumptionCache &AC =
6391 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6392 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6393 auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6394 return Changed;
6395 }
6396
6397 void getAnalysisUsage(AnalysisUsage &AU) const override {
6398 AU.addRequired<AssumptionCacheTracker>();
6399 AU.addRequired<DominatorTreeWrapperPass>();
6400 AU.addPreserved<GlobalsAAWrapperPass>();
6401 AU.addPreserved<DominatorTreeWrapperPass>();
6402 }
6403
6404 StringRef getPassName() const override { return "SROA"; }
6405};
6406
6407} // end anonymous namespace
6408
6409char SROALegacyPass::ID = 0;
6410
6411FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
6412 return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
6413 : SROAOptions::ModifyCFG,
6414 AggregateToVector));
6415}
6416
6417INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6418 "Scalar Replacement Of Aggregates", false, false)
6419INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6420INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
6421INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
6422 false, false)
6423