1//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This transformation implements the well known scalar replacement of
10/// aggregates transformation. It tries to identify promotable elements of an
11/// aggregate alloca, and promote them to registers. It will also try to
12/// convert uses of an element (or set of elements) of an alloca into a vector
13/// or bitfield-style integer scalar if appropriate.
14///
15/// It works to do this with minimal slicing of the alloca so that regions
16/// which are merely transferred in and out of external memory remain unchanged
17/// and are not decomposed to scalar code.
18///
19/// Because this also performs alloca promotion, it can be thought of as also
20/// serving the purpose of SSA formation. The algorithm iterates on the
21/// function until all opportunities for promotion have been realized.
22///
23//===----------------------------------------------------------------------===//
24
25#include "llvm/Transforms/Scalar/SROA.h"
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/MapVector.h"
30#include "llvm/ADT/PointerIntPair.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SetVector.h"
33#include "llvm/ADT/SmallBitVector.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/ADT/Twine.h"
39#include "llvm/ADT/iterator.h"
40#include "llvm/ADT/iterator_range.h"
41#include "llvm/Analysis/AssumptionCache.h"
42#include "llvm/Analysis/DomTreeUpdater.h"
43#include "llvm/Analysis/GlobalsModRef.h"
44#include "llvm/Analysis/Loads.h"
45#include "llvm/Analysis/PtrUseVisitor.h"
46#include "llvm/Analysis/ValueTracking.h"
47#include "llvm/Analysis/VectorUtils.h"
48#include "llvm/IR/BasicBlock.h"
49#include "llvm/IR/Constant.h"
50#include "llvm/IR/ConstantFolder.h"
51#include "llvm/IR/Constants.h"
52#include "llvm/IR/DIBuilder.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfo.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalAlias.h"
60#include "llvm/IR/IRBuilder.h"
61#include "llvm/IR/InstVisitor.h"
62#include "llvm/IR/Instruction.h"
63#include "llvm/IR/Instructions.h"
64#include "llvm/IR/IntrinsicInst.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/Module.h"
68#include "llvm/IR/Operator.h"
69#include "llvm/IR/PassManager.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/InitializePasses.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/Debug.h"
81#include "llvm/Support/ErrorHandling.h"
82#include "llvm/Support/raw_ostream.h"
83#include "llvm/Transforms/Scalar.h"
84#include "llvm/Transforms/Utils/BasicBlockUtils.h"
85#include "llvm/Transforms/Utils/Local.h"
86#include "llvm/Transforms/Utils/PromoteMemToReg.h"
87#include "llvm/Transforms/Utils/SSAUpdater.h"
88#include <algorithm>
89#include <cassert>
90#include <cstddef>
91#include <cstdint>
92#include <cstring>
93#include <iterator>
94#include <string>
95#include <tuple>
96#include <utility>
97#include <variant>
98#include <vector>
99
100using namespace llvm;
101
102#define DEBUG_TYPE "sroa"
103
104STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112STATISTIC(NumLoadsPredicated,
113 "Number of loads rewritten into predicated loads to allow promotion");
114STATISTIC(
115 NumStoresPredicated,
116 "Number of stores rewritten into predicated loads to allow promotion");
117STATISTIC(NumDeleted, "Number of instructions deleted");
118STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120namespace llvm {
121/// Disable running mem2reg during SROA in order to test or debug SROA.
122static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(Val: false),
123 cl::Hidden);
124extern cl::opt<bool> ProfcheckDisableMetadataFixes;
125} // namespace llvm
126
127namespace {
128
129class AllocaSliceRewriter;
130class AllocaSlices;
131class Partition;
132
133class SelectHandSpeculativity {
134 unsigned char Storage = 0; // None are speculatable by default.
135 using TrueVal = Bitfield::Element<bool, 0, 1>; // Low 0'th bit.
136 using FalseVal = Bitfield::Element<bool, 1, 1>; // Low 1'th bit.
137public:
138 SelectHandSpeculativity() = default;
139 SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
140 bool isSpeculatable(bool isTrueVal) const;
141 bool areAllSpeculatable() const;
142 bool areAnySpeculatable() const;
143 bool areNoneSpeculatable() const;
144 // For interop as int half of PointerIntPair.
145 explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
146 explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
147};
148static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
149
150using PossiblySpeculatableLoad =
151 PointerIntPair<LoadInst *, 2, SelectHandSpeculativity>;
152using UnspeculatableStore = StoreInst *;
153using RewriteableMemOp =
154 std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
155using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
156
157/// An optimization pass providing Scalar Replacement of Aggregates.
158///
159/// This pass takes allocations which can be completely analyzed (that is, they
160/// don't escape) and tries to turn them into scalar SSA values. There are
161/// a few steps to this process.
162///
163/// 1) It takes allocations of aggregates and analyzes the ways in which they
164/// are used to try to split them into smaller allocations, ideally of
165/// a single scalar data type. It will split up memcpy and memset accesses
166/// as necessary and try to isolate individual scalar accesses.
167/// 2) It will transform accesses into forms which are suitable for SSA value
168/// promotion. This can be replacing a memset with a scalar store of an
169/// integer value, or it can involve speculating operations on a PHI or
170/// select to be a PHI or select of the results.
171/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
172/// onto insert and extract operations on a vector value, and convert them to
173/// this form. By doing so, it will enable promotion of vector aggregates to
174/// SSA vector values.
175class SROA {
176 LLVMContext *const C;
177 DomTreeUpdater *const DTU;
178 AssumptionCache *const AC;
179 const bool PreserveCFG;
180 const bool AggregateToVector;
181
182 /// Worklist of alloca instructions to simplify.
183 ///
184 /// Each alloca in the function is added to this. Each new alloca formed gets
185 /// added to it as well to recursively simplify unless that alloca can be
186 /// directly promoted. Finally, each time we rewrite a use of an alloca other
187 /// the one being actively rewritten, we add it back onto the list if not
188 /// already present to ensure it is re-visited.
189 SmallSetVector<AllocaInst *, 16> Worklist;
190
191 /// A collection of instructions to delete.
192 /// We try to batch deletions to simplify code and make things a bit more
193 /// efficient. We also make sure there is no dangling pointers.
194 SmallVector<WeakVH, 8> DeadInsts;
195
196 /// Post-promotion worklist.
197 ///
198 /// Sometimes we discover an alloca which has a high probability of becoming
199 /// viable for SROA after a round of promotion takes place. In those cases,
200 /// the alloca is enqueued here for re-processing.
201 ///
202 /// Note that we have to be very careful to clear allocas out of this list in
203 /// the event they are deleted.
204 SmallSetVector<AllocaInst *, 16> PostPromotionWorklist;
205
206 /// A collection of alloca instructions we can directly promote.
207 SetVector<AllocaInst *, SmallVector<AllocaInst *>,
208 SmallPtrSet<AllocaInst *, 16>, 16>
209 PromotableAllocas;
210
211 /// A worklist of PHIs to speculate prior to promoting allocas.
212 ///
213 /// All of these PHIs have been checked for the safety of speculation and by
214 /// being speculated will allow promoting allocas currently in the promotable
215 /// queue.
216 SmallSetVector<PHINode *, 8> SpeculatablePHIs;
217
218 /// A worklist of select instructions to rewrite prior to promoting
219 /// allocas.
220 SmallMapVector<SelectInst *, RewriteableMemOps, 8> SelectsToRewrite;
221
222 /// Select instructions that use an alloca and are subsequently loaded can be
223 /// rewritten to load both input pointers and then select between the result,
224 /// allowing the load of the alloca to be promoted.
225 /// From this:
226 /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other
227 /// %V = load <type>, ptr %P2
228 /// to:
229 /// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
230 /// %V2 = load <type>, ptr %Other
231 /// %V = select i1 %cond, <type> %V1, <type> %V2
232 ///
233 /// We can do this to a select if its only uses are loads
234 /// and if either the operand to the select can be loaded unconditionally,
235 /// or if we are allowed to perform CFG modifications.
236 /// If found an intervening bitcast with a single use of the load,
237 /// allow the promotion.
238 static std::optional<RewriteableMemOps>
239 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
240
241public:
242 SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
243 SROAOptions Options)
244 : C(C), DTU(DTU), AC(AC),
245 PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
246 AggregateToVector(Options.AggregateToVector) {}
247
248 /// Main run method used by both the SROAPass and by the legacy pass.
249 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
250
251private:
252 friend class AllocaSliceRewriter;
253
254 bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
255 std::pair<AllocaInst *, uint64_t>
256 rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
257 bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
258 bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
259 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runOnAlloca(AllocaInst &AI);
260 void clobberUse(Use &U);
261 bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
262 bool promoteAllocas();
263};
264
265} // end anonymous namespace
266
267/// Calculate the fragment of a variable to use when slicing a store
268/// based on the slice dimensions, existing fragment, and base storage
269/// fragment.
270/// Results:
271/// UseFrag - Use Target as the new fragment.
272/// UseNoFrag - The new slice already covers the whole variable.
273/// Skip - The new alloca slice doesn't include this variable.
274/// FIXME: Can we use calculateFragmentIntersect instead?
275namespace {
276enum FragCalcResult { UseFrag, UseNoFrag, Skip };
277}
278static FragCalcResult
279calculateFragment(DILocalVariable *Variable,
280 uint64_t NewStorageSliceOffsetInBits,
281 uint64_t NewStorageSliceSizeInBits,
282 std::optional<DIExpression::FragmentInfo> StorageFragment,
283 std::optional<DIExpression::FragmentInfo> CurrentFragment,
284 DIExpression::FragmentInfo &Target) {
285 // If the base storage describes part of the variable apply the offset and
286 // the size constraint.
287 if (StorageFragment) {
288 Target.SizeInBits =
289 std::min(a: NewStorageSliceSizeInBits, b: StorageFragment->SizeInBits);
290 Target.OffsetInBits =
291 NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits;
292 } else {
293 Target.SizeInBits = NewStorageSliceSizeInBits;
294 Target.OffsetInBits = NewStorageSliceOffsetInBits;
295 }
296
297 // If this slice extracts the entirety of an independent variable from a
298 // larger alloca, do not produce a fragment expression, as the variable is
299 // not fragmented.
300 if (!CurrentFragment) {
301 if (auto Size = Variable->getSizeInBits()) {
302 // Treat the current fragment as covering the whole variable.
303 CurrentFragment = DIExpression::FragmentInfo(*Size, 0);
304 if (Target == CurrentFragment)
305 return UseNoFrag;
306 }
307 }
308
309 // No additional work to do if there isn't a fragment already, or there is
310 // but it already exactly describes the new assignment.
311 if (!CurrentFragment || *CurrentFragment == Target)
312 return UseFrag;
313
314 // Reject the target fragment if it doesn't fit wholly within the current
315 // fragment. TODO: We could instead chop up the target to fit in the case of
316 // a partial overlap.
317 if (Target.startInBits() < CurrentFragment->startInBits() ||
318 Target.endInBits() > CurrentFragment->endInBits())
319 return Skip;
320
321 // Target fits within the current fragment, return it.
322 return UseFrag;
323}
324
325static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
326 return DebugVariable(DVR->getVariable(), std::nullopt,
327 DVR->getDebugLoc().getInlinedAt());
328}
329
330/// Find linked dbg.assign and generate a new one with the correct
331/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
332/// value component is copied from the old dbg.assign to the new.
333/// \param OldAlloca Alloca for the variable before splitting.
334/// \param IsSplit True if the store (not necessarily alloca)
335/// is being split.
336/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
337/// \param SliceSizeInBits New number of bits being written to.
338/// \param OldInst Instruction that is being split.
339/// \param Inst New instruction performing this part of the
340/// split store.
341/// \param Dest Store destination.
342/// \param Value Stored value.
343/// \param DL Datalayout.
344static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
345 uint64_t OldAllocaOffsetInBits,
346 uint64_t SliceSizeInBits, Instruction *OldInst,
347 Instruction *Inst, Value *Dest, Value *Value,
348 const DataLayout &DL) {
349 // If we want allocas to be migrated using this helper then we need to ensure
350 // that the BaseFragments map code still works. A simple solution would be
351 // to choose to always clone alloca dbg_assigns (rather than sometimes
352 // "stealing" them).
353 assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
354
355 auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(Inst: OldInst);
356 // Nothing to do if OldInst has no linked dbg.assign intrinsics.
357 if (DVRAssignMarkerRange.empty())
358 return;
359
360 LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
361 LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
362 LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
363 LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
364 << "\n");
365 LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
366 LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
367 LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
368 LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
369 if (Value)
370 LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
371
372 /// Map of aggregate variables to their fragment associated with OldAlloca.
373 DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>>
374 BaseFragments;
375 for (auto *DVR : at::getDVRAssignmentMarkers(Inst: OldAlloca))
376 BaseFragments[getAggregateVariable(DVR)] =
377 DVR->getExpression()->getFragmentInfo();
378
379 // The new inst needs a DIAssignID unique metadata tag (if OldInst has
380 // one). It shouldn't already have one: assert this assumption.
381 assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
382 DIAssignID *NewID = nullptr;
383 auto &Ctx = Inst->getContext();
384 DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
385 assert(OldAlloca->isStaticAlloca());
386
387 auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
388 LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
389 << "\n");
390 auto *Expr = DbgAssign->getExpression();
391 bool SetKillLocation = false;
392
393 if (IsSplit) {
394 std::optional<DIExpression::FragmentInfo> BaseFragment;
395 {
396 auto R = BaseFragments.find(Val: getAggregateVariable(DVR: DbgAssign));
397 if (R == BaseFragments.end())
398 return;
399 BaseFragment = R->second;
400 }
401 std::optional<DIExpression::FragmentInfo> CurrentFragment =
402 Expr->getFragmentInfo();
403 DIExpression::FragmentInfo NewFragment;
404 FragCalcResult Result = calculateFragment(
405 Variable: DbgAssign->getVariable(), NewStorageSliceOffsetInBits: OldAllocaOffsetInBits, NewStorageSliceSizeInBits: SliceSizeInBits,
406 StorageFragment: BaseFragment, CurrentFragment, Target&: NewFragment);
407
408 if (Result == Skip)
409 return;
410 if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
411 if (CurrentFragment) {
412 // Rewrite NewFragment to be relative to the existing one (this is
413 // what createFragmentExpression wants). CalculateFragment has
414 // already resolved the size for us. FIXME: Should it return the
415 // relative fragment too?
416 NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits;
417 }
418 // Add the new fragment info to the existing expression if possible.
419 if (auto E = DIExpression::createFragmentExpression(
420 Expr, OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits)) {
421 Expr = *E;
422 } else {
423 // Otherwise, add the new fragment info to an empty expression and
424 // discard the value component of this dbg.assign as the value cannot
425 // be computed with the new fragment.
426 Expr = *DIExpression::createFragmentExpression(
427 Expr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
428 OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits);
429 SetKillLocation = true;
430 }
431 }
432 }
433
434 // If we haven't created a DIAssignID ID do that now and attach it to Inst.
435 if (!NewID) {
436 NewID = DIAssignID::getDistinct(Context&: Ctx);
437 Inst->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: NewID);
438 }
439
440 DbgVariableRecord *NewAssign;
441 if (IsSplit) {
442 ::Value *NewValue = Value ? Value : DbgAssign->getValue();
443 NewAssign = cast<DbgVariableRecord>(Val: DIB.insertDbgAssign(
444 LinkedInstr: Inst, Val: NewValue, SrcVar: DbgAssign->getVariable(), ValExpr: Expr, Addr: Dest,
445 AddrExpr: DIExpression::get(Context&: Expr->getContext(), Elements: {}), DL: DbgAssign->getDebugLoc()));
446 } else {
447 // The store is not split, simply steal the existing dbg_assign.
448 NewAssign = DbgAssign;
449 NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
450 NewAssign->setAddress(Dest);
451 if (Value)
452 NewAssign->replaceVariableLocationOp(OpIdx: 0u, NewValue: Value);
453 assert(Expr == NewAssign->getExpression());
454 }
455
456 // If we've updated the value but the original dbg.assign has an arglist
457 // then kill it now - we can't use the requested new value.
458 // We can't replace the DIArgList with the new value as it'd leave
459 // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
460 // an arglist). And we can't keep the DIArgList in case the linked store
461 // is being split - in which case the DIArgList + expression may no longer
462 // be computing the correct value.
463 // This should be a very rare situation as it requires the value being
464 // stored to differ from the dbg.assign (i.e., the value has been
465 // represented differently in the debug intrinsic for some reason).
466 SetKillLocation |=
467 Value && (DbgAssign->hasArgList() ||
468 !DbgAssign->getExpression()->isSingleLocationExpression());
469 if (SetKillLocation)
470 NewAssign->setKillLocation();
471
472 // We could use more precision here at the cost of some additional (code)
473 // complexity - if the original dbg.assign was adjacent to its store, we
474 // could position this new dbg.assign adjacent to its store rather than the
475 // old dbg.assgn. That would result in interleaved dbg.assigns rather than
476 // what we get now:
477 // split store !1
478 // split store !2
479 // dbg.assign !1
480 // dbg.assign !2
481 // This (current behaviour) results results in debug assignments being
482 // noted as slightly offset (in code) from the store. In practice this
483 // should have little effect on the debugging experience due to the fact
484 // that all the split stores should get the same line number.
485 if (NewAssign != DbgAssign) {
486 NewAssign->moveBefore(MoveBefore: DbgAssign->getIterator());
487 NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
488 }
489 LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
490 };
491
492 for_each(Range&: DVRAssignMarkerRange, F: MigrateDbgAssign);
493}
494
495namespace {
496
497/// A custom IRBuilder inserter which prefixes all names, but only in
498/// Assert builds.
499class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
500 std::string Prefix;
501
502 Twine getNameWithPrefix(const Twine &Name) const {
503 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
504 }
505
506public:
507 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
508
509 void InsertHelper(Instruction *I, const Twine &Name,
510 BasicBlock::iterator InsertPt) const override {
511 IRBuilderDefaultInserter::InsertHelper(I, Name: getNameWithPrefix(Name),
512 InsertPt);
513 }
514};
515
516/// Provide a type for IRBuilder that drops names in release builds.
517using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
518
519/// A used slice of an alloca.
520///
521/// This structure represents a slice of an alloca used by some instruction. It
522/// stores both the begin and end offsets of this use, a pointer to the use
523/// itself, and a flag indicating whether we can classify the use as splittable
524/// or not when forming partitions of the alloca.
525class Slice {
526 /// The beginning offset of the range.
527 uint64_t BeginOffset = 0;
528
529 /// The ending offset, not included in the range.
530 uint64_t EndOffset = 0;
531
532 /// Storage for both the use of this slice and whether it can be
533 /// split.
534 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
535
536public:
537 Slice() = default;
538
539 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
540 : BeginOffset(BeginOffset), EndOffset(EndOffset),
541 UseAndIsSplittable(U, IsSplittable) {}
542
543 uint64_t beginOffset() const { return BeginOffset; }
544 uint64_t endOffset() const { return EndOffset; }
545
546 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
547 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
548
549 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
550
551 bool isDead() const { return getUse() == nullptr; }
552 void kill() { UseAndIsSplittable.setPointer(nullptr); }
553
554 /// Support for ordering ranges.
555 ///
556 /// This provides an ordering over ranges such that start offsets are
557 /// always increasing, and within equal start offsets, the end offsets are
558 /// decreasing. Thus the spanning range comes first in a cluster with the
559 /// same start position.
560 bool operator<(const Slice &RHS) const {
561 if (beginOffset() < RHS.beginOffset())
562 return true;
563 if (beginOffset() > RHS.beginOffset())
564 return false;
565 if (isSplittable() != RHS.isSplittable())
566 return !isSplittable();
567 if (endOffset() > RHS.endOffset())
568 return true;
569 return false;
570 }
571
572 /// Support comparison with a single offset to allow binary searches.
573 [[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
574 return LHS.beginOffset() < RHSOffset;
575 }
576 [[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
577 return LHSOffset < RHS.beginOffset();
578 }
579
580 bool operator==(const Slice &RHS) const {
581 return isSplittable() == RHS.isSplittable() &&
582 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
583 }
584 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
585};
586
587/// Representation of the alloca slices.
588///
589/// This class represents the slices of an alloca which are formed by its
590/// various uses. If a pointer escapes, we can't fully build a representation
591/// for the slices used and we reflect that in this structure. The uses are
592/// stored, sorted by increasing beginning offset and with unsplittable slices
593/// starting at a particular offset before splittable slices.
594class AllocaSlices {
595public:
596 /// Construct the slices of a particular alloca.
597 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
598
599 /// Test whether a pointer to the allocation escapes our analysis.
600 ///
601 /// If this is true, the slices are never fully built and should be
602 /// ignored.
603 bool isEscaped() const { return PointerEscapingInstr; }
604 bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
605
606 /// Support for iterating over the slices.
607 /// @{
608 using iterator = SmallVectorImpl<Slice>::iterator;
609 using range = iterator_range<iterator>;
610
611 iterator begin() { return Slices.begin(); }
612 iterator end() { return Slices.end(); }
613
614 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
615 using const_range = iterator_range<const_iterator>;
616
617 const_iterator begin() const { return Slices.begin(); }
618 const_iterator end() const { return Slices.end(); }
619 /// @}
620
621 /// Erase a range of slices.
622 void erase(iterator Start, iterator Stop) { Slices.erase(CS: Start, CE: Stop); }
623
624 /// Insert new slices for this alloca.
625 ///
626 /// This moves the slices into the alloca's slices collection, and re-sorts
627 /// everything so that the usual ordering properties of the alloca's slices
628 /// hold.
629 void insert(ArrayRef<Slice> NewSlices) {
630 int OldSize = Slices.size();
631 Slices.append(in_start: NewSlices.begin(), in_end: NewSlices.end());
632 auto SliceI = Slices.begin() + OldSize;
633 std::stable_sort(first: SliceI, last: Slices.end());
634 std::inplace_merge(first: Slices.begin(), middle: SliceI, last: Slices.end());
635 }
636
637 // Forward declare the iterator and range accessor for walking the
638 // partitions.
639 class partition_iterator;
640 iterator_range<partition_iterator> partitions();
641
642 /// Access the dead users for this alloca.
643 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
644
645 /// Access Uses that should be dropped if the alloca is promotable.
646 ArrayRef<Use *> getDeadUsesIfPromotable() const {
647 return DeadUseIfPromotable;
648 }
649
650 /// Access the dead operands referring to this alloca.
651 ///
652 /// These are operands which have cannot actually be used to refer to the
653 /// alloca as they are outside its range and the user doesn't correct for
654 /// that. These mostly consist of PHI node inputs and the like which we just
655 /// need to replace with undef.
656 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
657
658#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
659 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
660 void printSlice(raw_ostream &OS, const_iterator I,
661 StringRef Indent = " ") const;
662 void printUse(raw_ostream &OS, const_iterator I,
663 StringRef Indent = " ") const;
664 void print(raw_ostream &OS) const;
665 void dump(const_iterator I) const;
666 void dump() const;
667#endif
668
669private:
670 template <typename DerivedT, typename RetT = void> class BuilderBase;
671 class SliceBuilder;
672
673 friend class AllocaSlices::SliceBuilder;
674
675#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
676 /// Handle to alloca instruction to simplify method interfaces.
677 AllocaInst &AI;
678#endif
679
680 /// The instruction responsible for this alloca not having a known set
681 /// of slices.
682 ///
683 /// When an instruction (potentially) escapes the pointer to the alloca, we
684 /// store a pointer to that here and abort trying to form slices of the
685 /// alloca. This will be null if the alloca slices are analyzed successfully.
686 Instruction *PointerEscapingInstr;
687 Instruction *PointerEscapingInstrReadOnly;
688
689 /// The slices of the alloca.
690 ///
691 /// We store a vector of the slices formed by uses of the alloca here. This
692 /// vector is sorted by increasing begin offset, and then the unsplittable
693 /// slices before the splittable ones. See the Slice inner class for more
694 /// details.
695 SmallVector<Slice, 8> Slices;
696
697 /// Instructions which will become dead if we rewrite the alloca.
698 ///
699 /// Note that these are not separated by slice. This is because we expect an
700 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
701 /// all these instructions can simply be removed and replaced with poison as
702 /// they come from outside of the allocated space.
703 SmallVector<Instruction *, 8> DeadUsers;
704
705 /// Uses which will become dead if can promote the alloca.
706 SmallVector<Use *, 8> DeadUseIfPromotable;
707
708 /// Operands which will become dead if we rewrite the alloca.
709 ///
710 /// These are operands that in their particular use can be replaced with
711 /// poison when we rewrite the alloca. These show up in out-of-bounds inputs
712 /// to PHI nodes and the like. They aren't entirely dead (there might be
713 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
714 /// want to swap this particular input for poison to simplify the use lists of
715 /// the alloca.
716 SmallVector<Use *, 8> DeadOperands;
717};
718
719/// A partition of the slices.
720///
721/// An ephemeral representation for a range of slices which can be viewed as
722/// a partition of the alloca. This range represents a span of the alloca's
723/// memory which cannot be split, and provides access to all of the slices
724/// overlapping some part of the partition.
725///
726/// Objects of this type are produced by traversing the alloca's slices, but
727/// are only ephemeral and not persistent.
728class Partition {
729private:
730 friend class AllocaSlices;
731 friend class AllocaSlices::partition_iterator;
732
733 using iterator = AllocaSlices::iterator;
734
735 /// The beginning and ending offsets of the alloca for this
736 /// partition.
737 uint64_t BeginOffset = 0, EndOffset = 0;
738
739 /// The start and end iterators of this partition.
740 iterator SI, SJ;
741
742 /// A collection of split slice tails overlapping the partition.
743 SmallVector<Slice *, 4> SplitTails;
744
745 /// Raw constructor builds an empty partition starting and ending at
746 /// the given iterator.
747 Partition(iterator SI) : SI(SI), SJ(SI) {}
748
749public:
750 /// The start offset of this partition.
751 ///
752 /// All of the contained slices start at or after this offset.
753 uint64_t beginOffset() const { return BeginOffset; }
754
755 /// The end offset of this partition.
756 ///
757 /// All of the contained slices end at or before this offset.
758 uint64_t endOffset() const { return EndOffset; }
759
760 /// The size of the partition.
761 ///
762 /// Note that this can never be zero.
763 uint64_t size() const {
764 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
765 return EndOffset - BeginOffset;
766 }
767
768 /// Test whether this partition contains no slices, and merely spans
769 /// a region occupied by split slices.
770 bool empty() const { return SI == SJ; }
771
772 /// \name Iterate slices that start within the partition.
773 /// These may be splittable or unsplittable. They have a begin offset >= the
774 /// partition begin offset.
775 /// @{
776 // FIXME: We should probably define a "concat_iterator" helper and use that
777 // to stitch together pointee_iterators over the split tails and the
778 // contiguous iterators of the partition. That would give a much nicer
779 // interface here. We could then additionally expose filtered iterators for
780 // split, unsplit, and unsplittable splices based on the usage patterns.
781 iterator begin() const { return SI; }
782 iterator end() const { return SJ; }
783 /// @}
784
785 /// Get the sequence of split slice tails.
786 ///
787 /// These tails are of slices which start before this partition but are
788 /// split and overlap into the partition. We accumulate these while forming
789 /// partitions.
790 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
791};
792
793} // end anonymous namespace
794
795/// An iterator over partitions of the alloca's slices.
796///
797/// This iterator implements the core algorithm for partitioning the alloca's
798/// slices. It is a forward iterator as we don't support backtracking for
799/// efficiency reasons, and re-use a single storage area to maintain the
800/// current set of split slices.
801///
802/// It is templated on the slice iterator type to use so that it can operate
803/// with either const or non-const slice iterators.
804class AllocaSlices::partition_iterator
805 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
806 Partition> {
807 friend class AllocaSlices;
808
809 /// Most of the state for walking the partitions is held in a class
810 /// with a nice interface for examining them.
811 Partition P;
812
813 /// We need to keep the end of the slices to know when to stop.
814 AllocaSlices::iterator SE;
815
816 /// We also need to keep track of the maximum split end offset seen.
817 /// FIXME: Do we really?
818 uint64_t MaxSplitSliceEndOffset = 0;
819
820 /// Sets the partition to be empty at given iterator, and sets the
821 /// end iterator.
822 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
823 : P(SI), SE(SE) {
824 // If not already at the end, advance our state to form the initial
825 // partition.
826 if (SI != SE)
827 advance();
828 }
829
830 /// Advance the iterator to the next partition.
831 ///
832 /// Requires that the iterator not be at the end of the slices.
833 void advance() {
834 assert((P.SI != SE || !P.SplitTails.empty()) &&
835 "Cannot advance past the end of the slices!");
836
837 // Clear out any split uses which have ended.
838 if (!P.SplitTails.empty()) {
839 if (P.EndOffset >= MaxSplitSliceEndOffset) {
840 // If we've finished all splits, this is easy.
841 P.SplitTails.clear();
842 MaxSplitSliceEndOffset = 0;
843 } else {
844 // Remove the uses which have ended in the prior partition. This
845 // cannot change the max split slice end because we just checked that
846 // the prior partition ended prior to that max.
847 llvm::erase_if(C&: P.SplitTails,
848 P: [&](Slice *S) { return S->endOffset() <= P.EndOffset; });
849 assert(llvm::any_of(P.SplitTails,
850 [&](Slice *S) {
851 return S->endOffset() == MaxSplitSliceEndOffset;
852 }) &&
853 "Could not find the current max split slice offset!");
854 assert(llvm::all_of(P.SplitTails,
855 [&](Slice *S) {
856 return S->endOffset() <= MaxSplitSliceEndOffset;
857 }) &&
858 "Max split slice end offset is not actually the max!");
859 }
860 }
861
862 // If P.SI is already at the end, then we've cleared the split tail and
863 // now have an end iterator.
864 if (P.SI == SE) {
865 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
866 return;
867 }
868
869 // If we had a non-empty partition previously, set up the state for
870 // subsequent partitions.
871 if (P.SI != P.SJ) {
872 // Accumulate all the splittable slices which started in the old
873 // partition into the split list.
874 for (Slice &S : P)
875 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
876 P.SplitTails.push_back(Elt: &S);
877 MaxSplitSliceEndOffset =
878 std::max(a: S.endOffset(), b: MaxSplitSliceEndOffset);
879 }
880
881 // Start from the end of the previous partition.
882 P.SI = P.SJ;
883
884 // If P.SI is now at the end, we at most have a tail of split slices.
885 if (P.SI == SE) {
886 P.BeginOffset = P.EndOffset;
887 P.EndOffset = MaxSplitSliceEndOffset;
888 return;
889 }
890
891 // If the we have split slices and the next slice is after a gap and is
892 // not splittable immediately form an empty partition for the split
893 // slices up until the next slice begins.
894 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
895 !P.SI->isSplittable()) {
896 P.BeginOffset = P.EndOffset;
897 P.EndOffset = P.SI->beginOffset();
898 return;
899 }
900 }
901
902 // OK, we need to consume new slices. Set the end offset based on the
903 // current slice, and step SJ past it. The beginning offset of the
904 // partition is the beginning offset of the next slice unless we have
905 // pre-existing split slices that are continuing, in which case we begin
906 // at the prior end offset.
907 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
908 P.EndOffset = P.SI->endOffset();
909 ++P.SJ;
910
911 // There are two strategies to form a partition based on whether the
912 // partition starts with an unsplittable slice or a splittable slice.
913 if (!P.SI->isSplittable()) {
914 // When we're forming an unsplittable region, it must always start at
915 // the first slice and will extend through its end.
916 assert(P.BeginOffset == P.SI->beginOffset());
917
918 // Form a partition including all of the overlapping slices with this
919 // unsplittable slice.
920 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
921 if (!P.SJ->isSplittable())
922 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
923 ++P.SJ;
924 }
925
926 // We have a partition across a set of overlapping unsplittable
927 // partitions.
928 return;
929 }
930
931 // If we're starting with a splittable slice, then we need to form
932 // a synthetic partition spanning it and any other overlapping splittable
933 // splices.
934 assert(P.SI->isSplittable() && "Forming a splittable partition!");
935
936 // Collect all of the overlapping splittable slices.
937 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
938 P.SJ->isSplittable()) {
939 P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
940 ++P.SJ;
941 }
942
943 // Back upiP.EndOffset if we ended the span early when encountering an
944 // unsplittable slice. This synthesizes the early end offset of
945 // a partition spanning only splittable slices.
946 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
947 assert(!P.SJ->isSplittable());
948 P.EndOffset = P.SJ->beginOffset();
949 }
950 }
951
952public:
953 bool operator==(const partition_iterator &RHS) const {
954 assert(SE == RHS.SE &&
955 "End iterators don't match between compared partition iterators!");
956
957 // The observed positions of partitions is marked by the P.SI iterator and
958 // the emptiness of the split slices. The latter is only relevant when
959 // P.SI == SE, as the end iterator will additionally have an empty split
960 // slices list, but the prior may have the same P.SI and a tail of split
961 // slices.
962 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
963 assert(P.SJ == RHS.P.SJ &&
964 "Same set of slices formed two different sized partitions!");
965 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
966 "Same slice position with differently sized non-empty split "
967 "slice tails!");
968 return true;
969 }
970 return false;
971 }
972
973 partition_iterator &operator++() {
974 advance();
975 return *this;
976 }
977
978 Partition &operator*() { return P; }
979};
980
981/// A forward range over the partitions of the alloca's slices.
982///
983/// This accesses an iterator range over the partitions of the alloca's
984/// slices. It computes these partitions on the fly based on the overlapping
985/// offsets of the slices and the ability to split them. It will visit "empty"
986/// partitions to cover regions of the alloca only accessed via split
987/// slices.
988iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
989 return make_range(x: partition_iterator(begin(), end()),
990 y: partition_iterator(end(), end()));
991}
992
993static Value *foldSelectInst(SelectInst &SI) {
994 // If the condition being selected on is a constant or the same value is
995 // being selected between, fold the select. Yes this does (rarely) happen
996 // early on.
997 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: SI.getCondition()))
998 return SI.getOperand(i_nocapture: 1 + CI->isZero());
999 if (SI.getOperand(i_nocapture: 1) == SI.getOperand(i_nocapture: 2))
1000 return SI.getOperand(i_nocapture: 1);
1001
1002 return nullptr;
1003}
1004
1005/// A helper that folds a PHI node or a select.
1006static Value *foldPHINodeOrSelectInst(Instruction &I) {
1007 if (PHINode *PN = dyn_cast<PHINode>(Val: &I)) {
1008 // If PN merges together the same value, return that value.
1009 return PN->hasConstantValue();
1010 }
1011 return foldSelectInst(SI&: cast<SelectInst>(Val&: I));
1012}
1013
1014/// Builder for the alloca slices.
1015///
1016/// This class builds a set of alloca slices by recursively visiting the uses
1017/// of an alloca and making a slice for each load and store at each offset.
1018class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1019 friend class PtrUseVisitor<SliceBuilder>;
1020 friend class InstVisitor<SliceBuilder>;
1021
1022 using Base = PtrUseVisitor<SliceBuilder>;
1023
1024 const uint64_t AllocSize;
1025 AllocaSlices &AS;
1026
1027 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
1028 SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
1029
1030 /// Set to de-duplicate dead instructions found in the use walk.
1031 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
1032
1033public:
1034 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1035 : PtrUseVisitor<SliceBuilder>(DL),
1036 AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1037
1038private:
1039 void markAsDead(Instruction &I) {
1040 if (VisitedDeadInsts.insert(Ptr: &I).second)
1041 AS.DeadUsers.push_back(Elt: &I);
1042 }
1043
1044 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1045 bool IsSplittable = false) {
1046 // Completely skip uses which have a zero size or start either before or
1047 // past the end of the allocation.
1048 if (Size == 0 || Offset.uge(RHS: AllocSize)) {
1049 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1050 << Offset
1051 << " which has zero size or starts outside of the "
1052 << AllocSize << " byte alloca:\n"
1053 << " alloca: " << AS.AI << "\n"
1054 << " use: " << I << "\n");
1055 return markAsDead(I);
1056 }
1057
1058 uint64_t BeginOffset = Offset.getZExtValue();
1059 uint64_t EndOffset = BeginOffset + Size;
1060
1061 // Clamp the end offset to the end of the allocation. Note that this is
1062 // formulated to handle even the case where "BeginOffset + Size" overflows.
1063 // This may appear superficially to be something we could ignore entirely,
1064 // but that is not so! There may be widened loads or PHI-node uses where
1065 // some instructions are dead but not others. We can't completely ignore
1066 // them, and so have to record at least the information here.
1067 assert(AllocSize >= BeginOffset); // Established above.
1068 if (Size > AllocSize - BeginOffset) {
1069 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1070 << Offset << " to remain within the " << AllocSize
1071 << " byte alloca:\n"
1072 << " alloca: " << AS.AI << "\n"
1073 << " use: " << I << "\n");
1074 EndOffset = AllocSize;
1075 }
1076
1077 AS.Slices.push_back(Elt: Slice(BeginOffset, EndOffset, U, IsSplittable));
1078 }
1079
1080 void visitBitCastInst(BitCastInst &BC) {
1081 if (BC.use_empty())
1082 return markAsDead(I&: BC);
1083
1084 return Base::visitBitCastInst(BC);
1085 }
1086
1087 void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1088 if (ASC.use_empty())
1089 return markAsDead(I&: ASC);
1090
1091 return Base::visitAddrSpaceCastInst(ASC);
1092 }
1093
1094 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1095 if (GEPI.use_empty())
1096 return markAsDead(I&: GEPI);
1097
1098 return Base::visitGetElementPtrInst(GEPI);
1099 }
1100
1101 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
1102 uint64_t Size, bool IsVolatile) {
1103 // We allow splitting of non-volatile loads and stores where the type is an
1104 // integer type. These may be used to implement 'memcpy' or other "transfer
1105 // of bits" patterns.
1106 bool IsSplittable =
1107 Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1108
1109 insertUse(I, Offset, Size, IsSplittable);
1110 }
1111
1112 void visitLoadInst(LoadInst &LI) {
1113 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
1114 "All simple FCA loads should have been pre-split");
1115
1116 // If there is a load with an unknown offset, we can still perform store
1117 // to load forwarding for other known-offset loads.
1118 if (!IsOffsetKnown)
1119 return PI.setEscapedReadOnly(&LI);
1120
1121 TypeSize Size = DL.getTypeStoreSize(Ty: LI.getType());
1122 if (Size.isScalable()) {
1123 unsigned VScale = LI.getFunction()->getVScaleValue();
1124 if (!VScale)
1125 return PI.setAborted(&LI);
1126
1127 Size = TypeSize::getFixed(ExactSize: Size.getKnownMinValue() * VScale);
1128 }
1129
1130 return handleLoadOrStore(Ty: LI.getType(), I&: LI, Offset, Size: Size.getFixedValue(),
1131 IsVolatile: LI.isVolatile());
1132 }
1133
1134 void visitStoreInst(StoreInst &SI) {
1135 Value *ValOp = SI.getValueOperand();
1136 if (ValOp == *U)
1137 return PI.setEscapedAndAborted(&SI);
1138 if (!IsOffsetKnown)
1139 return PI.setAborted(&SI);
1140
1141 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValOp->getType());
1142 if (StoreSize.isScalable()) {
1143 unsigned VScale = SI.getFunction()->getVScaleValue();
1144 if (!VScale)
1145 return PI.setAborted(&SI);
1146
1147 StoreSize = TypeSize::getFixed(ExactSize: StoreSize.getKnownMinValue() * VScale);
1148 }
1149
1150 uint64_t Size = StoreSize.getFixedValue();
1151
1152 // If this memory access can be shown to *statically* extend outside the
1153 // bounds of the allocation, it's behavior is undefined, so simply
1154 // ignore it. Note that this is more strict than the generic clamping
1155 // behavior of insertUse. We also try to handle cases which might run the
1156 // risk of overflow.
1157 // FIXME: We should instead consider the pointer to have escaped if this
1158 // function is being instrumented for addressing bugs or race conditions.
1159 if (Size > AllocSize || Offset.ugt(RHS: AllocSize - Size)) {
1160 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1161 << Offset << " which extends past the end of the "
1162 << AllocSize << " byte alloca:\n"
1163 << " alloca: " << AS.AI << "\n"
1164 << " use: " << SI << "\n");
1165 return markAsDead(I&: SI);
1166 }
1167
1168 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1169 "All simple FCA stores should have been pre-split");
1170 handleLoadOrStore(Ty: ValOp->getType(), I&: SI, Offset, Size, IsVolatile: SI.isVolatile());
1171 }
1172
1173 void visitMemSetInst(MemSetInst &II) {
1174 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1175 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1176 if ((Length && Length->getValue() == 0) ||
1177 (IsOffsetKnown && Offset.uge(RHS: AllocSize)))
1178 // Zero-length mem transfer intrinsics can be ignored entirely.
1179 return markAsDead(I&: II);
1180
1181 if (!IsOffsetKnown)
1182 return PI.setAborted(&II);
1183
1184 insertUse(I&: II, Offset,
1185 Size: Length ? Length->getLimitedValue()
1186 : AllocSize - Offset.getLimitedValue(),
1187 IsSplittable: (bool)Length);
1188 }
1189
1190 void visitMemTransferInst(MemTransferInst &II) {
1191 ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1192 if (Length && Length->getValue() == 0)
1193 // Zero-length mem transfer intrinsics can be ignored entirely.
1194 return markAsDead(I&: II);
1195
1196 // Because we can visit these intrinsics twice, also check to see if the
1197 // first time marked this instruction as dead. If so, skip it.
1198 if (VisitedDeadInsts.count(Ptr: &II))
1199 return;
1200
1201 if (!IsOffsetKnown)
1202 return PI.setAborted(&II);
1203
1204 // This side of the transfer is completely out-of-bounds, and so we can
1205 // nuke the entire transfer. However, we also need to nuke the other side
1206 // if already added to our partitions.
1207 // FIXME: Yet another place we really should bypass this when
1208 // instrumenting for ASan.
1209 if (Offset.uge(RHS: AllocSize)) {
1210 auto MTPI = MemTransferSliceMap.find(Val: &II);
1211 if (MTPI != MemTransferSliceMap.end())
1212 AS.Slices[MTPI->second].kill();
1213 return markAsDead(I&: II);
1214 }
1215
1216 uint64_t RawOffset = Offset.getLimitedValue();
1217 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1218
1219 // Check for the special case where the same exact value is used for both
1220 // source and dest.
1221 if (*U == II.getRawDest() && *U == II.getRawSource()) {
1222 // For non-volatile transfers this is a no-op.
1223 if (!II.isVolatile())
1224 return markAsDead(I&: II);
1225
1226 return insertUse(I&: II, Offset, Size, /*IsSplittable=*/false);
1227 }
1228
1229 // If we have seen both source and destination for a mem transfer, then
1230 // they both point to the same alloca.
1231 bool Inserted;
1232 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
1233 std::tie(args&: MTPI, args&: Inserted) =
1234 MemTransferSliceMap.insert(KV: std::make_pair(x: &II, y: AS.Slices.size()));
1235 unsigned PrevIdx = MTPI->second;
1236 if (!Inserted) {
1237 Slice &PrevP = AS.Slices[PrevIdx];
1238
1239 // Check if the begin offsets match and this is a non-volatile transfer.
1240 // In that case, we can completely elide the transfer.
1241 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1242 PrevP.kill();
1243 return markAsDead(I&: II);
1244 }
1245
1246 // Otherwise we have an offset transfer within the same alloca. We can't
1247 // split those.
1248 PrevP.makeUnsplittable();
1249 }
1250
1251 // Insert the use now that we've fixed up the splittable nature.
1252 insertUse(I&: II, Offset, Size, /*IsSplittable=*/Inserted && Length);
1253
1254 // Check that we ended up with a valid index in the map.
1255 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1256 "Map index doesn't point back to a slice with this user.");
1257 }
1258
1259 // Disable SRoA for any intrinsics except for lifetime invariants.
1260 // FIXME: What about debug intrinsics? This matches old behavior, but
1261 // doesn't make sense.
1262 void visitIntrinsicInst(IntrinsicInst &II) {
1263 if (II.isDroppable()) {
1264 AS.DeadUseIfPromotable.push_back(Elt: U);
1265 return;
1266 }
1267
1268 if (!IsOffsetKnown)
1269 return PI.setAborted(&II);
1270
1271 if (II.isLifetimeStartOrEnd()) {
1272 insertUse(I&: II, Offset, Size: AllocSize, IsSplittable: true);
1273 return;
1274 }
1275
1276 Base::visitIntrinsicInst(II);
1277 }
1278
1279 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
1280 // We consider any PHI or select that results in a direct load or store of
1281 // the same offset to be a viable use for slicing purposes. These uses
1282 // are considered unsplittable and the size is the maximum loaded or stored
1283 // size.
1284 SmallPtrSet<Instruction *, 4> Visited;
1285 SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses;
1286 Visited.insert(Ptr: Root);
1287 Uses.push_back(Elt: std::make_pair(x: cast<Instruction>(Val&: *U), y&: Root));
1288 const DataLayout &DL = Root->getDataLayout();
1289 // If there are no loads or stores, the access is dead. We mark that as
1290 // a size zero access.
1291 Size = 0;
1292 do {
1293 Instruction *I, *UsedI;
1294 std::tie(args&: UsedI, args&: I) = Uses.pop_back_val();
1295
1296 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1297 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
1298 if (LoadSize.isScalable()) {
1299 PI.setAborted(LI);
1300 return nullptr;
1301 }
1302 Size = std::max(a: Size, b: LoadSize.getFixedValue());
1303 continue;
1304 }
1305 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1306 Value *Op = SI->getOperand(i_nocapture: 0);
1307 if (Op == UsedI)
1308 return SI;
1309 TypeSize StoreSize = DL.getTypeStoreSize(Ty: Op->getType());
1310 if (StoreSize.isScalable()) {
1311 PI.setAborted(SI);
1312 return nullptr;
1313 }
1314 Size = std::max(a: Size, b: StoreSize.getFixedValue());
1315 continue;
1316 }
1317
1318 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1319 if (!GEP->hasAllZeroIndices())
1320 return GEP;
1321 } else if (!isa<BitCastInst>(Val: I) && !isa<PHINode>(Val: I) &&
1322 !isa<SelectInst>(Val: I) && !isa<AddrSpaceCastInst>(Val: I)) {
1323 return I;
1324 }
1325
1326 for (User *U : I->users())
1327 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
1328 Uses.push_back(Elt: std::make_pair(x&: I, y: cast<Instruction>(Val: U)));
1329 } while (!Uses.empty());
1330
1331 return nullptr;
1332 }
1333
1334 void visitPHINodeOrSelectInst(Instruction &I) {
1335 assert(isa<PHINode>(I) || isa<SelectInst>(I));
1336 if (I.use_empty())
1337 return markAsDead(I);
1338
1339 // If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1340 // instructions in this BB, which may be required during rewriting. Bail out
1341 // on these cases.
1342 if (isa<PHINode>(Val: I) && !I.getParent()->hasInsertionPt())
1343 return PI.setAborted(&I);
1344
1345 // TODO: We could use simplifyInstruction here to fold PHINodes and
1346 // SelectInsts. However, doing so requires to change the current
1347 // dead-operand-tracking mechanism. For instance, suppose neither loading
1348 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1349 // trap either. However, if we simply replace %U with undef using the
1350 // current dead-operand-tracking mechanism, "load (select undef, undef,
1351 // %other)" may trap because the select may return the first operand
1352 // "undef".
1353 if (Value *Result = foldPHINodeOrSelectInst(I)) {
1354 if (Result == *U)
1355 // If the result of the constant fold will be the pointer, recurse
1356 // through the PHI/select as if we had RAUW'ed it.
1357 enqueueUsers(I);
1358 else
1359 // Otherwise the operand to the PHI/select is dead, and we can replace
1360 // it with poison.
1361 AS.DeadOperands.push_back(Elt: U);
1362
1363 return;
1364 }
1365
1366 if (!IsOffsetKnown)
1367 return PI.setAborted(&I);
1368
1369 // See if we already have computed info on this node.
1370 uint64_t &Size = PHIOrSelectSizes[&I];
1371 if (!Size) {
1372 // This is a new PHI/Select, check for an unsafe use of it.
1373 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(Root: &I, Size))
1374 return PI.setAborted(UnsafeI);
1375 }
1376
1377 // For PHI and select operands outside the alloca, we can't nuke the entire
1378 // phi or select -- the other side might still be relevant, so we special
1379 // case them here and use a separate structure to track the operands
1380 // themselves which should be replaced with poison.
1381 // FIXME: This should instead be escaped in the event we're instrumenting
1382 // for address sanitization.
1383 if (Offset.uge(RHS: AllocSize)) {
1384 AS.DeadOperands.push_back(Elt: U);
1385 return;
1386 }
1387
1388 insertUse(I, Offset, Size);
1389 }
1390
1391 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(I&: PN); }
1392
1393 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(I&: SI); }
1394
1395 /// Disable SROA entirely if there are unhandled users of the alloca.
1396 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1397
1398 void visitCallBase(CallBase &CB) {
1399 // If the call operand is read-only and only does a read-only or address
1400 // capture, then we mark it as EscapedReadOnly.
1401 if (CB.isDataOperand(U) &&
1402 !capturesFullProvenance(CC: CB.getCaptureInfo(OpNo: U->getOperandNo())) &&
1403 CB.onlyReadsMemory(OpNo: U->getOperandNo())) {
1404 PI.setEscapedReadOnly(&CB);
1405 return;
1406 }
1407
1408 Base::visitCallBase(CB);
1409 }
1410};
1411
1412AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1413 :
1414#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1415 AI(AI),
1416#endif
1417 PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1418 SliceBuilder PB(DL, AI, *this);
1419 SliceBuilder::PtrInfo PtrI = PB.visitPtr(I&: AI);
1420 if (PtrI.isEscaped() || PtrI.isAborted()) {
1421 // FIXME: We should sink the escape vs. abort info into the caller nicely,
1422 // possibly by just storing the PtrInfo in the AllocaSlices.
1423 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1424 : PtrI.getAbortingInst();
1425 assert(PointerEscapingInstr && "Did not track a bad instruction");
1426 return;
1427 }
1428 PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1429
1430 llvm::erase_if(C&: Slices, P: [](const Slice &S) { return S.isDead(); });
1431
1432 // Sort the uses. This arranges for the offsets to be in ascending order,
1433 // and the sizes to be in descending order.
1434 llvm::stable_sort(Range&: Slices);
1435}
1436
1437#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1438
1439void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1440 StringRef Indent) const {
1441 printSlice(OS, I, Indent);
1442 OS << "\n";
1443 printUse(OS, I, Indent);
1444}
1445
1446void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1447 StringRef Indent) const {
1448 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1449 << " slice #" << (I - begin())
1450 << (I->isSplittable() ? " (splittable)" : "");
1451}
1452
1453void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1454 StringRef Indent) const {
1455 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1456}
1457
1458void AllocaSlices::print(raw_ostream &OS) const {
1459 if (PointerEscapingInstr) {
1460 OS << "Can't analyze slices for alloca: " << AI << "\n"
1461 << " A pointer to this alloca escaped by:\n"
1462 << " " << *PointerEscapingInstr << "\n";
1463 return;
1464 }
1465
1466 if (PointerEscapingInstrReadOnly)
1467 OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1468
1469 OS << "Slices of alloca: " << AI << "\n";
1470 for (const_iterator I = begin(), E = end(); I != E; ++I)
1471 print(OS, I);
1472}
1473
1474LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1475 print(dbgs(), I);
1476}
1477LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1478
1479#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1480
1481/// Walk the range of a partitioning looking for a common type to cover this
1482/// sequence of slices.
1483static std::pair<Type *, IntegerType *>
1484findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1485 uint64_t EndOffset) {
1486 Type *Ty = nullptr;
1487 bool TyIsCommon = true;
1488 IntegerType *ITy = nullptr;
1489
1490 // Note that we need to look at *every* alloca slice's Use to ensure we
1491 // always get consistent results regardless of the order of slices.
1492 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1493 Use *U = I->getUse();
1494 if (isa<IntrinsicInst>(Val: *U->getUser()))
1495 continue;
1496 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1497 continue;
1498
1499 Type *UserTy = nullptr;
1500 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
1501 UserTy = LI->getType();
1502 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
1503 UserTy = SI->getValueOperand()->getType();
1504 }
1505
1506 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(Val: UserTy)) {
1507 // If the type is larger than the partition, skip it. We only encounter
1508 // this for split integer operations where we want to use the type of the
1509 // entity causing the split. Also skip if the type is not a byte width
1510 // multiple.
1511 if (UserITy->getBitWidth() % 8 != 0 ||
1512 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
1513 continue;
1514
1515 // Track the largest bitwidth integer type used in this way in case there
1516 // is no common type.
1517 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1518 ITy = UserITy;
1519 }
1520
1521 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1522 // depend on types skipped above.
1523 if (!UserTy || (Ty && Ty != UserTy))
1524 TyIsCommon = false; // Give up on anything but an iN type.
1525 else
1526 Ty = UserTy;
1527 }
1528
1529 return {TyIsCommon ? Ty : nullptr, ITy};
1530}
1531
1532/// PHI instructions that use an alloca and are subsequently loaded can be
1533/// rewritten to load both input pointers in the pred blocks and then PHI the
1534/// results, allowing the load of the alloca to be promoted.
1535/// From this:
1536/// %P2 = phi [i32* %Alloca, i32* %Other]
1537/// %V = load i32* %P2
1538/// to:
1539/// %V1 = load i32* %Alloca -> will be mem2reg'd
1540/// ...
1541/// %V2 = load i32* %Other
1542/// ...
1543/// %V = phi [i32 %V1, i32 %V2]
1544///
1545/// We can do this to a select if its only uses are loads and if the operands
1546/// to the select can be loaded unconditionally.
1547///
1548/// FIXME: This should be hoisted into a generic utility, likely in
1549/// Transforms/Util/Local.h
1550static bool isSafePHIToSpeculate(PHINode &PN) {
1551 const DataLayout &DL = PN.getDataLayout();
1552
1553 // For now, we can only do this promotion if the load is in the same block
1554 // as the PHI, and if there are no stores between the phi and load.
1555 // TODO: Allow recursive phi users.
1556 // TODO: Allow stores.
1557 BasicBlock *BB = PN.getParent();
1558 Align MaxAlign;
1559 uint64_t APWidth = DL.getIndexTypeSizeInBits(Ty: PN.getType());
1560 Type *LoadType = nullptr;
1561 for (User *U : PN.users()) {
1562 LoadInst *LI = dyn_cast<LoadInst>(Val: U);
1563 if (!LI || !LI->isSimple())
1564 return false;
1565
1566 // For now we only allow loads in the same block as the PHI. This is
1567 // a common case that happens when instcombine merges two loads through
1568 // a PHI.
1569 if (LI->getParent() != BB)
1570 return false;
1571
1572 if (LoadType) {
1573 if (LoadType != LI->getType())
1574 return false;
1575 } else {
1576 LoadType = LI->getType();
1577 }
1578
1579 // Ensure that there are no instructions between the PHI and the load that
1580 // could store.
1581 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1582 if (BBI->mayWriteToMemory())
1583 return false;
1584
1585 MaxAlign = std::max(a: MaxAlign, b: LI->getAlign());
1586 }
1587
1588 if (!LoadType)
1589 return false;
1590
1591 APInt LoadSize =
1592 APInt(APWidth, DL.getTypeStoreSize(Ty: LoadType).getFixedValue());
1593
1594 // We can only transform this if it is safe to push the loads into the
1595 // predecessor blocks. The only thing to watch out for is that we can't put
1596 // a possibly trapping load in the predecessor if it is a critical edge.
1597 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1598 Instruction *TI = PN.getIncomingBlock(i: Idx)->getTerminator();
1599 Value *InVal = PN.getIncomingValue(i: Idx);
1600
1601 // If the value is produced by the terminator of the predecessor (an
1602 // invoke) or it has side-effects, there is no valid place to put a load
1603 // in the predecessor.
1604 if (TI == InVal || TI->mayHaveSideEffects())
1605 return false;
1606
1607 // If the predecessor has a single successor, then the edge isn't
1608 // critical.
1609 if (TI->getNumSuccessors() == 1)
1610 continue;
1611
1612 // If this pointer is always safe to load, or if we can prove that there
1613 // is already a load in the block, then we can move the load to the pred
1614 // block.
1615 if (isSafeToLoadUnconditionally(V: InVal, Alignment: MaxAlign, Size: LoadSize,
1616 SQ: SimplifyQuery(DL, TI)))
1617 continue;
1618
1619 return false;
1620 }
1621
1622 return true;
1623}
1624
1625static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1626 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1627
1628 LoadInst *SomeLoad = cast<LoadInst>(Val: PN.user_back());
1629 Type *LoadTy = SomeLoad->getType();
1630 IRB.SetInsertPoint(&PN);
1631 PHINode *NewPN = IRB.CreatePHI(Ty: LoadTy, NumReservedValues: PN.getNumIncomingValues(),
1632 Name: PN.getName() + ".sroa.speculated");
1633
1634 // Get the AA tags and alignment to use from one of the loads. It does not
1635 // matter which one we get and if any differ.
1636 AAMDNodes AATags = SomeLoad->getAAMetadata();
1637 Align Alignment = SomeLoad->getAlign();
1638
1639 // Rewrite all loads of the PN to use the new PHI.
1640 while (!PN.use_empty()) {
1641 LoadInst *LI = cast<LoadInst>(Val: PN.user_back());
1642 LI->replaceAllUsesWith(V: NewPN);
1643 LI->eraseFromParent();
1644 }
1645
1646 // Inject loads into all of the pred blocks.
1647 DenseMap<BasicBlock *, Value *> InjectedLoads;
1648 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1649 BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1650 Value *InVal = PN.getIncomingValue(i: Idx);
1651
1652 // A PHI node is allowed to have multiple (duplicated) entries for the same
1653 // basic block, as long as the value is the same. So if we already injected
1654 // a load in the predecessor, then we should reuse the same load for all
1655 // duplicated entries.
1656 if (Value *V = InjectedLoads.lookup(Val: Pred)) {
1657 NewPN->addIncoming(V, BB: Pred);
1658 continue;
1659 }
1660
1661 Instruction *TI = Pred->getTerminator();
1662 IRB.SetInsertPoint(TI);
1663
1664 LoadInst *Load = IRB.CreateAlignedLoad(
1665 Ty: LoadTy, Ptr: InVal, Align: Alignment,
1666 Name: (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1667 ++NumLoadsSpeculated;
1668 if (AATags)
1669 Load->setAAMetadata(AATags);
1670 NewPN->addIncoming(V: Load, BB: Pred);
1671 InjectedLoads[Pred] = Load;
1672 }
1673
1674 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1675 PN.eraseFromParent();
1676}
1677
1678SelectHandSpeculativity &
1679SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1680 if (isTrueVal)
1681 Bitfield::set<SelectHandSpeculativity::TrueVal>(Packed&: Storage, Value: true);
1682 else
1683 Bitfield::set<SelectHandSpeculativity::FalseVal>(Packed&: Storage, Value: true);
1684 return *this;
1685}
1686
1687bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1688 return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Packed: Storage)
1689 : Bitfield::get<SelectHandSpeculativity::FalseVal>(Packed: Storage);
1690}
1691
1692bool SelectHandSpeculativity::areAllSpeculatable() const {
1693 return isSpeculatable(/*isTrueVal=*/true) &&
1694 isSpeculatable(/*isTrueVal=*/false);
1695}
1696
1697bool SelectHandSpeculativity::areAnySpeculatable() const {
1698 return isSpeculatable(/*isTrueVal=*/true) ||
1699 isSpeculatable(/*isTrueVal=*/false);
1700}
1701bool SelectHandSpeculativity::areNoneSpeculatable() const {
1702 return !areAnySpeculatable();
1703}
1704
1705static SelectHandSpeculativity
1706isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1707 assert(LI.isSimple() && "Only for simple loads");
1708 SelectHandSpeculativity Spec;
1709
1710 const DataLayout &DL = SI.getDataLayout();
1711 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1712 if (isSafeToLoadUnconditionally(V: Value, Ty: LI.getType(), Alignment: LI.getAlign(),
1713 SQ: SimplifyQuery(DL, &LI)))
1714 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1715 else if (PreserveCFG)
1716 return Spec;
1717
1718 return Spec;
1719}
1720
1721std::optional<RewriteableMemOps>
1722SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1723 RewriteableMemOps Ops;
1724
1725 for (User *U : SI.users()) {
1726 if (auto *BC = dyn_cast<BitCastInst>(Val: U); BC && BC->hasOneUse())
1727 U = *BC->user_begin();
1728
1729 if (auto *Store = dyn_cast<StoreInst>(Val: U)) {
1730 // Note that atomic stores can be transformed; atomic semantics do not
1731 // have any meaning for a local alloca. Stores are not speculatable,
1732 // however, so if we can't turn it into a predicated store, we are done.
1733 if (Store->isVolatile() || PreserveCFG)
1734 return {}; // Give up on this `select`.
1735 Ops.emplace_back(Args&: Store);
1736 continue;
1737 }
1738
1739 auto *LI = dyn_cast<LoadInst>(Val: U);
1740
1741 // Note that atomic loads can be transformed;
1742 // atomic semantics do not have any meaning for a local alloca.
1743 if (!LI || LI->isVolatile())
1744 return {}; // Give up on this `select`.
1745
1746 PossiblySpeculatableLoad Load(LI);
1747 if (!LI->isSimple()) {
1748 // If the `load` is not simple, we can't speculatively execute it,
1749 // but we could handle this via a CFG modification. But can we?
1750 if (PreserveCFG)
1751 return {}; // Give up on this `select`.
1752 Ops.emplace_back(Args&: Load);
1753 continue;
1754 }
1755
1756 SelectHandSpeculativity Spec =
1757 isSafeLoadOfSelectToSpeculate(LI&: *LI, SI, PreserveCFG);
1758 if (PreserveCFG && !Spec.areAllSpeculatable())
1759 return {}; // Give up on this `select`.
1760
1761 Load.setInt(Spec);
1762 Ops.emplace_back(Args&: Load);
1763 }
1764
1765 return Ops;
1766}
1767
1768static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1769 IRBuilderTy &IRB) {
1770 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1771
1772 Value *TV = SI.getTrueValue();
1773 Value *FV = SI.getFalseValue();
1774 // Replace the given load of the select with a select of two loads.
1775
1776 assert(LI.isSimple() && "We only speculate simple loads");
1777
1778 IRB.SetInsertPoint(&LI);
1779
1780 LoadInst *TL =
1781 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: TV, Align: LI.getAlign(),
1782 Name: LI.getName() + ".sroa.speculate.load.true");
1783 LoadInst *FL =
1784 IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: FV, Align: LI.getAlign(),
1785 Name: LI.getName() + ".sroa.speculate.load.false");
1786 NumLoadsSpeculated += 2;
1787
1788 // Transfer alignment and AA info if present.
1789 TL->setAlignment(LI.getAlign());
1790 FL->setAlignment(LI.getAlign());
1791
1792 AAMDNodes Tags = LI.getAAMetadata();
1793 if (Tags) {
1794 TL->setAAMetadata(Tags);
1795 FL->setAAMetadata(Tags);
1796 }
1797
1798 Value *V = IRB.CreateSelect(C: SI.getCondition(), True: TL, False: FL,
1799 Name: LI.getName() + ".sroa.speculated",
1800 MDFrom: ProfcheckDisableMetadataFixes ? nullptr : &SI);
1801
1802 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1803 LI.replaceAllUsesWith(V);
1804}
1805
1806template <typename T>
1807static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1808 SelectHandSpeculativity Spec,
1809 DomTreeUpdater &DTU) {
1810 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!");
1811 LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1812 BasicBlock *Head = I.getParent();
1813 Instruction *ThenTerm = nullptr;
1814 Instruction *ElseTerm = nullptr;
1815 if (Spec.areNoneSpeculatable())
1816 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1817 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU);
1818 else {
1819 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1820 SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU,
1821 /*LI=*/nullptr, /*ThenBlock=*/nullptr);
1822 if (Spec.isSpeculatable(/*isTrueVal=*/true))
1823 cast<CondBrInst>(Val: Head->getTerminator())->swapSuccessors();
1824 }
1825 auto *HeadBI = cast<CondBrInst>(Val: Head->getTerminator());
1826 Spec = {}; // Do not use `Spec` beyond this point.
1827 BasicBlock *Tail = I.getParent();
1828 Tail->setName(Head->getName() + ".cont");
1829 PHINode *PN;
1830 if (isa<LoadInst>(I))
1831 PN = PHINode::Create(Ty: I.getType(), NumReservedValues: 2, NameStr: "", InsertBefore: I.getIterator());
1832 for (BasicBlock *SuccBB : successors(BB: Head)) {
1833 bool IsThen = SuccBB == HeadBI->getSuccessor(i: 0);
1834 int SuccIdx = IsThen ? 0 : 1;
1835 auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1836 auto &CondMemOp = cast<T>(*I.clone());
1837 if (NewMemOpBB != Head) {
1838 NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1839 if (isa<LoadInst>(I))
1840 ++NumLoadsPredicated;
1841 else
1842 ++NumStoresPredicated;
1843 } else {
1844 CondMemOp.dropUBImplyingAttrsAndMetadata();
1845 ++NumLoadsSpeculated;
1846 }
1847 CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
1848 Value *Ptr = SI.getOperand(i_nocapture: 1 + SuccIdx);
1849 CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
1850 if (isa<LoadInst>(I)) {
1851 CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
1852 PN->addIncoming(V: &CondMemOp, BB: NewMemOpBB);
1853 } else
1854 LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
1855 }
1856 if (isa<LoadInst>(I)) {
1857 PN->takeName(V: &I);
1858 LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
1859 I.replaceAllUsesWith(PN);
1860 }
1861}
1862
1863static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I,
1864 SelectHandSpeculativity Spec,
1865 DomTreeUpdater &DTU) {
1866 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1867 rewriteMemOpOfSelect(SI&: SelInst, I&: *LI, Spec, DTU);
1868 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1869 rewriteMemOpOfSelect(SI&: SelInst, I&: *SI, Spec, DTU);
1870 else
1871 llvm_unreachable_internal(msg: "Only for load and store.");
1872}
1873
1874static bool rewriteSelectInstMemOps(SelectInst &SI,
1875 const RewriteableMemOps &Ops,
1876 IRBuilderTy &IRB, DomTreeUpdater *DTU) {
1877 bool CFGChanged = false;
1878 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
1879
1880 for (const RewriteableMemOp &Op : Ops) {
1881 SelectHandSpeculativity Spec;
1882 Instruction *I;
1883 if (auto *const *US = std::get_if<UnspeculatableStore>(ptr: &Op)) {
1884 I = *US;
1885 } else {
1886 auto PSL = std::get<PossiblySpeculatableLoad>(v: Op);
1887 I = PSL.getPointer();
1888 Spec = PSL.getInt();
1889 }
1890 if (Spec.areAllSpeculatable()) {
1891 speculateSelectInstLoads(SI, LI&: cast<LoadInst>(Val&: *I), IRB);
1892 } else {
1893 assert(DTU && "Should not get here when not allowed to modify the CFG!");
1894 rewriteMemOpOfSelect(SelInst&: SI, I&: *I, Spec, DTU&: *DTU);
1895 CFGChanged = true;
1896 }
1897 I->eraseFromParent();
1898 }
1899
1900 for (User *U : make_early_inc_range(Range: SI.users()))
1901 cast<BitCastInst>(Val: U)->eraseFromParent();
1902 SI.eraseFromParent();
1903 return CFGChanged;
1904}
1905
1906/// Compute an adjusted pointer from Ptr by Offset bytes where the
1907/// resulting pointer has PointerTy.
1908static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
1909 APInt Offset, Type *PointerTy,
1910 const Twine &NamePrefix) {
1911 if (Offset != 0)
1912 Ptr = IRB.CreateInBoundsPtrAdd(Ptr, Offset: IRB.getInt(AI: Offset),
1913 Name: NamePrefix + "sroa_idx");
1914 return IRB.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: PointerTy,
1915 Name: NamePrefix + "sroa_cast");
1916}
1917
1918/// Compute the adjusted alignment for a load or store from an offset.
1919static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
1920 return commonAlignment(A: getLoadStoreAlignment(I), Offset);
1921}
1922
1923/// Test whether we can convert a value from the old to the new type.
1924///
1925/// This predicate should be used to guard calls to convertValue in order to
1926/// ensure that we only try to convert viable values. The strategy is that we
1927/// will peel off single element struct and array wrappings to get to an
1928/// underlying value, and convert that value.
1929static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
1930 unsigned VScale = 0) {
1931 if (OldTy == NewTy)
1932 return true;
1933
1934 // For integer types, we can't handle any bit-width differences. This would
1935 // break both vector conversions with extension and introduce endianness
1936 // issues when in conjunction with loads and stores.
1937 if (isa<IntegerType>(Val: OldTy) && isa<IntegerType>(Val: NewTy)) {
1938 assert(cast<IntegerType>(OldTy)->getBitWidth() !=
1939 cast<IntegerType>(NewTy)->getBitWidth() &&
1940 "We can't have the same bitwidth for different int types");
1941 return false;
1942 }
1943
1944 TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
1945 TypeSize OldSize = DL.getTypeSizeInBits(Ty: OldTy);
1946
1947 if ((isa<ScalableVectorType>(Val: NewTy) && isa<FixedVectorType>(Val: OldTy)) ||
1948 (isa<ScalableVectorType>(Val: OldTy) && isa<FixedVectorType>(Val: NewTy))) {
1949 // Conversion is only possible when the size of scalable vectors is known.
1950 if (!VScale)
1951 return false;
1952
1953 // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
1954 // a single domain (either fixed or scalable). Any additional conversion
1955 // between fixed and scalable types is handled through integer types.
1956 auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
1957 auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
1958
1959 if (isa<ScalableVectorType>(Val: NewTy)) {
1960 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: NewVTy), EltTy: OldVTy))
1961 return false;
1962
1963 NewSize = TypeSize::getFixed(ExactSize: NewSize.getKnownMinValue() * VScale);
1964 } else {
1965 if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: OldVTy), EltTy: NewVTy))
1966 return false;
1967
1968 OldSize = TypeSize::getFixed(ExactSize: OldSize.getKnownMinValue() * VScale);
1969 }
1970 }
1971
1972 if (NewSize != OldSize)
1973 return false;
1974 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
1975 return false;
1976
1977 // We can convert pointers to integers and vice-versa. Same for vectors
1978 // of pointers and integers.
1979 OldTy = OldTy->getScalarType();
1980 NewTy = NewTy->getScalarType();
1981 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
1982 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
1983 unsigned OldAS = OldTy->getPointerAddressSpace();
1984 unsigned NewAS = NewTy->getPointerAddressSpace();
1985 // Convert pointers if they are pointers from the same address space or
1986 // different integral (not non-integral) address spaces with the same
1987 // pointer size.
1988 return OldAS == NewAS ||
1989 (!DL.isNonIntegralAddressSpace(AddrSpace: OldAS) &&
1990 !DL.isNonIntegralAddressSpace(AddrSpace: NewAS) &&
1991 DL.getPointerSize(AS: OldAS) == DL.getPointerSize(AS: NewAS));
1992 }
1993
1994 // We can convert integers to integral pointers, but not to non-integral
1995 // pointers.
1996 if (OldTy->isIntegerTy())
1997 return !DL.isNonIntegralPointerType(Ty: NewTy);
1998
1999 // We can convert integral pointers to integers, but non-integral pointers
2000 // need to remain pointers.
2001 if (!DL.isNonIntegralPointerType(Ty: OldTy))
2002 return NewTy->isIntegerTy();
2003
2004 return false;
2005 }
2006
2007 if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy())
2008 return false;
2009
2010 return true;
2011}
2012
2013/// Test whether the given slice use can be promoted to a vector.
2014///
2015/// This function is called to test each entry in a partition which is slated
2016/// for a single slice.
2017static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2018 VectorType *Ty,
2019 uint64_t ElementSize,
2020 const DataLayout &DL,
2021 unsigned VScale) {
2022 // First validate the slice offsets.
2023 uint64_t BeginOffset =
2024 std::max(a: S.beginOffset(), b: P.beginOffset()) - P.beginOffset();
2025 uint64_t BeginIndex = BeginOffset / ElementSize;
2026 if (BeginIndex * ElementSize != BeginOffset ||
2027 BeginIndex >= cast<FixedVectorType>(Val: Ty)->getNumElements())
2028 return false;
2029 uint64_t EndOffset = std::min(a: S.endOffset(), b: P.endOffset()) - P.beginOffset();
2030 uint64_t EndIndex = EndOffset / ElementSize;
2031 if (EndIndex * ElementSize != EndOffset ||
2032 EndIndex > cast<FixedVectorType>(Val: Ty)->getNumElements())
2033 return false;
2034
2035 assert(EndIndex > BeginIndex && "Empty vector!");
2036 uint64_t NumElements = EndIndex - BeginIndex;
2037 Type *SliceTy = (NumElements == 1)
2038 ? Ty->getElementType()
2039 : FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElements);
2040
2041 Type *SplitIntTy =
2042 Type::getIntNTy(C&: Ty->getContext(), N: NumElements * ElementSize * 8);
2043
2044 Use *U = S.getUse();
2045
2046 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2047 if (MI->isVolatile())
2048 return false;
2049 if (!S.isSplittable())
2050 return false; // Skip any unsplittable intrinsics.
2051 if (isa<MemSetInst>(Val: MI)) {
2052 Type *SplatTy = Type::getIntNTy(C&: Ty->getContext(), N: ElementSize * 8);
2053 if (!canConvertValue(DL, OldTy: SplatTy, NewTy: Ty->getElementType(), VScale))
2054 return false;
2055 }
2056 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2057 if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2058 return false;
2059 } else if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2060 if (LI->isVolatile())
2061 return false;
2062 Type *LTy = LI->getType();
2063 // Disable vector promotion when there are loads or stores of an FCA.
2064 if (LTy->isStructTy())
2065 return false;
2066 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2067 assert(LTy->isIntegerTy());
2068 LTy = SplitIntTy;
2069 }
2070 if (!canConvertValue(DL, OldTy: SliceTy, NewTy: LTy, VScale))
2071 return false;
2072 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2073 if (SI->isVolatile())
2074 return false;
2075 Type *STy = SI->getValueOperand()->getType();
2076 // Disable vector promotion when there are loads or stores of an FCA.
2077 if (STy->isStructTy())
2078 return false;
2079 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2080 assert(STy->isIntegerTy());
2081 STy = SplitIntTy;
2082 }
2083 if (!canConvertValue(DL, OldTy: STy, NewTy: SliceTy, VScale))
2084 return false;
2085 } else {
2086 return false;
2087 }
2088
2089 return true;
2090}
2091
2092/// Test whether any vector type in \p CandidateTys is viable for promotion.
2093///
2094/// This implements the necessary checking for \c isVectorPromotionViable over
2095/// all slices of the alloca for the given VectorType.
2096static VectorType *
2097checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
2098 SmallVectorImpl<VectorType *> &CandidateTys,
2099 bool HaveCommonEltTy, Type *CommonEltTy,
2100 bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2101 VectorType *CommonVecPtrTy, unsigned VScale) {
2102 // If we didn't find a vector type, nothing to do here.
2103 if (CandidateTys.empty())
2104 return nullptr;
2105
2106 // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2107 // then we should choose it, not some other alternative.
2108 // But, we can't perform a no-op pointer address space change via bitcast,
2109 // so if we didn't have a common pointer element type, bail.
2110 if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2111 return nullptr;
2112
2113 // Try to pick the "best" element type out of the choices.
2114 if (!HaveCommonEltTy && HaveVecPtrTy) {
2115 // If there was a pointer element type, there's really only one choice.
2116 CandidateTys.clear();
2117 CandidateTys.push_back(Elt: CommonVecPtrTy);
2118 } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2119 // Integer-ify vector types.
2120 for (VectorType *&VTy : CandidateTys) {
2121 if (!VTy->getElementType()->isIntegerTy())
2122 VTy = cast<VectorType>(Val: VTy->getWithNewType(EltTy: IntegerType::getIntNTy(
2123 C&: VTy->getContext(), N: VTy->getScalarSizeInBits())));
2124 }
2125
2126 // Rank the remaining candidate vector types. This is easy because we know
2127 // they're all integer vectors. We sort by ascending number of elements.
2128 auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2129 (void)DL;
2130 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2131 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2132 "Cannot have vector types of different sizes!");
2133 assert(RHSTy->getElementType()->isIntegerTy() &&
2134 "All non-integer types eliminated!");
2135 assert(LHSTy->getElementType()->isIntegerTy() &&
2136 "All non-integer types eliminated!");
2137 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() <
2138 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2139 };
2140 auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2141 (void)DL;
2142 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2143 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2144 "Cannot have vector types of different sizes!");
2145 assert(RHSTy->getElementType()->isIntegerTy() &&
2146 "All non-integer types eliminated!");
2147 assert(LHSTy->getElementType()->isIntegerTy() &&
2148 "All non-integer types eliminated!");
2149 return cast<FixedVectorType>(Val: RHSTy)->getNumElements() ==
2150 cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2151 };
2152 llvm::sort(C&: CandidateTys, Comp: RankVectorTypesComp);
2153 CandidateTys.erase(CS: llvm::unique(R&: CandidateTys, P: RankVectorTypesEq),
2154 CE: CandidateTys.end());
2155 } else {
2156// The only way to have the same element type in every vector type is to
2157// have the same vector type. Check that and remove all but one.
2158#ifndef NDEBUG
2159 for (VectorType *VTy : CandidateTys) {
2160 assert(VTy->getElementType() == CommonEltTy &&
2161 "Unaccounted for element type!");
2162 assert(VTy == CandidateTys[0] &&
2163 "Different vector types with the same element type!");
2164 }
2165#endif
2166 CandidateTys.resize(N: 1);
2167 }
2168
2169 // FIXME: hack. Do we have a named constant for this?
2170 // SDAG SDNode can't have more than 65535 operands.
2171 llvm::erase_if(C&: CandidateTys, P: [](VectorType *VTy) {
2172 return cast<FixedVectorType>(Val: VTy)->getNumElements() >
2173 std::numeric_limits<unsigned short>::max();
2174 });
2175
2176 // Find a vector type viable for promotion by iterating over all slices.
2177 auto *VTy = llvm::find_if(Range&: CandidateTys, P: [&](VectorType *VTy) -> bool {
2178 uint64_t ElementSize =
2179 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2180
2181 // While the definition of LLVM vectors is bitpacked, we don't support sizes
2182 // that aren't byte sized.
2183 if (ElementSize % 8)
2184 return false;
2185 assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&
2186 "vector size not a multiple of element size?");
2187 ElementSize /= 8;
2188
2189 for (const Slice &S : P)
2190 if (!isVectorPromotionViableForSlice(P, S, Ty: VTy, ElementSize, DL, VScale))
2191 return false;
2192
2193 for (const Slice *S : P.splitSliceTails())
2194 if (!isVectorPromotionViableForSlice(P, S: *S, Ty: VTy, ElementSize, DL, VScale))
2195 return false;
2196
2197 return true;
2198 });
2199 return VTy != CandidateTys.end() ? *VTy : nullptr;
2200}
2201
2202static VectorType *createAndCheckVectorTypesForPromotion(
2203 SetVector<Type *> &OtherTys, ArrayRef<VectorType *> CandidateTysCopy,
2204 function_ref<void(Type *)> CheckCandidateType, Partition &P,
2205 const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2206 bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
2207 bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
2208 [[maybe_unused]] VectorType *OriginalElt =
2209 CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
2210 // Consider additional vector types where the element type size is a
2211 // multiple of load/store element size.
2212 for (Type *Ty : OtherTys) {
2213 if (!VectorType::isValidElementType(ElemTy: Ty))
2214 continue;
2215 unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2216 // Make a copy of CandidateTys and iterate through it, because we
2217 // might append to CandidateTys in the loop.
2218 for (VectorType *const VTy : CandidateTysCopy) {
2219 // The elements in the copy should remain invariant throughout the loop
2220 assert(CandidateTysCopy[0] == OriginalElt && "Different Element");
2221 unsigned VectorSize = DL.getTypeSizeInBits(Ty: VTy).getFixedValue();
2222 unsigned ElementSize =
2223 DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2224 if (TypeSize != VectorSize && TypeSize != ElementSize &&
2225 VectorSize % TypeSize == 0) {
2226 VectorType *NewVTy = VectorType::get(ElementType: Ty, NumElements: VectorSize / TypeSize, Scalable: false);
2227 CheckCandidateType(NewVTy);
2228 }
2229 }
2230 }
2231
2232 return checkVectorTypesForPromotion(
2233 P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2234 HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2235}
2236
2237/// Test whether the given alloca partitioning and range of slices can be
2238/// promoted to a vector.
2239///
2240/// This is a quick test to check whether we can rewrite a particular alloca
2241/// partition (and its newly formed alloca) into a vector alloca with only
2242/// whole-vector loads and stores such that it could be promoted to a vector
2243/// SSA value. We only can ensure this for a limited set of operations, and we
2244/// don't want to do the rewrites unless we are confident that the result will
2245/// be promotable, so we have an early test here.
2246static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL,
2247 unsigned VScale) {
2248 // Collect the candidate types for vector-based promotion. Also track whether
2249 // we have different element types.
2250 SmallVector<VectorType *, 4> CandidateTys;
2251 SetVector<Type *> LoadStoreTys;
2252 SetVector<Type *> DeferredTys;
2253 Type *CommonEltTy = nullptr;
2254 VectorType *CommonVecPtrTy = nullptr;
2255 bool HaveVecPtrTy = false;
2256 bool HaveCommonEltTy = true;
2257 bool HaveCommonVecPtrTy = true;
2258 auto CheckCandidateType = [&](Type *Ty) {
2259 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2260 // Return if bitcast to vectors is different for total size in bits.
2261 if (!CandidateTys.empty()) {
2262 VectorType *V = CandidateTys[0];
2263 if (DL.getTypeSizeInBits(Ty: VTy).getFixedValue() !=
2264 DL.getTypeSizeInBits(Ty: V).getFixedValue()) {
2265 CandidateTys.clear();
2266 return;
2267 }
2268 }
2269 CandidateTys.push_back(Elt: VTy);
2270 Type *EltTy = VTy->getElementType();
2271
2272 if (!CommonEltTy)
2273 CommonEltTy = EltTy;
2274 else if (CommonEltTy != EltTy)
2275 HaveCommonEltTy = false;
2276
2277 if (EltTy->isPointerTy()) {
2278 HaveVecPtrTy = true;
2279 if (!CommonVecPtrTy)
2280 CommonVecPtrTy = VTy;
2281 else if (CommonVecPtrTy != VTy)
2282 HaveCommonVecPtrTy = false;
2283 }
2284 }
2285 };
2286
2287 // Put load and store types into a set for de-duplication.
2288 for (const Slice &S : P) {
2289 Type *Ty;
2290 if (auto *LI = dyn_cast<LoadInst>(Val: S.getUse()->getUser()))
2291 Ty = LI->getType();
2292 else if (auto *SI = dyn_cast<StoreInst>(Val: S.getUse()->getUser()))
2293 Ty = SI->getValueOperand()->getType();
2294 else
2295 continue;
2296
2297 auto CandTy = Ty->getScalarType();
2298 if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() ||
2299 S.endOffset() != P.endOffset())) {
2300 DeferredTys.insert(X: Ty);
2301 continue;
2302 }
2303
2304 LoadStoreTys.insert(X: Ty);
2305 // Consider any loads or stores that are the exact size of the slice.
2306 if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2307 CheckCandidateType(Ty);
2308 }
2309
2310 SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
2311 if (auto *VTy = createAndCheckVectorTypesForPromotion(
2312 OtherTys&: LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2313 CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2314 HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2315 return VTy;
2316
2317 CandidateTys.clear();
2318 return createAndCheckVectorTypesForPromotion(
2319 OtherTys&: DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2320 HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2321 CommonVecPtrTy, VScale);
2322}
2323
2324/// Test whether a slice of an alloca is valid for integer widening.
2325///
2326/// This implements the necessary checking for the \c isIntegerWideningViable
2327/// test below on a single slice of the alloca.
2328static bool isIntegerWideningViableForSlice(const Slice &S,
2329 uint64_t AllocBeginOffset,
2330 Type *AllocaTy,
2331 const DataLayout &DL,
2332 bool &WholeAllocaOp) {
2333 uint64_t Size = DL.getTypeStoreSize(Ty: AllocaTy).getFixedValue();
2334
2335 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2336 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2337
2338 Use *U = S.getUse();
2339
2340 // Lifetime intrinsics operate over the whole alloca whose sizes are usually
2341 // larger than other load/store slices (RelEnd > Size). But lifetime are
2342 // always promotable and should not impact other slices' promotability of the
2343 // partition.
2344 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2345 if (II->isLifetimeStartOrEnd() || II->isDroppable())
2346 return true;
2347 }
2348
2349 // We can't reasonably handle cases where the load or store extends past
2350 // the end of the alloca's type and into its padding.
2351 if (RelEnd > Size)
2352 return false;
2353
2354 if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2355 if (LI->isVolatile())
2356 return false;
2357 // We can't handle loads that extend past the allocated memory.
2358 TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
2359 if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
2360 return false;
2361 // So far, AllocaSliceRewriter does not support widening split slice tails
2362 // in rewriteIntegerLoad.
2363 if (S.beginOffset() < AllocBeginOffset)
2364 return false;
2365 // Note that we don't count vector loads or stores as whole-alloca
2366 // operations which enable integer widening because we would prefer to use
2367 // vector widening instead.
2368 if (!isa<VectorType>(Val: LI->getType()) && RelBegin == 0 && RelEnd == Size)
2369 WholeAllocaOp = true;
2370 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: LI->getType())) {
2371 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2372 return false;
2373 } else if (RelBegin != 0 || RelEnd != Size ||
2374 !canConvertValue(DL, OldTy: AllocaTy, NewTy: LI->getType())) {
2375 // Non-integer loads need to be convertible from the alloca type so that
2376 // they are promotable.
2377 return false;
2378 }
2379 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2380 Type *ValueTy = SI->getValueOperand()->getType();
2381 if (SI->isVolatile())
2382 return false;
2383 // We can't handle stores that extend past the allocated memory.
2384 TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValueTy);
2385 if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
2386 return false;
2387 // So far, AllocaSliceRewriter does not support widening split slice tails
2388 // in rewriteIntegerStore.
2389 if (S.beginOffset() < AllocBeginOffset)
2390 return false;
2391 // Note that we don't count vector loads or stores as whole-alloca
2392 // operations which enable integer widening because we would prefer to use
2393 // vector widening instead.
2394 if (!isa<VectorType>(Val: ValueTy) && RelBegin == 0 && RelEnd == Size)
2395 WholeAllocaOp = true;
2396 if (IntegerType *ITy = dyn_cast<IntegerType>(Val: ValueTy)) {
2397 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2398 return false;
2399 } else if (RelBegin != 0 || RelEnd != Size ||
2400 !canConvertValue(DL, OldTy: ValueTy, NewTy: AllocaTy)) {
2401 // Non-integer stores need to be convertible to the alloca type so that
2402 // they are promotable.
2403 return false;
2404 }
2405 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2406 if (MI->isVolatile() || !isa<Constant>(Val: MI->getLength()))
2407 return false;
2408 if (!S.isSplittable())
2409 return false; // Skip any unsplittable intrinsics.
2410 } else {
2411 return false;
2412 }
2413
2414 return true;
2415}
2416
2417/// Test whether the given alloca partition's integer operations can be
2418/// widened to promotable ones.
2419///
2420/// This is a quick test to check whether we can rewrite the integer loads and
2421/// stores to a particular alloca into wider loads and stores and be able to
2422/// promote the resulting alloca.
2423static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2424 const DataLayout &DL) {
2425 uint64_t SizeInBits = DL.getTypeSizeInBits(Ty: AllocaTy).getFixedValue();
2426 // Don't create integer types larger than the maximum bitwidth.
2427 if (SizeInBits > IntegerType::MAX_INT_BITS)
2428 return false;
2429
2430 // Don't try to handle allocas with bit-padding.
2431 if (SizeInBits != DL.getTypeStoreSizeInBits(Ty: AllocaTy).getFixedValue())
2432 return false;
2433
2434 // We need to ensure that an integer type with the appropriate bitwidth can
2435 // be converted to the alloca type, whatever that is. We don't want to force
2436 // the alloca itself to have an integer type if there is a more suitable one.
2437 Type *IntTy = Type::getIntNTy(C&: AllocaTy->getContext(), N: SizeInBits);
2438 if (!canConvertValue(DL, OldTy: AllocaTy, NewTy: IntTy) ||
2439 !canConvertValue(DL, OldTy: IntTy, NewTy: AllocaTy))
2440 return false;
2441
2442 // While examining uses, we ensure that the alloca has a covering load or
2443 // store. We don't want to widen the integer operations only to fail to
2444 // promote due to some other unsplittable entry (which we may make splittable
2445 // later). However, if there are only splittable uses, go ahead and assume
2446 // that we cover the alloca.
2447 // FIXME: We shouldn't consider split slices that happen to start in the
2448 // partition here...
2449 bool WholeAllocaOp = P.empty() && DL.isLegalInteger(Width: SizeInBits);
2450
2451 for (const Slice &S : P)
2452 if (!isIntegerWideningViableForSlice(S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2453 WholeAllocaOp))
2454 return false;
2455
2456 for (const Slice *S : P.splitSliceTails())
2457 if (!isIntegerWideningViableForSlice(S: *S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2458 WholeAllocaOp))
2459 return false;
2460
2461 return WholeAllocaOp;
2462}
2463
2464static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
2465 IntegerType *Ty, uint64_t Offset,
2466 const Twine &Name) {
2467 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2468 IntegerType *IntTy = cast<IntegerType>(Val: V->getType());
2469 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2470 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2471 "Element extends past full value");
2472 uint64_t ShAmt = 8 * Offset;
2473 if (DL.isBigEndian())
2474 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2475 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2476 if (ShAmt) {
2477 V = IRB.CreateLShr(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2478 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2479 }
2480 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2481 "Cannot extract to a larger integer!");
2482 if (Ty != IntTy) {
2483 V = IRB.CreateTrunc(V, DestTy: Ty, Name: Name + ".trunc");
2484 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2485 }
2486 return V;
2487}
2488
2489static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2490 Value *V, uint64_t Offset, const Twine &Name) {
2491 IntegerType *IntTy = cast<IntegerType>(Val: Old->getType());
2492 IntegerType *Ty = cast<IntegerType>(Val: V->getType());
2493 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2494 "Cannot insert a larger integer!");
2495 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2496 if (Ty != IntTy) {
2497 V = IRB.CreateZExt(V, DestTy: IntTy, Name: Name + ".ext");
2498 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2499 }
2500 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2501 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2502 "Element store outside of alloca store");
2503 uint64_t ShAmt = 8 * Offset;
2504 if (DL.isBigEndian())
2505 ShAmt = 8 * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2506 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2507 if (ShAmt) {
2508 V = IRB.CreateShl(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2509 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2510 }
2511
2512 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2513 APInt Mask = ~Ty->getMask().zext(width: IntTy->getBitWidth()).shl(shiftAmt: ShAmt);
2514 Old = IRB.CreateAnd(LHS: Old, RHS: Mask, Name: Name + ".mask");
2515 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2516 V = IRB.CreateOr(LHS: Old, RHS: V, Name: Name + ".insert");
2517 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2518 }
2519 return V;
2520}
2521
2522static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2523 unsigned EndIndex, const Twine &Name) {
2524 auto *VecTy = cast<FixedVectorType>(Val: V->getType());
2525 unsigned NumElements = EndIndex - BeginIndex;
2526 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2527
2528 if (NumElements == VecTy->getNumElements())
2529 return V;
2530
2531 if (NumElements == 1) {
2532 V = IRB.CreateExtractElement(Vec: V, Idx: BeginIndex, Name: Name + ".extract");
2533 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2534 return V;
2535 }
2536
2537 auto Mask = llvm::to_vector<8>(Range: llvm::seq<int>(Begin: BeginIndex, End: EndIndex));
2538 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".extract");
2539 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2540 return V;
2541}
2542
2543static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
2544 unsigned BeginIndex, const Twine &Name) {
2545 VectorType *VecTy = cast<VectorType>(Val: Old->getType());
2546 assert(VecTy && "Can only insert a vector into a vector");
2547
2548 VectorType *Ty = dyn_cast<VectorType>(Val: V->getType());
2549 if (!Ty) {
2550 // Single element to insert.
2551 V = IRB.CreateInsertElement(Vec: Old, NewElt: V, Idx: BeginIndex, Name: Name + ".insert");
2552 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2553 return V;
2554 }
2555
2556 unsigned NumSubElements = cast<FixedVectorType>(Val: Ty)->getNumElements();
2557 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2558
2559 assert(NumSubElements <= NumElements && "Too many elements!");
2560 if (NumSubElements == NumElements) {
2561 assert(V->getType() == VecTy && "Vector type mismatch");
2562 return V;
2563 }
2564 unsigned EndIndex = BeginIndex + NumSubElements;
2565
2566 // When inserting a smaller vector into the larger to store, we first
2567 // use a shuffle vector to widen it with undef elements, and then
2568 // a second shuffle vector to select between the loaded vector and the
2569 // incoming vector.
2570 SmallVector<int, 8> Mask;
2571 Mask.reserve(N: NumElements);
2572 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2573 if (Idx >= BeginIndex && Idx < EndIndex)
2574 Mask.push_back(Elt: Idx - BeginIndex);
2575 else
2576 Mask.push_back(Elt: -1);
2577 V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".expand");
2578 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2579
2580 Mask.clear();
2581 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2582 if (Idx >= BeginIndex && Idx < EndIndex)
2583 Mask.push_back(Elt: Idx);
2584 else
2585 Mask.push_back(Elt: Idx + NumElements);
2586 V = IRB.CreateShuffleVector(V1: V, V2: Old, Mask, Name: Name + "blend");
2587 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2588 return V;
2589}
2590
2591/// This function takes two vector values and combines them into a single vector
2592/// by concatenating their elements. The function handles:
2593///
2594/// 1. Element type mismatch: If either vector's element type differs from
2595/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2596/// preserving the total bit width (adjusting the number of elements
2597/// accordingly).
2598///
2599/// 2. Size mismatch: After transforming the vectors to have the desired element
2600/// type, if the two vectors have different numbers of elements, the smaller
2601/// vector is extended with poison values to match the size of the larger
2602/// vector before concatenation.
2603///
2604/// 3. Concatenation: The vectors are merged using a shuffle operation that
2605/// places all elements of V0 first, followed by all elements of V1.
2606///
2607/// \param V0 The first vector to merge (must be a vector type)
2608/// \param V1 The second vector to merge (must be a vector type)
2609/// \param DL The data layout for size calculations
2610/// \param NewAIEltTy The desired element type for the result vector
2611/// \param Builder IRBuilder for creating new instructions
2612/// \return A new vector containing all elements from V0 followed by all
2613/// elements from V1
2614static Value *mergeTwoVectors(Value *V0, Value *V1, const DataLayout &DL,
2615 Type *NewAIEltTy, IRBuilder<> &Builder) {
2616 // V0 and V1 are vectors
2617 // Create a new vector type with combined elements
2618 // Use ShuffleVector to concatenate the vectors
2619 auto *VecType0 = cast<FixedVectorType>(Val: V0->getType());
2620 auto *VecType1 = cast<FixedVectorType>(Val: V1->getType());
2621
2622 // If V0/V1 element types are different from NewAllocaElementType,
2623 // we need to introduce bitcasts before merging them
2624 auto BitcastIfNeeded = [&](Value *&V, FixedVectorType *&VecType,
2625 const char *DebugName) {
2626 Type *EltType = VecType->getElementType();
2627 if (EltType != NewAIEltTy) {
2628 // Calculate new number of elements to maintain same bit width
2629 unsigned TotalBits =
2630 VecType->getNumElements() * DL.getTypeSizeInBits(Ty: EltType);
2631 unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(Ty: NewAIEltTy);
2632
2633 auto *NewVecType = FixedVectorType::get(ElementType: NewAIEltTy, NumElts: NewNumElts);
2634 V = Builder.CreateBitCast(V, DestTy: NewVecType);
2635 VecType = NewVecType;
2636 LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2637 }
2638 };
2639
2640 BitcastIfNeeded(V0, VecType0, "V0");
2641 BitcastIfNeeded(V1, VecType1, "V1");
2642
2643 unsigned NumElts0 = VecType0->getNumElements();
2644 unsigned NumElts1 = VecType1->getNumElements();
2645
2646 SmallVector<int, 16> ShuffleMask;
2647
2648 if (NumElts0 == NumElts1) {
2649 for (unsigned i = 0; i < NumElts0 + NumElts1; ++i)
2650 ShuffleMask.push_back(Elt: i);
2651 } else {
2652 // If two vectors have different sizes, we need to extend
2653 // the smaller vector to the size of the larger vector.
2654 unsigned SmallSize = std::min(a: NumElts0, b: NumElts1);
2655 unsigned LargeSize = std::max(a: NumElts0, b: NumElts1);
2656 bool IsV0Smaller = NumElts0 < NumElts1;
2657 Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2658 SmallVector<int, 16> ExtendMask;
2659 for (unsigned i = 0; i < SmallSize; ++i)
2660 ExtendMask.push_back(Elt: i);
2661 for (unsigned i = SmallSize; i < LargeSize; ++i)
2662 ExtendMask.push_back(Elt: PoisonMaskElem);
2663 ExtendedVec = Builder.CreateShuffleVector(
2664 V1: ExtendedVec, V2: PoisonValue::get(T: ExtendedVec->getType()), Mask: ExtendMask);
2665 LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2666 for (unsigned i = 0; i < NumElts0; ++i)
2667 ShuffleMask.push_back(Elt: i);
2668 for (unsigned i = 0; i < NumElts1; ++i)
2669 ShuffleMask.push_back(Elt: LargeSize + i);
2670 }
2671
2672 return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
2673}
2674
2675namespace {
2676
2677/// Visitor to rewrite instructions using p particular slice of an alloca
2678/// to use a new alloca.
2679///
2680/// Also implements the rewriting to vector-based accesses when the partition
2681/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2682/// lives here.
2683class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2684 // Befriend the base class so it can delegate to private visit methods.
2685 friend class InstVisitor<AllocaSliceRewriter, bool>;
2686
2687 using Base = InstVisitor<AllocaSliceRewriter, bool>;
2688
2689 const DataLayout &DL;
2690 AllocaSlices &AS;
2691 SROA &Pass;
2692 AllocaInst &OldAI, &NewAI;
2693 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2694 Type *NewAllocaTy;
2695
2696 // This is a convenience and flag variable that will be null unless the new
2697 // alloca's integer operations should be widened to this integer type due to
2698 // passing isIntegerWideningViable above. If it is non-null, the desired
2699 // integer type will be stored here for easy access during rewriting.
2700 IntegerType *IntTy;
2701
2702 // If we are rewriting an alloca partition which can be written as pure
2703 // vector operations, we stash extra information here. When VecTy is
2704 // non-null, we have some strict guarantees about the rewritten alloca:
2705 // - The new alloca is exactly the size of the vector type here.
2706 // - The accesses all either map to the entire vector or to a single
2707 // element.
2708 // - The set of accessing instructions is only one of those handled above
2709 // in isVectorPromotionViable. Generally these are the same access kinds
2710 // which are promotable via mem2reg.
2711 VectorType *VecTy;
2712 Type *ElementTy;
2713 uint64_t ElementSize;
2714
2715 // The original offset of the slice currently being rewritten relative to
2716 // the original alloca.
2717 uint64_t BeginOffset = 0;
2718 uint64_t EndOffset = 0;
2719
2720 // The new offsets of the slice currently being rewritten relative to the
2721 // original alloca.
2722 uint64_t NewBeginOffset = 0, NewEndOffset = 0;
2723
2724 uint64_t SliceSize = 0;
2725 bool IsSplittable = false;
2726 bool IsSplit = false;
2727 Use *OldUse = nullptr;
2728 Instruction *OldPtr = nullptr;
2729
2730 // Track post-rewrite users which are PHI nodes and Selects.
2731 SmallSetVector<PHINode *, 8> &PHIUsers;
2732 SmallSetVector<SelectInst *, 8> &SelectUsers;
2733
2734 // Utility IR builder, whose name prefix is setup for each visited use, and
2735 // the insertion point is set to point to the user.
2736 IRBuilderTy IRB;
2737
2738 // Return the new alloca, addrspacecasted if required to avoid changing the
2739 // addrspace of a volatile access.
2740 Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) {
2741 if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace())
2742 return &NewAI;
2743
2744 Type *AccessTy = IRB.getPtrTy(AddrSpace);
2745 return IRB.CreateAddrSpaceCast(V: &NewAI, DestTy: AccessTy);
2746 }
2747
2748public:
2749 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2750 AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2751 uint64_t NewAllocaBeginOffset,
2752 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2753 VectorType *PromotableVecTy,
2754 SmallSetVector<PHINode *, 8> &PHIUsers,
2755 SmallSetVector<SelectInst *, 8> &SelectUsers)
2756 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2757 NewAllocaBeginOffset(NewAllocaBeginOffset),
2758 NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2759 IntTy(IsIntegerPromotable
2760 ? Type::getIntNTy(
2761 C&: NewAI.getContext(),
2762 N: DL.getTypeSizeInBits(Ty: NewAllocaTy).getFixedValue())
2763 : nullptr),
2764 VecTy(PromotableVecTy),
2765 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2766 ElementSize(VecTy ? DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8
2767 : 0),
2768 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2769 IRB(NewAI.getContext(), ConstantFolder()) {
2770 if (VecTy) {
2771 assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
2772 "Only multiple-of-8 sized vector elements are viable");
2773 ++NumVectorized;
2774 }
2775 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
2776 }
2777
2778 bool visit(AllocaSlices::const_iterator I) {
2779 bool CanSROA = true;
2780 BeginOffset = I->beginOffset();
2781 EndOffset = I->endOffset();
2782 IsSplittable = I->isSplittable();
2783 IsSplit =
2784 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
2785 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2786 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2787 LLVM_DEBUG(dbgs() << "\n");
2788
2789 // Compute the intersecting offset range.
2790 assert(BeginOffset < NewAllocaEndOffset);
2791 assert(EndOffset > NewAllocaBeginOffset);
2792 NewBeginOffset = std::max(a: BeginOffset, b: NewAllocaBeginOffset);
2793 NewEndOffset = std::min(a: EndOffset, b: NewAllocaEndOffset);
2794
2795 SliceSize = NewEndOffset - NewBeginOffset;
2796 LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2797 << ") NewBegin:(" << NewBeginOffset << ", "
2798 << NewEndOffset << ") NewAllocaBegin:("
2799 << NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2800 << ")\n");
2801 assert(IsSplit || NewBeginOffset == BeginOffset);
2802 OldUse = I->getUse();
2803 OldPtr = cast<Instruction>(Val: OldUse->get());
2804
2805 Instruction *OldUserI = cast<Instruction>(Val: OldUse->getUser());
2806 IRB.SetInsertPoint(OldUserI);
2807 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2808 // Avoid materializing the name prefix when it is discarded anyway.
2809 if (!IRB.getContext().shouldDiscardValueNames())
2810 IRB.getInserter().SetNamePrefix(Twine(NewAI.getName()) + "." +
2811 Twine(BeginOffset) + ".");
2812
2813 CanSROA &= visit(I: cast<Instruction>(Val: OldUse->getUser()));
2814 if (VecTy || IntTy)
2815 assert(CanSROA);
2816 return CanSROA;
2817 }
2818
2819 /// Attempts to rewrite a partition using tree-structured merge optimization.
2820 ///
2821 /// This function handles two patterns. Both produce an O(log n) tree of
2822 /// shufflevectors in place of the linear expand+blend chain that SROA would
2823 /// otherwise emit for each partial store.
2824 ///
2825 /// Pattern 1 (stores-only):
2826 /// Multiple non-overlapping partial stores completely fill the alloca
2827 /// and there is exactly one full-width load coming after the stores.
2828 /// The stores are tree-merged into a single vector and stored once.
2829 ///
2830 /// Example transformation:
2831 /// Before: (stores do not have to be in order)
2832 /// %alloca = alloca <8 x float>
2833 /// store <2 x float> %val0, ptr %alloca ; offset 0-1
2834 /// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2835 /// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2836 /// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2837 /// %r = load <8 x float>, ptr %alloca
2838 ///
2839 /// After: tree of shufflevectors producing <8 x float> directly.
2840 ///
2841 /// Pattern 2 (init + RMW, possibly multi-round):
2842 /// A single full-width init store, followed by partial loads and
2843 /// partial stores that read-modify-write the alloca one or more
2844 /// times, optionally followed by a full-width load. The only
2845 /// structural requirement is that the distinct [begin, end) ranges
2846 /// touched by the partial loads and stores, taken together, tile
2847 /// the alloca disjointly.
2848 ///
2849 /// We keep a map from each slice range to the SSA value that
2850 /// currently lives there, `SliceValues[r] -> Value*`:
2851 /// - initialize each entry to the corresponding piece of the
2852 /// init store's value (via a shufflevector picking the
2853 /// range's elements out of the init value),
2854 /// - walk partial loads and stores in block order,
2855 /// - for a partial load at range r: RAUW with `SliceValues[r]`,
2856 /// - for a partial store at range r: update `SliceValues[r]` to
2857 /// the stored value and drop the store.
2858 /// At the end, the final `SliceValues[r]` entries are tree-merged
2859 /// (in range order) into a single store to the alloca, and the
2860 /// optional full-width load is replaced by a load of the alloca.
2861 ///
2862 /// Because the ranges are disjoint by construction, a store at one
2863 /// range cannot affect another range's tracked value, so a single
2864 /// block-order walk correctly tracks the memory state at each
2865 /// range. The algorithm handles multi-round RMW, partial loads
2866 /// and stores interleaved in any order, read-only slices (the
2867 /// tracked value stays at the init extract), and write-only
2868 /// slices (the tracked value never flows into a load).
2869 ///
2870 /// \param P The partition to analyze and potentially rewrite
2871 /// \return An optional vector of values that were deleted during the
2872 /// rewrite, or std::nullopt if the partition cannot be optimized.
2873 std::optional<SmallVector<Value *, 4>>
2874 rewriteTreeStructuredMerge(Partition &P) {
2875 // No tail slices that overlap with the partition
2876 if (P.splitSliceTails().size() > 0)
2877 return std::nullopt;
2878
2879 // Structure to hold store information
2880 struct StoreInfo {
2881 StoreInst *Store;
2882 uint64_t BeginOffset;
2883 uint64_t EndOffset;
2884 Value *StoredValue;
2885 StoreInfo(StoreInst *SI, uint64_t Begin, uint64_t End, Value *Val)
2886 : Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
2887 };
2888 struct LoadInfo {
2889 LoadInst *Load;
2890 uint64_t BeginOffset;
2891 uint64_t EndOffset;
2892 };
2893
2894 SmallVector<StoreInfo, 4> StoreInfos; // partial stores only
2895 SmallVector<LoadInfo, 4> LoadInfos; // partial loads only
2896 LoadInst *FullLoad = nullptr; // optional full-width load
2897 StoreInst *InitStore = nullptr; // optional full-width init store
2898
2899 // If the new alloca is a fixed vector type, we use its element type as the
2900 // allocated element type, otherwise we use i8 as the allocated element
2901 Type *AllocatedEltTy =
2902 isa<FixedVectorType>(Val: NewAllocaTy)
2903 ? cast<FixedVectorType>(Val: NewAllocaTy)->getElementType()
2904 : Type::getInt8Ty(C&: NewAI.getContext());
2905 unsigned AllocatedEltTySize = DL.getTypeSizeInBits(Ty: AllocatedEltTy);
2906
2907 // Helper to check if a type is
2908 // 1. A fixed vector type
2909 // 2. The element type is not a pointer
2910 // 3. The element type size is byte-aligned
2911 // We only handle the cases that the ld/st meet these conditions
2912 auto IsTypeValidForTreeStructuredMerge = [&](Type *Ty) -> bool {
2913 auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: Ty);
2914 return FixedVecTy &&
2915 DL.getTypeSizeInBits(Ty: FixedVecTy->getElementType()) % 8 == 0 &&
2916 !FixedVecTy->getElementType()->isPointerTy();
2917 };
2918
2919 for (Slice &S : P) {
2920 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
2921 // A "full-width" slice spans the entire alloca; it's either the single
2922 // init store (Pattern 2) or the single final load (both patterns).
2923 bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
2924 S.endOffset() == NewAllocaEndOffset);
2925 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
2926 // Only handle simple (non-volatile, non-atomic) loads.
2927 if (!LI->isSimple() ||
2928 !IsTypeValidForTreeStructuredMerge(LI->getType()))
2929 return std::nullopt;
2930 if (IsFullWidth) {
2931 // We accept at most one full-width load (the "final" load, after
2932 // all the partial stores).
2933 if (FullLoad)
2934 return std::nullopt;
2935 FullLoad = LI;
2936 } else {
2937 // Partial load (RMW pattern only).
2938 LoadInfos.push_back(Elt: {.Load: LI, .BeginOffset: S.beginOffset(), .EndOffset: S.endOffset()});
2939 }
2940 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
2941 // Do not handle the case if
2942 // 1. The store does not meet the conditions in the helper function
2943 // 2. The store is not simple — we drop stores as part of the
2944 // rewrite, so volatile stores (which must be kept) and atomic
2945 // stores (which carry memory-ordering semantics) are unsound
2946 // to replace with SSA bookkeeping.
2947 // 3. The total store size is not a multiple of the allocated
2948 // element type size (required so the tree merge can produce a
2949 // vector whose element type matches the alloca).
2950 if (!SI->isSimple() || !IsTypeValidForTreeStructuredMerge(
2951 SI->getValueOperand()->getType()))
2952 return std::nullopt;
2953 auto *StVecTy = cast<FixedVectorType>(Val: SI->getValueOperand()->getType());
2954 unsigned NumElts = StVecTy->getNumElements();
2955 unsigned EltSize = DL.getTypeSizeInBits(Ty: StVecTy->getElementType());
2956 if (NumElts * EltSize % AllocatedEltTySize != 0)
2957 return std::nullopt;
2958 if (IsFullWidth) {
2959 // At most one full-width store is allowed — it's the init store
2960 // for the RMW pattern.
2961 if (InitStore)
2962 return std::nullopt;
2963 InitStore = SI;
2964 } else {
2965 StoreInfos.emplace_back(Args&: SI, Args: S.beginOffset(), Args: S.endOffset(),
2966 Args: SI->getValueOperand());
2967 }
2968 } else {
2969 // If we have instructions other than load and store, we cannot do
2970 // the tree structured merge.
2971 return std::nullopt;
2972 }
2973 }
2974
2975 // Need at least two partial stores to benefit from tree-merging; a
2976 // single store is already optimal as-is. This applies to both patterns
2977 // below, so check it before classifying.
2978 if (StoreInfos.size() < 2)
2979 return std::nullopt;
2980
2981 // Classify the pattern by looking at what we collected:
2982 // Pattern 1 (stores-only): only partial stores + exactly one full load.
2983 // Pattern 2 (RMW): one full init store + partial loads + partial stores
2984 // (+ optional full final load). RMW also needs VecTy to be set
2985 // because we use getIndex() to convert byte offsets to element
2986 // indices, which requires a promoted vector alloca.
2987 bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
2988 bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
2989 if (!IsRMWPattern && !IsStoresOnlyPattern)
2990 return std::nullopt;
2991
2992 // All partial stores must live in the same basic block — the tree merge
2993 // is built in a single BB using block-order ordering (comesBefore).
2994 BasicBlock *StoreBB = StoreInfos[0].Store->getParent();
2995 for (auto &Info : StoreInfos)
2996 if (Info.Store->getParent() != StoreBB)
2997 return std::nullopt;
2998
2999 SmallVector<Value *, 4> DeletedValues;
3000
3001 // Helper: pairwise tree-merge a list of vectors into a single vector.
3002 // At each iteration we merge each adjacent pair via mergeTwoVectors,
3003 // collect the merged values into Next, and (if Vals had odd length)
3004 // carry the trailing element through unchanged. Loop until one value
3005 // remains — the fully-merged vector.
3006 auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3007 IRBuilder<> &B) -> Value * {
3008 LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3009 while (Vals.size() > 1) {
3010 SmallVector<Value *, 8> Next;
3011 for (unsigned I = 0, E = Vals.size(); I + 1 < E; I += 2) {
3012 Value *M =
3013 mergeTwoVectors(V0: Vals[I], V1: Vals[I + 1], DL, NewAIEltTy: AllocatedEltTy, Builder&: B);
3014 LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3015 Next.push_back(Elt: M);
3016 }
3017 if (Vals.size() % 2 == 1)
3018 Next.push_back(Elt: Vals.back());
3019 Vals = std::move(Next);
3020 }
3021 return Vals[0];
3022 };
3023
3024 // Replace a full-width load with a load of the freshly-merged alloca.
3025 // The merge stored a value of type Merged->getType() into NewAI; we load
3026 // that same type back so every access to NewAI stays consistently typed
3027 // (otherwise the alloca is no longer promotable).
3028 auto ReplaceFullLoad = [&](LoadInst *LoadToReplace, Value *Merged) {
3029 IRBuilder<> LoadBuilder(LoadToReplace);
3030 Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3031 Ty: Merged->getType(), Ptr: &NewAI, Align: getSliceAlign(),
3032 isVolatile: LoadToReplace->isVolatile(),
3033 Name: LoadToReplace->getName() + ".sroa.new.load");
3034 if (NewLoad->getType() != LoadToReplace->getType())
3035 NewLoad = LoadBuilder.CreateBitCast(V: NewLoad, DestTy: LoadToReplace->getType());
3036 LoadToReplace->replaceAllUsesWith(V: NewLoad);
3037 DeletedValues.push_back(Elt: LoadToReplace);
3038 };
3039
3040 if (IsStoresOnlyPattern) {
3041 // Stores should not overlap and should cover the whole alloca.
3042 // Sort by begin offset to verify this with a single linear scan.
3043 llvm::sort(C&: StoreInfos, Comp: [](const StoreInfo &A, const StoreInfo &B) {
3044 return A.BeginOffset < B.BeginOffset;
3045 });
3046 // Check for gap or overlap: each begin offset must equal the previous
3047 // end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3048 // NewAllocaEndOffset) exactly.
3049 uint64_t Expected = NewAllocaBeginOffset;
3050 for (auto &Info : StoreInfos) {
3051 if (Info.BeginOffset != Expected)
3052 return std::nullopt;
3053 Expected = Info.EndOffset;
3054 }
3055 // Stores cover the entire alloca (no trailing gap either).
3056 if (Expected != NewAllocaEndOffset)
3057 return std::nullopt;
3058
3059 // The load should not be in the middle of the stores.
3060 // Note:
3061 // If the load is in a different basic block from the stores, we can
3062 // still do the tree-structured merge. We don't have store->load
3063 // forwarding here — the merged vector is stored back to NewAI and
3064 // the new load loads from NewAI. The forwarding will be handled
3065 // later when NewAI is promoted.
3066 BasicBlock *LoadBB = FullLoad->getParent();
3067 if (LoadBB == StoreBB) {
3068 for (auto &Info : StoreInfos)
3069 if (!Info.Store->comesBefore(Other: FullLoad))
3070 return std::nullopt;
3071 }
3072
3073 LLVM_DEBUG({
3074 dbgs() << "Tree structured merge rewrite (stores-only):\n";
3075 dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3076 for (auto [I, Info] : enumerate(StoreInfos)) {
3077 dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3078 << Info.EndOffset << ") \tStore: " << *Info.Store
3079 << "\tValue: " << *Info.StoredValue << "\n";
3080 }
3081 });
3082
3083 // StoreInfos is sorted by offset, not by block order. Anchoring to
3084 // StoreInfos.back().Store (last by offset) can place shuffles before
3085 // operands that appear later in the block (invalid SSA). Insert before
3086 // FullLoad when it shares the store block (after all stores, before
3087 // any later IR in that block). Otherwise insert before the store
3088 // block's terminator so the merge runs after every store and any
3089 // trailing instructions in that block.
3090 IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(Val: FullLoad)
3091 : StoreBB->getTerminator());
3092 SmallVector<Value *, 8> Vals;
3093 for (const auto &Info : StoreInfos) {
3094 DeletedValues.push_back(Elt: Info.Store);
3095 Vals.push_back(Elt: Info.StoredValue);
3096 }
3097 // Merge all stored values and store the merged value into the alloca.
3098 Value *Merged = TreeMerge(Vals, Builder);
3099 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3100
3101 // Replace the original load with a load of the newly-merged alloca.
3102 ReplaceFullLoad(FullLoad, Merged);
3103 return DeletedValues;
3104 }
3105
3106 // RMW pattern handling starts from here.
3107 // Like StoreBB above: keep the init store, all partial loads and all
3108 // partial stores in one basic block so we can reason about ordering
3109 // with comesBefore and build SSA without PHIs.
3110 if (InitStore->getParent() != StoreBB)
3111 return std::nullopt;
3112 if (any_of(Range&: LoadInfos, P: [&](const LoadInfo &I) {
3113 return I.Load->getParent() != StoreBB;
3114 }))
3115 return std::nullopt;
3116 // FullLoad (if any) is allowed to live in a different basic block. See
3117 // the note on the stores-only path: we don't do store->load forwarding
3118 // directly — the merged vector is stored to NewAI and the new load
3119 // loads from NewAI, so cross-BB ordering is resolved later when NewAI
3120 // is promoted.
3121
3122 // Collect the combined partial-load/partial-store accesses sorted
3123 // by block order. Used both for ordering checks and for the rewrite
3124 // walk below.
3125 struct Access {
3126 Instruction *Inst;
3127 uint64_t BeginOffset, EndOffset;
3128 bool IsStore;
3129 };
3130 SmallVector<Access, 16> Accesses;
3131 Accesses.reserve(N: LoadInfos.size() + StoreInfos.size());
3132 for (const auto &L : LoadInfos)
3133 Accesses.push_back(Elt: {.Inst: L.Load, .BeginOffset: L.BeginOffset, .EndOffset: L.EndOffset, .IsStore: false});
3134 for (const auto &S : StoreInfos)
3135 Accesses.push_back(Elt: {.Inst: S.Store, .BeginOffset: S.BeginOffset, .EndOffset: S.EndOffset, .IsStore: true});
3136 llvm::sort(C&: Accesses, Comp: [](const Access &A, const Access &B) {
3137 return A.Inst->comesBefore(Other: B.Inst);
3138 });
3139
3140 // Ordering constraint 1: InitStore must come before every partial
3141 // access — they read/write the RMW state initialised by InitStore.
3142 // Accesses is sorted by block order, so the first element is the
3143 // earliest; checking it is enough.
3144 if (!InitStore->comesBefore(Other: Accesses.front().Inst))
3145 return std::nullopt;
3146 // Ordering constraint 2: when FullLoad shares the block with the
3147 // partial accesses, it must come after every one of them — otherwise
3148 // it could read a stale value. Accesses is sorted, so the last
3149 // element is the latest; checking it is enough. If FullLoad is in
3150 // another block, mem2reg forwards the merged store to it.
3151 if (FullLoad && FullLoad->getParent() == StoreBB &&
3152 !Accesses.back().Inst->comesBefore(Other: FullLoad))
3153 return std::nullopt;
3154
3155 // Coverage check: the distinct [begin, end) ranges touched by the
3156 // partial loads and stores must tile the alloca disjointly. That is
3157 // the only precondition the per-range SliceValues tracking below
3158 // needs — a disjoint tile guarantees the entries don't alias each
3159 // other. We don't check per-range load/store counts: a range with
3160 // only loads ends with SliceValues[r] = the init extract
3161 // (contributed to the final tree-merge), and a range with only
3162 // stores ends with SliceValues[r] = its last stored value. Both are
3163 // correct.
3164 using SliceRange = std::pair<uint64_t, uint64_t>;
3165 SmallVector<SliceRange, 8> SortedRanges;
3166 SortedRanges.reserve(N: Accesses.size());
3167 for (auto &Acc : Accesses)
3168 SortedRanges.emplace_back(Args&: Acc.BeginOffset, Args&: Acc.EndOffset);
3169 llvm::sort(C&: SortedRanges);
3170 SortedRanges.erase(CS: llvm::unique(R&: SortedRanges), CE: SortedRanges.end());
3171 // Disjoint + contiguous tile of the whole alloca.
3172 uint64_t Expected = NewAllocaBeginOffset;
3173 for (auto &Range : SortedRanges) {
3174 if (Range.first != Expected)
3175 return std::nullopt;
3176 Expected = Range.second;
3177 }
3178 if (Expected != NewAllocaEndOffset)
3179 return std::nullopt;
3180
3181 LLVM_DEBUG({
3182 dbgs() << "Tree structured merge rewrite (RMW):\n";
3183 dbgs() << " Init store: " << *InitStore << "\n";
3184 if (FullLoad)
3185 dbgs() << " Final load: " << *FullLoad << "\n";
3186 dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3187 for (auto &Range : SortedRanges)
3188 dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3189 });
3190
3191 // Initialize SliceValues: one SSA value per slice range, tracking
3192 // the value the alloca currently holds at that range. Each entry
3193 // starts at the corresponding piece of the init store, obtained by
3194 // bitcasting the init value to the alloca's vector type (if needed)
3195 // and extracting the slice's sub-range.
3196 IRB.SetInsertPoint(InitStore->getNextNode());
3197 Value *InitVec = InitStore->getValueOperand();
3198 if (InitVec->getType() != NewAllocaTy)
3199 InitVec = IRB.CreateBitCast(V: InitVec, DestTy: NewAllocaTy, Name: "init.cast");
3200 DenseMap<SliceRange, Value *> SliceValues;
3201 for (auto &Range : SortedRanges) {
3202 unsigned BeginIdx = getIndex(Offset: Range.first);
3203 unsigned EndIdx = getIndex(Offset: Range.second);
3204 SliceValues[Range] = IRB.CreateShuffleVector(
3205 V: InitVec, Mask: createSequentialMask(Start: BeginIdx, NumInts: EndIdx - BeginIdx, NumUndefs: 0),
3206 Name: "init.extract");
3207 }
3208 // The init store itself becomes dead — its value is consumed via the
3209 // extracts above.
3210 DeletedValues.push_back(Elt: InitStore);
3211
3212 // Walk accesses in block order:
3213 // - partial load at range r: replace with SliceValues[r] (bitcast
3214 // if the load's type differs from the current tracked value's
3215 // type, e.g. because a previous store wrote a vector with a
3216 // different element type);
3217 // - partial store at range r: update SliceValues[r] to the stored
3218 // value and drop the store.
3219 for (auto &Acc : Accesses) {
3220 SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3221 if (!Acc.IsStore) {
3222 Value *V = SliceValues[Range];
3223 if (V->getType() != Acc.Inst->getType()) {
3224 IRB.SetInsertPoint(cast<LoadInst>(Val: Acc.Inst));
3225 V = IRB.CreateBitCast(V, DestTy: Acc.Inst->getType());
3226 }
3227 Acc.Inst->replaceAllUsesWith(V);
3228 } else {
3229 SliceValues[Range] = cast<StoreInst>(Val: Acc.Inst)->getValueOperand();
3230 }
3231 DeletedValues.push_back(Elt: Acc.Inst);
3232 }
3233
3234 // Tree-merge the final per-range values (in range order) into the
3235 // alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3236 // shares the partial-access block) or otherwise to the block's
3237 // terminator — never to a partial access, since those are queued for
3238 // deletion. Both anchors are guaranteed to dominate every SliceValues
3239 // entry: each one is either an init extract (before any access) or a
3240 // stored value defined before its (now-deleted) store.
3241 IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3242 ? cast<Instruction>(Val: FullLoad)
3243 : StoreBB->getTerminator());
3244 SmallVector<Value *, 8> Vals;
3245 for (auto &Range : SortedRanges)
3246 Vals.push_back(Elt: SliceValues[Range]);
3247 Value *Merged = TreeMerge(Vals, Builder);
3248 Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3249
3250 // Replace the optional final full-width load with a load of the newly
3251 // merged alloca. Later promotion will forward the store above to it.
3252 if (FullLoad)
3253 ReplaceFullLoad(FullLoad, Merged);
3254
3255 return DeletedValues;
3256 }
3257
3258private:
3259 // Make sure the other visit overloads are visible.
3260 using Base::visit;
3261
3262 // Every instruction which can end up as a user must have a rewrite rule.
3263 bool visitInstruction(Instruction &I) {
3264 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3265 llvm_unreachable("No rewrite rule for this instruction!");
3266 }
3267
3268 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
3269 // Note that the offset computation can use BeginOffset or NewBeginOffset
3270 // interchangeably for unsplit slices.
3271 assert(IsSplit || BeginOffset == NewBeginOffset);
3272 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3273
3274 StringRef OldName = OldPtr->getName();
3275 // Skip through the last '.sroa.' component of the name.
3276 size_t LastSROAPrefix = OldName.rfind(Str: ".sroa.");
3277 if (LastSROAPrefix != StringRef::npos) {
3278 OldName = OldName.substr(Start: LastSROAPrefix + strlen(s: ".sroa."));
3279 // Look for an SROA slice index.
3280 size_t IndexEnd = OldName.find_first_not_of(Chars: "0123456789");
3281 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
3282 // Strip the index and look for the offset.
3283 OldName = OldName.substr(Start: IndexEnd + 1);
3284 size_t OffsetEnd = OldName.find_first_not_of(Chars: "0123456789");
3285 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
3286 // Strip the offset.
3287 OldName = OldName.substr(Start: OffsetEnd + 1);
3288 }
3289 }
3290 // Strip any SROA suffixes as well.
3291 OldName = OldName.substr(Start: 0, N: OldName.find(Str: ".sroa_"));
3292
3293 return getAdjustedPtr(IRB, DL, Ptr: &NewAI,
3294 Offset: APInt(DL.getIndexTypeSizeInBits(Ty: PointerTy), Offset),
3295 PointerTy, NamePrefix: Twine(OldName) + ".");
3296 }
3297
3298 /// Compute suitable alignment to access this slice of the *new*
3299 /// alloca.
3300 ///
3301 /// You can optionally pass a type to this routine and if that type's ABI
3302 /// alignment is itself suitable, this will return zero.
3303 Align getSliceAlign() {
3304 return commonAlignment(A: NewAI.getAlign(),
3305 Offset: NewBeginOffset - NewAllocaBeginOffset);
3306 }
3307
3308 unsigned getIndex(uint64_t Offset) {
3309 assert(VecTy && "Can only call getIndex when rewriting a vector");
3310 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3311 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3312 uint32_t Index = RelOffset / ElementSize;
3313 assert(Index * ElementSize == RelOffset);
3314 return Index;
3315 }
3316
3317 void deleteIfTriviallyDead(Value *V) {
3318 Instruction *I = cast<Instruction>(Val: V);
3319 if (isInstructionTriviallyDead(I))
3320 Pass.DeadInsts.push_back(Elt: I);
3321 }
3322
3323 Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3324 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3325 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3326 assert(EndIndex > BeginIndex && "Empty vector!");
3327
3328 LoadInst *Load =
3329 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3330
3331 Load->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3332 LLVMContext::MD_access_group});
3333 return extractVector(IRB, V: Load, BeginIndex, EndIndex, Name: "vec");
3334 }
3335
3336 Value *rewriteIntegerLoad(LoadInst &LI) {
3337 assert(IntTy && "We cannot insert an integer to the alloca");
3338 assert(!LI.isVolatile());
3339 Value *V =
3340 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3341 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: IntTy);
3342 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3343 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3344 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
3345 IntegerType *ExtractTy = Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8);
3346 V = extractInteger(DL, IRB, V, Ty: ExtractTy, Offset, Name: "extract");
3347 }
3348 // It is possible that the extracted type is not the load type. This
3349 // happens if there is a load past the end of the alloca, and as
3350 // a consequence the slice is narrower but still a candidate for integer
3351 // lowering. To handle this case, we just zero extend the extracted
3352 // integer.
3353 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
3354 "Can only handle an extract for an overly wide load");
3355 if (cast<IntegerType>(Val: LI.getType())->getBitWidth() > SliceSize * 8)
3356 V = IRB.CreateZExt(V, DestTy: LI.getType());
3357 return V;
3358 }
3359
3360 bool visitLoadInst(LoadInst &LI) {
3361 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3362 Value *OldOp = LI.getOperand(i_nocapture: 0);
3363 assert(OldOp == OldPtr);
3364
3365 AAMDNodes AATags = LI.getAAMetadata();
3366
3367 unsigned AS = LI.getPointerAddressSpace();
3368
3369 Type *TargetTy = IsSplit ? Type::getIntNTy(C&: LI.getContext(), N: SliceSize * 8)
3370 : LI.getType();
3371 bool IsPtrAdjusted = false;
3372 Value *V;
3373 if (VecTy) {
3374 V = rewriteVectorizedLoadInst(LI);
3375 } else if (IntTy && LI.getType()->isIntegerTy()) {
3376 V = rewriteIntegerLoad(LI);
3377 } else if (NewBeginOffset == NewAllocaBeginOffset &&
3378 NewEndOffset == NewAllocaEndOffset &&
3379 (canConvertValue(DL, OldTy: NewAllocaTy, NewTy: TargetTy) ||
3380 (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3381 DL.getTypeStoreSize(Ty: TargetTy).getFixedValue() > SliceSize &&
3382 !LI.isVolatile()))) {
3383 Value *NewPtr =
3384 getPtrToNewAI(AddrSpace: LI.getPointerAddressSpace(), IsVolatile: LI.isVolatile());
3385 LoadInst *NewLI = IRB.CreateAlignedLoad(
3386 Ty: NewAllocaTy, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3387 if (LI.isVolatile())
3388 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3389 if (NewLI->isAtomic())
3390 NewLI->setAlignment(LI.getAlign());
3391
3392 // Copy any metadata that is valid for the new load. This may require
3393 // conversion to a different kind of metadata, e.g. !nonnull might change
3394 // to !range or vice versa.
3395 copyMetadataForLoad(Dest&: *NewLI, Source: LI);
3396
3397 // Do this after copyMetadataForLoad() to preserve the TBAA shift.
3398 if (AATags)
3399 NewLI->setAAMetadata(AATags.adjustForAccess(
3400 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3401
3402 // Try to preserve nonnull metadata
3403 V = NewLI;
3404
3405 // If this is an integer load past the end of the slice (which means the
3406 // bytes outside the slice are undef or this load is dead) just forcibly
3407 // fix the integer size with correct handling of endianness.
3408 if (auto *AITy = dyn_cast<IntegerType>(Val: NewAllocaTy))
3409 if (auto *TITy = dyn_cast<IntegerType>(Val: TargetTy))
3410 if (AITy->getBitWidth() < TITy->getBitWidth()) {
3411 V = IRB.CreateZExt(V, DestTy: TITy, Name: "load.ext");
3412 if (DL.isBigEndian())
3413 V = IRB.CreateShl(LHS: V, RHS: TITy->getBitWidth() - AITy->getBitWidth(),
3414 Name: "endian_shift");
3415 }
3416 } else {
3417 Type *LTy = IRB.getPtrTy(AddrSpace: AS);
3418 LoadInst *NewLI =
3419 IRB.CreateAlignedLoad(Ty: TargetTy, Ptr: getNewAllocaSlicePtr(IRB, PointerTy: LTy),
3420 Align: getSliceAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3421
3422 if (AATags)
3423 NewLI->setAAMetadata(AATags.adjustForAccess(
3424 Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3425
3426 if (LI.isVolatile())
3427 NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3428 NewLI->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3429 LLVMContext::MD_access_group});
3430
3431 V = NewLI;
3432 IsPtrAdjusted = true;
3433 }
3434 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: TargetTy);
3435
3436 if (IsSplit) {
3437 assert(!LI.isVolatile());
3438 assert(LI.getType()->isIntegerTy() &&
3439 "Only integer type loads and stores are split");
3440 assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3441 "Split load isn't smaller than original load");
3442 assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3443 "Non-byte-multiple bit width");
3444 // Move the insertion point just past the load so that we can refer to it.
3445 BasicBlock::iterator LIIt = std::next(x: LI.getIterator());
3446 // Ensure the insertion point comes before any debug-info immediately
3447 // after the load, so that variable values referring to the load are
3448 // dominated by it.
3449 LIIt.setHeadBit(true);
3450 IRB.SetInsertPoint(TheBB: LI.getParent(), IP: LIIt);
3451 // Create a placeholder value with the same type as LI to use as the
3452 // basis for the new value. This allows us to replace the uses of LI with
3453 // the computed value, and then replace the placeholder with LI, leaving
3454 // LI only used for this computation.
3455 Value *Placeholder =
3456 new LoadInst(LI.getType(), PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AS)), "",
3457 false, Align(1));
3458 V = insertInteger(DL, IRB, Old: Placeholder, V, Offset: NewBeginOffset - BeginOffset,
3459 Name: "insert");
3460 LI.replaceAllUsesWith(V);
3461 Placeholder->replaceAllUsesWith(V: &LI);
3462 Placeholder->deleteValue();
3463 } else {
3464 LI.replaceAllUsesWith(V);
3465 }
3466
3467 Pass.DeadInsts.push_back(Elt: &LI);
3468 deleteIfTriviallyDead(V: OldOp);
3469 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3470 return !LI.isVolatile() && !IsPtrAdjusted;
3471 }
3472
3473 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
3474 AAMDNodes AATags) {
3475 // Capture V for the purpose of debug-info accounting once it's converted
3476 // to a vector store.
3477 Value *OrigV = V;
3478 if (V->getType() != VecTy) {
3479 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3480 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3481 assert(EndIndex > BeginIndex && "Empty vector!");
3482 unsigned NumElements = EndIndex - BeginIndex;
3483 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3484 "Too many elements!");
3485 Type *SliceTy = (NumElements == 1)
3486 ? ElementTy
3487 : FixedVectorType::get(ElementType: ElementTy, NumElts: NumElements);
3488 if (V->getType() != SliceTy)
3489 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: SliceTy);
3490
3491 // Mix in the existing elements.
3492 Value *Old =
3493 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3494 V = insertVector(IRB, Old, V, BeginIndex, Name: "vec");
3495 }
3496 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3497 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3498 LLVMContext::MD_access_group});
3499 if (AATags)
3500 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3501 AccessTy: V->getType(), DL));
3502 Pass.DeadInsts.push_back(Elt: &SI);
3503
3504 // NOTE: Careful to use OrigV rather than V.
3505 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3506 Inst: Store, Dest: Store->getPointerOperand(), Value: OrigV, DL);
3507 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3508 return true;
3509 }
3510
3511 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3512 assert(IntTy && "We cannot extract an integer from the alloca");
3513 assert(!SI.isVolatile());
3514 if (DL.getTypeSizeInBits(Ty: V->getType()).getFixedValue() !=
3515 IntTy->getBitWidth()) {
3516 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3517 Name: "oldload");
3518 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3519 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3520 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3521 V = insertInteger(DL, IRB, Old, V: SI.getValueOperand(), Offset, Name: "insert");
3522 }
3523 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3524 StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3525 Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3526 LLVMContext::MD_access_group});
3527 if (AATags)
3528 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3529 AccessTy: V->getType(), DL));
3530
3531 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3532 Inst: Store, Dest: Store->getPointerOperand(),
3533 Value: Store->getValueOperand(), DL);
3534
3535 Pass.DeadInsts.push_back(Elt: &SI);
3536 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3537 return true;
3538 }
3539
3540 bool visitStoreInst(StoreInst &SI) {
3541 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3542 Value *OldOp = SI.getOperand(i_nocapture: 1);
3543 assert(OldOp == OldPtr);
3544
3545 AAMDNodes AATags = SI.getAAMetadata();
3546 Value *V = SI.getValueOperand();
3547
3548 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3549 // alloca that should be re-examined after promoting this alloca.
3550 if (V->getType()->isPointerTy())
3551 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V->stripInBoundsOffsets()))
3552 Pass.PostPromotionWorklist.insert(X: AI);
3553
3554 TypeSize StoreSize = DL.getTypeStoreSize(Ty: V->getType());
3555 if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3556 assert(!SI.isVolatile());
3557 assert(V->getType()->isIntegerTy() &&
3558 "Only integer type loads and stores are split");
3559 assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3560 "Non-byte-multiple bit width");
3561 IntegerType *NarrowTy = Type::getIntNTy(C&: SI.getContext(), N: SliceSize * 8);
3562 V = extractInteger(DL, IRB, V, Ty: NarrowTy, Offset: NewBeginOffset - BeginOffset,
3563 Name: "extract");
3564 }
3565
3566 if (VecTy)
3567 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3568 if (IntTy && V->getType()->isIntegerTy())
3569 return rewriteIntegerStore(V, SI, AATags);
3570
3571 StoreInst *NewSI;
3572 if (NewBeginOffset == NewAllocaBeginOffset &&
3573 NewEndOffset == NewAllocaEndOffset &&
3574 canConvertValue(DL, OldTy: V->getType(), NewTy: NewAllocaTy)) {
3575 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3576 Value *NewPtr =
3577 getPtrToNewAI(AddrSpace: SI.getPointerAddressSpace(), IsVolatile: SI.isVolatile());
3578
3579 NewSI =
3580 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: SI.isVolatile());
3581 } else {
3582 unsigned AS = SI.getPointerAddressSpace();
3583 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: IRB.getPtrTy(AddrSpace: AS));
3584 NewSI =
3585 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: getSliceAlign(), isVolatile: SI.isVolatile());
3586 }
3587 NewSI->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3588 LLVMContext::MD_access_group});
3589 if (AATags)
3590 NewSI->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3591 AccessTy: V->getType(), DL));
3592 if (SI.isVolatile())
3593 NewSI->setAtomic(Ordering: SI.getOrdering(), SSID: SI.getSyncScopeID());
3594 if (NewSI->isAtomic())
3595 NewSI->setAlignment(SI.getAlign());
3596
3597 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &SI,
3598 Inst: NewSI, Dest: NewSI->getPointerOperand(),
3599 Value: NewSI->getValueOperand(), DL);
3600
3601 Pass.DeadInsts.push_back(Elt: &SI);
3602 deleteIfTriviallyDead(V: OldOp);
3603
3604 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3605 return NewSI->getPointerOperand() == &NewAI &&
3606 NewSI->getValueOperand()->getType() == NewAllocaTy &&
3607 !SI.isVolatile();
3608 }
3609
3610 /// Compute an integer value from splatting an i8 across the given
3611 /// number of bytes.
3612 ///
3613 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3614 /// call this routine.
3615 /// FIXME: Heed the advice above.
3616 ///
3617 /// \param V The i8 value to splat.
3618 /// \param Size The number of bytes in the output (assuming i8 is one byte)
3619 Value *getIntegerSplat(Value *V, unsigned Size) {
3620 assert(Size > 0 && "Expected a positive number of bytes.");
3621 IntegerType *VTy = cast<IntegerType>(Val: V->getType());
3622 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
3623 if (Size == 1)
3624 return V;
3625
3626 Type *SplatIntTy = Type::getIntNTy(C&: VTy->getContext(), N: Size * 8);
3627 V = IRB.CreateMul(
3628 LHS: IRB.CreateZExt(V, DestTy: SplatIntTy, Name: "zext"),
3629 RHS: IRB.CreateUDiv(LHS: Constant::getAllOnesValue(Ty: SplatIntTy),
3630 RHS: IRB.CreateZExt(V: Constant::getAllOnesValue(Ty: V->getType()),
3631 DestTy: SplatIntTy)),
3632 Name: "isplat");
3633 return V;
3634 }
3635
3636 /// Compute a vector splat for a given element value.
3637 Value *getVectorSplat(Value *V, unsigned NumElements) {
3638 V = IRB.CreateVectorSplat(NumElts: NumElements, V, Name: "vsplat");
3639 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3640 return V;
3641 }
3642
3643 bool visitMemSetInst(MemSetInst &II) {
3644 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3645 assert(II.getRawDest() == OldPtr);
3646
3647 AAMDNodes AATags = II.getAAMetadata();
3648
3649 // If the memset has a variable size, it cannot be split, just adjust the
3650 // pointer to the new alloca.
3651 if (!isa<ConstantInt>(Val: II.getLength())) {
3652 assert(!IsSplit);
3653 assert(NewBeginOffset == BeginOffset);
3654 II.setDest(getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()));
3655 II.setDestAlignment(getSliceAlign());
3656 // In theory we should call migrateDebugInfo here. However, we do not
3657 // emit dbg.assign intrinsics for mem intrinsics storing through non-
3658 // constant geps, or storing a variable number of bytes.
3659 assert(at::getDVRAssignmentMarkers(&II).empty() &&
3660 "AT: Unexpected link to non-const GEP");
3661 deleteIfTriviallyDead(V: OldPtr);
3662 return false;
3663 }
3664
3665 // Record this instruction for deletion.
3666 Pass.DeadInsts.push_back(Elt: &II);
3667
3668 Type *ScalarTy = NewAllocaTy->getScalarType();
3669
3670 const bool CanContinue = [&]() {
3671 if (VecTy || IntTy)
3672 return true;
3673 if (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset)
3674 return false;
3675 // Length must be in range for FixedVectorType.
3676 auto *C = cast<ConstantInt>(Val: II.getLength());
3677 const uint64_t Len = C->getLimitedValue();
3678 if (Len > std::numeric_limits<unsigned>::max())
3679 return false;
3680 auto *Int8Ty = IntegerType::getInt8Ty(C&: NewAI.getContext());
3681 auto *SrcTy = FixedVectorType::get(ElementType: Int8Ty, NumElts: Len);
3682 return canConvertValue(DL, OldTy: SrcTy, NewTy: NewAllocaTy) &&
3683 DL.isLegalInteger(Width: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue());
3684 }();
3685
3686 // If this doesn't map cleanly onto the alloca type, and that type isn't
3687 // a single value type, just emit a memset.
3688 if (!CanContinue) {
3689 Type *SizeTy = II.getLength()->getType();
3690 unsigned Sz = NewEndOffset - NewBeginOffset;
3691 Constant *Size = ConstantInt::get(Ty: SizeTy, V: Sz);
3692 MemIntrinsic *New = cast<MemIntrinsic>(Val: IRB.CreateMemSet(
3693 Ptr: getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()), Val: II.getValue(), Size,
3694 Align: MaybeAlign(getSliceAlign()), isVolatile: II.isVolatile()));
3695 if (AATags)
3696 New->setAAMetadata(
3697 AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset, AccessSize: Sz));
3698
3699 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3700 Inst: New, Dest: New->getRawDest(), Value: nullptr, DL);
3701
3702 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3703 return false;
3704 }
3705
3706 // If we can represent this as a simple value, we have to build the actual
3707 // value to store, which requires expanding the byte present in memset to
3708 // a sensible representation for the alloca type. This is essentially
3709 // splatting the byte to a sufficiently wide integer, splatting it across
3710 // any desired vector width, and bitcasting to the final type.
3711 Value *V;
3712
3713 if (VecTy) {
3714 // If this is a memset of a vectorized alloca, insert it.
3715 assert(ElementTy == ScalarTy);
3716
3717 unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3718 unsigned EndIndex = getIndex(Offset: NewEndOffset);
3719 assert(EndIndex > BeginIndex && "Empty vector!");
3720 unsigned NumElements = EndIndex - BeginIndex;
3721 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3722 "Too many elements!");
3723
3724 Value *Splat = getIntegerSplat(
3725 V: II.getValue(), Size: DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / 8);
3726 Splat = IRB.CreateBitPreservingCastChain(DL, V: Splat, NewTy: ElementTy);
3727 if (NumElements > 1)
3728 Splat = getVectorSplat(V: Splat, NumElements);
3729
3730 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3731 Name: "oldload");
3732 V = insertVector(IRB, Old, V: Splat, BeginIndex, Name: "vec");
3733 } else if (IntTy) {
3734 // If this is a memset on an alloca where we can widen stores, insert the
3735 // set integer.
3736 assert(!II.isVolatile());
3737
3738 uint64_t Size = NewEndOffset - NewBeginOffset;
3739 V = getIntegerSplat(V: II.getValue(), Size);
3740
3741 if (IntTy && (NewBeginOffset != NewAllocaBeginOffset ||
3742 NewEndOffset != NewAllocaEndOffset)) {
3743 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI,
3744 Align: NewAI.getAlign(), Name: "oldload");
3745 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3746 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3747 V = insertInteger(DL, IRB, Old, V, Offset, Name: "insert");
3748 } else {
3749 assert(V->getType() == IntTy &&
3750 "Wrong type for an alloca wide integer!");
3751 }
3752 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3753 } else {
3754 // Established these invariants above.
3755 assert(NewBeginOffset == NewAllocaBeginOffset);
3756 assert(NewEndOffset == NewAllocaEndOffset);
3757
3758 V = getIntegerSplat(V: II.getValue(),
3759 Size: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue() / 8);
3760 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(Val: NewAllocaTy))
3761 V = getVectorSplat(
3762 V, NumElements: cast<FixedVectorType>(Val: AllocaVecTy)->getNumElements());
3763
3764 V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3765 }
3766
3767 Value *NewPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3768 StoreInst *New =
3769 IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: II.isVolatile());
3770 New->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3771 LLVMContext::MD_access_group});
3772 if (AATags)
3773 New->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3774 AccessTy: V->getType(), DL));
3775
3776 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
3777 Inst: New, Dest: New->getPointerOperand(), Value: V, DL);
3778
3779 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3780 return !II.isVolatile();
3781 }
3782
3783 bool visitMemTransferInst(MemTransferInst &II) {
3784 // Rewriting of memory transfer instructions can be a bit tricky. We break
3785 // them into two categories: split intrinsics and unsplit intrinsics.
3786
3787 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3788
3789 AAMDNodes AATags = II.getAAMetadata();
3790
3791 bool IsDest = &II.getRawDestUse() == OldUse;
3792 assert((IsDest && II.getRawDest() == OldPtr) ||
3793 (!IsDest && II.getRawSource() == OldPtr));
3794
3795 Align SliceAlign = getSliceAlign();
3796 // For unsplit intrinsics, we simply modify the source and destination
3797 // pointers in place. This isn't just an optimization, it is a matter of
3798 // correctness. With unsplit intrinsics we may be dealing with transfers
3799 // within a single alloca before SROA ran, or with transfers that have
3800 // a variable length. We may also be dealing with memmove instead of
3801 // memcpy, and so simply updating the pointers is the necessary for us to
3802 // update both source and dest of a single call.
3803 if (!IsSplittable) {
3804 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3805 if (IsDest) {
3806 // Update the address component of linked dbg.assigns.
3807 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(Inst: &II)) {
3808 if (llvm::is_contained(Range: DbgAssign->location_ops(), Element: II.getDest()) ||
3809 DbgAssign->getAddress() == II.getDest())
3810 DbgAssign->replaceVariableLocationOp(OldValue: II.getDest(), NewValue: AdjustedPtr);
3811 }
3812 II.setDest(AdjustedPtr);
3813 II.setDestAlignment(SliceAlign);
3814 } else {
3815 II.setSource(AdjustedPtr);
3816 II.setSourceAlignment(SliceAlign);
3817 }
3818
3819 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3820 deleteIfTriviallyDead(V: OldPtr);
3821 return false;
3822 }
3823 // For split transfer intrinsics we have an incredibly useful assurance:
3824 // the source and destination do not reside within the same alloca, and at
3825 // least one of them does not escape. This means that we can replace
3826 // memmove with memcpy, and we don't need to worry about all manner of
3827 // downsides to splitting and transforming the operations.
3828
3829 // If this doesn't map cleanly onto the alloca type, and that type isn't
3830 // a single value type, just emit a memcpy.
3831 bool EmitMemCpy =
3832 !VecTy && !IntTy &&
3833 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
3834 SliceSize != DL.getTypeStoreSize(Ty: NewAllocaTy).getFixedValue() ||
3835 !DL.typeSizeEqualsStoreSize(Ty: NewAllocaTy) ||
3836 !NewAllocaTy->isSingleValueType());
3837
3838 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
3839 // size hasn't been shrunk based on analysis of the viable range, this is
3840 // a no-op.
3841 if (EmitMemCpy && &OldAI == &NewAI) {
3842 // Ensure the start lines up.
3843 assert(NewBeginOffset == BeginOffset);
3844
3845 // Rewrite the size as needed.
3846 if (NewEndOffset != EndOffset)
3847 II.setLength(NewEndOffset - NewBeginOffset);
3848 return false;
3849 }
3850 // Record this instruction for deletion.
3851 Pass.DeadInsts.push_back(Elt: &II);
3852
3853 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3854 // alloca that should be re-examined after rewriting this instruction.
3855 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
3856 if (AllocaInst *AI =
3857 dyn_cast<AllocaInst>(Val: OtherPtr->stripInBoundsOffsets())) {
3858 assert(AI != &OldAI && AI != &NewAI &&
3859 "Splittable transfers cannot reach the same alloca on both ends.");
3860 Pass.Worklist.insert(X: AI);
3861 }
3862
3863 Type *OtherPtrTy = OtherPtr->getType();
3864 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
3865
3866 // Compute the relative offset for the other pointer within the transfer.
3867 unsigned OffsetWidth = DL.getIndexSizeInBits(AS: OtherAS);
3868 APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
3869 Align OtherAlign =
3870 (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
3871 OtherAlign =
3872 commonAlignment(A: OtherAlign, Offset: OtherOffset.zextOrTrunc(width: 64).getZExtValue());
3873
3874 if (EmitMemCpy) {
3875 // Compute the other pointer, folding as much as possible to produce
3876 // a single, simple GEP in most cases.
3877 OtherPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
3878 NamePrefix: OtherPtr->getName() + ".");
3879
3880 Value *OurPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3881 Type *SizeTy = II.getLength()->getType();
3882 Constant *Size = ConstantInt::get(Ty: SizeTy, V: NewEndOffset - NewBeginOffset);
3883
3884 Value *DestPtr, *SrcPtr;
3885 MaybeAlign DestAlign, SrcAlign;
3886 // Note: IsDest is true iff we're copying into the new alloca slice
3887 if (IsDest) {
3888 DestPtr = OurPtr;
3889 DestAlign = SliceAlign;
3890 SrcPtr = OtherPtr;
3891 SrcAlign = OtherAlign;
3892 } else {
3893 DestPtr = OtherPtr;
3894 DestAlign = OtherAlign;
3895 SrcPtr = OurPtr;
3896 SrcAlign = SliceAlign;
3897 }
3898 CallInst *New = IRB.CreateMemCpy(Dst: DestPtr, DstAlign: DestAlign, Src: SrcPtr, SrcAlign,
3899 Size, isVolatile: II.isVolatile());
3900 if (AATags)
3901 New->setAAMetadata(AATags.shift(Offset: NewBeginOffset - BeginOffset));
3902
3903 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DestPtr->getType()), 0);
3904 if (IsDest) {
3905 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8,
3906 OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
3907 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
3908 Val: DestPtr->stripAndAccumulateConstantOffsets(
3909 DL, Offset, /*AllowNonInbounds*/ true))) {
3910 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
3911 SliceSizeInBits: SliceSize * 8, OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
3912 }
3913 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3914 return false;
3915 }
3916
3917 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
3918 NewEndOffset == NewAllocaEndOffset;
3919 uint64_t Size = NewEndOffset - NewBeginOffset;
3920 unsigned BeginIndex = VecTy ? getIndex(Offset: NewBeginOffset) : 0;
3921 unsigned EndIndex = VecTy ? getIndex(Offset: NewEndOffset) : 0;
3922 unsigned NumElements = EndIndex - BeginIndex;
3923 IntegerType *SubIntTy =
3924 IntTy ? Type::getIntNTy(C&: IntTy->getContext(), N: Size * 8) : nullptr;
3925
3926 // Reset the other pointer type to match the register type we're going to
3927 // use, but using the address space of the original other pointer.
3928 Type *OtherTy;
3929 if (VecTy && !IsWholeAlloca) {
3930 if (NumElements == 1)
3931 OtherTy = VecTy->getElementType();
3932 else
3933 OtherTy = FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: NumElements);
3934 } else if (IntTy && !IsWholeAlloca) {
3935 OtherTy = SubIntTy;
3936 } else {
3937 OtherTy = NewAllocaTy;
3938 }
3939
3940 Value *AdjPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
3941 NamePrefix: OtherPtr->getName() + ".");
3942 MaybeAlign SrcAlign = OtherAlign;
3943 MaybeAlign DstAlign = SliceAlign;
3944 if (!IsDest)
3945 std::swap(a&: SrcAlign, b&: DstAlign);
3946
3947 Value *SrcPtr;
3948 Value *DstPtr;
3949
3950 if (IsDest) {
3951 DstPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3952 SrcPtr = AdjPtr;
3953 } else {
3954 DstPtr = AdjPtr;
3955 SrcPtr = getPtrToNewAI(AddrSpace: II.getSourceAddressSpace(), IsVolatile: II.isVolatile());
3956 }
3957
3958 Value *Src;
3959 if (VecTy && !IsWholeAlloca && !IsDest) {
3960 Src =
3961 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3962 Src = extractVector(IRB, V: Src, BeginIndex, EndIndex, Name: "vec");
3963 } else if (IntTy && !IsWholeAlloca && !IsDest) {
3964 Src =
3965 IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3966 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: IntTy);
3967 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3968 Src = extractInteger(DL, IRB, V: Src, Ty: SubIntTy, Offset, Name: "extract");
3969 } else {
3970 LoadInst *Load = IRB.CreateAlignedLoad(Ty: OtherTy, Ptr: SrcPtr, Align: SrcAlign,
3971 isVolatile: II.isVolatile(), Name: "copyload");
3972 Load->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3973 LLVMContext::MD_access_group});
3974 if (AATags)
3975 Load->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3976 AccessTy: Load->getType(), DL));
3977 Src = Load;
3978 }
3979
3980 if (VecTy && !IsWholeAlloca && IsDest) {
3981 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3982 Name: "oldload");
3983 Src = insertVector(IRB, Old, V: Src, BeginIndex, Name: "vec");
3984 } else if (IntTy && !IsWholeAlloca && IsDest) {
3985 Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3986 Name: "oldload");
3987 Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3988 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3989 Src = insertInteger(DL, IRB, Old, V: Src, Offset, Name: "insert");
3990 Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: NewAllocaTy);
3991 }
3992
3993 StoreInst *Store = cast<StoreInst>(
3994 Val: IRB.CreateAlignedStore(Val: Src, Ptr: DstPtr, Align: DstAlign, isVolatile: II.isVolatile()));
3995 Store->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3996 LLVMContext::MD_access_group});
3997 if (AATags)
3998 Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3999 AccessTy: Src->getType(), DL));
4000
4001 APInt Offset(DL.getIndexTypeSizeInBits(Ty: DstPtr->getType()), 0);
4002 if (IsDest) {
4003
4004 migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * 8, SliceSizeInBits: SliceSize * 8, OldInst: &II,
4005 Inst: Store, Dest: DstPtr, Value: Src, DL);
4006 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4007 Val: DstPtr->stripAndAccumulateConstantOffsets(
4008 DL, Offset, /*AllowNonInbounds*/ true))) {
4009 migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * 8, SliceSizeInBits: SliceSize * 8,
4010 OldInst: &II, Inst: Store, Dest: DstPtr, Value: Src, DL);
4011 }
4012
4013 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4014 return !II.isVolatile();
4015 }
4016
4017 bool visitIntrinsicInst(IntrinsicInst &II) {
4018 assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
4019 "Unexpected intrinsic!");
4020 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4021
4022 // Record this instruction for deletion.
4023 Pass.DeadInsts.push_back(Elt: &II);
4024
4025 if (II.isDroppable()) {
4026 assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4027 // TODO For now we forget assumed information, this can be improved.
4028 OldPtr->dropDroppableUsesIn(Usr&: II);
4029 return true;
4030 }
4031
4032 assert(II.getArgOperand(0) == OldPtr);
4033 Type *PointerTy = IRB.getPtrTy(AddrSpace: OldPtr->getType()->getPointerAddressSpace());
4034 Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4035 Value *New;
4036 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4037 New = IRB.CreateLifetimeStart(Ptr);
4038 else
4039 New = IRB.CreateLifetimeEnd(Ptr);
4040
4041 (void)New;
4042 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4043
4044 return true;
4045 }
4046
4047 void fixLoadStoreAlign(Instruction &Root) {
4048 // This algorithm implements the same visitor loop as
4049 // hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4050 // or store found.
4051 SmallPtrSet<Instruction *, 4> Visited;
4052 SmallVector<Instruction *, 4> Uses;
4053 Visited.insert(Ptr: &Root);
4054 Uses.push_back(Elt: &Root);
4055 do {
4056 Instruction *I = Uses.pop_back_val();
4057
4058 if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
4059 LI->setAlignment(std::min(a: LI->getAlign(), b: getSliceAlign()));
4060 continue;
4061 }
4062 if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
4063 SI->setAlignment(std::min(a: SI->getAlign(), b: getSliceAlign()));
4064 continue;
4065 }
4066
4067 assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||
4068 isa<PHINode>(I) || isa<SelectInst>(I) ||
4069 isa<GetElementPtrInst>(I));
4070 for (User *U : I->users())
4071 if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
4072 Uses.push_back(Elt: cast<Instruction>(Val: U));
4073 } while (!Uses.empty());
4074 }
4075
4076 bool visitPHINode(PHINode &PN) {
4077 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4078 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4079 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4080
4081 // We would like to compute a new pointer in only one place, but have it be
4082 // as local as possible to the PHI. To do that, we re-use the location of
4083 // the old pointer, which necessarily must be in the right position to
4084 // dominate the PHI.
4085 IRBuilderBase::InsertPointGuard Guard(IRB);
4086 if (isa<PHINode>(Val: OldPtr))
4087 IRB.SetInsertPoint(TheBB: OldPtr->getParent(),
4088 IP: OldPtr->getParent()->getFirstInsertionPt());
4089 else
4090 IRB.SetInsertPoint(OldPtr);
4091 IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4092
4093 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4094 // Replace the operands which were using the old pointer.
4095 std::replace(first: PN.op_begin(), last: PN.op_end(), old_value: cast<Value>(Val: OldPtr), new_value: NewPtr);
4096
4097 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4098 deleteIfTriviallyDead(V: OldPtr);
4099
4100 // Fix the alignment of any loads or stores using this PHI node.
4101 fixLoadStoreAlign(Root&: PN);
4102
4103 // PHIs can't be promoted on their own, but often can be speculated. We
4104 // check the speculation outside of the rewriter so that we see the
4105 // fully-rewritten alloca.
4106 PHIUsers.insert(X: &PN);
4107 return true;
4108 }
4109
4110 bool visitSelectInst(SelectInst &SI) {
4111 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4112 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
4113 "Pointer isn't an operand!");
4114 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4115 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4116
4117 Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4118 // Replace the operands which were using the old pointer.
4119 if (SI.getOperand(i_nocapture: 1) == OldPtr)
4120 SI.setOperand(i_nocapture: 1, Val_nocapture: NewPtr);
4121 if (SI.getOperand(i_nocapture: 2) == OldPtr)
4122 SI.setOperand(i_nocapture: 2, Val_nocapture: NewPtr);
4123
4124 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4125 deleteIfTriviallyDead(V: OldPtr);
4126
4127 // Fix the alignment of any loads or stores using this select.
4128 fixLoadStoreAlign(Root&: SI);
4129
4130 // Selects can't be promoted on their own, but often can be speculated. We
4131 // check the speculation outside of the rewriter so that we see the
4132 // fully-rewritten alloca.
4133 SelectUsers.insert(X: &SI);
4134 return true;
4135 }
4136};
4137
4138/// Visitor to rewrite aggregate loads and stores as scalar.
4139///
4140/// This pass aggressively rewrites all aggregate loads and stores on
4141/// a particular pointer (or any pointer derived from it which we can identify)
4142/// with scalar loads and stores.
4143class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4144 // Befriend the base class so it can delegate to private visit methods.
4145 friend class InstVisitor<AggLoadStoreRewriter, bool>;
4146
4147 /// Queue of pointer uses to analyze and potentially rewrite.
4148 SmallVector<Use *, 8> Queue;
4149
4150 /// Set to prevent us from cycling with phi nodes and loops.
4151 SmallPtrSet<User *, 8> Visited;
4152
4153 /// The current pointer use being rewritten. This is used to dig up the used
4154 /// value (as opposed to the user).
4155 Use *U = nullptr;
4156
4157 /// Used to calculate offsets, and hence alignment, of subobjects.
4158 const DataLayout &DL;
4159
4160 IRBuilderTy &IRB;
4161
4162public:
4163 AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4164 : DL(DL), IRB(IRB) {}
4165
4166 /// Rewrite loads and stores through a pointer and all pointers derived from
4167 /// it.
4168 bool rewrite(Instruction &I) {
4169 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4170 enqueueUsers(I);
4171 bool Changed = false;
4172 while (!Queue.empty()) {
4173 U = Queue.pop_back_val();
4174 Changed |= visit(I: cast<Instruction>(Val: U->getUser()));
4175 }
4176 return Changed;
4177 }
4178
4179private:
4180 /// Enqueue all the users of the given instruction for further processing.
4181 /// This uses a set to de-duplicate users.
4182 void enqueueUsers(Instruction &I) {
4183 for (Use &U : I.uses())
4184 if (Visited.insert(Ptr: U.getUser()).second)
4185 Queue.push_back(Elt: &U);
4186 }
4187
4188 // Conservative default is to not rewrite anything.
4189 bool visitInstruction(Instruction &I) { return false; }
4190
4191 /// Generic recursive split emission class.
4192 template <typename Derived> class OpSplitter {
4193 protected:
4194 /// The builder used to form new instructions.
4195 IRBuilderTy &IRB;
4196
4197 /// The indices which to be used with insert- or extractvalue to select the
4198 /// appropriate value within the aggregate.
4199 SmallVector<unsigned, 4> Indices;
4200
4201 /// The indices to a GEP instruction which will move Ptr to the correct slot
4202 /// within the aggregate.
4203 SmallVector<Value *, 4> GEPIndices;
4204
4205 /// The base pointer of the original op, used as a base for GEPing the
4206 /// split operations.
4207 Value *Ptr;
4208
4209 /// The base pointee type being GEPed into.
4210 Type *BaseTy;
4211
4212 /// Known alignment of the base pointer.
4213 Align BaseAlign;
4214
4215 /// To calculate offset of each component so we can correctly deduce
4216 /// alignments.
4217 const DataLayout &DL;
4218
4219 /// Initialize the splitter with an insertion point, Ptr and start with a
4220 /// single zero GEP index.
4221 OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4222 Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4223 : IRB(IRB), GEPIndices(1, IRB.getInt32(C: 0)), Ptr(Ptr), BaseTy(BaseTy),
4224 BaseAlign(BaseAlign), DL(DL) {
4225 IRB.SetInsertPoint(InsertionPoint);
4226 }
4227
4228 public:
4229 /// Generic recursive split emission routine.
4230 ///
4231 /// This method recursively splits an aggregate op (load or store) into
4232 /// scalar or vector ops. It splits recursively until it hits a single value
4233 /// and emits that single value operation via the template argument.
4234 ///
4235 /// The logic of this routine relies on GEPs and insertvalue and
4236 /// extractvalue all operating with the same fundamental index list, merely
4237 /// formatted differently (GEPs need actual values).
4238 ///
4239 /// \param Ty The type being split recursively into smaller ops.
4240 /// \param Agg The aggregate value being built up or stored, depending on
4241 /// whether this is splitting a load or a store respectively.
4242 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
4243 if (Ty->isSingleValueType()) {
4244 unsigned Offset = DL.getIndexedOffsetInType(ElemTy: BaseTy, Indices: GEPIndices);
4245 return static_cast<Derived *>(this)->emitFunc(
4246 Ty, Agg, commonAlignment(A: BaseAlign, Offset), Name);
4247 }
4248
4249 if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
4250 unsigned OldSize = Indices.size();
4251 (void)OldSize;
4252 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
4253 ++Idx) {
4254 assert(Indices.size() == OldSize && "Did not return to the old size");
4255 Indices.push_back(Elt: Idx);
4256 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4257 emitSplitOps(Ty: ATy->getElementType(), Agg, Name: Name + "." + Twine(Idx));
4258 GEPIndices.pop_back();
4259 Indices.pop_back();
4260 }
4261 return;
4262 }
4263
4264 if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4265 unsigned OldSize = Indices.size();
4266 (void)OldSize;
4267 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
4268 ++Idx) {
4269 assert(Indices.size() == OldSize && "Did not return to the old size");
4270 Indices.push_back(Elt: Idx);
4271 GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4272 emitSplitOps(Ty: STy->getElementType(N: Idx), Agg, Name: Name + "." + Twine(Idx));
4273 GEPIndices.pop_back();
4274 Indices.pop_back();
4275 }
4276 return;
4277 }
4278
4279 llvm_unreachable("Only arrays and structs are aggregate loadable types");
4280 }
4281 };
4282
4283 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4284 AAMDNodes AATags;
4285 // A vector to hold the split components that we want to emit
4286 // separate fake uses for.
4287 SmallVector<Value *, 4> Components;
4288 // A vector to hold all the fake uses of the struct that we are splitting.
4289 // Usually there should only be one, but we are handling the general case.
4290 SmallVector<Instruction *, 1> FakeUses;
4291
4292 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4293 AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4294 IRBuilderTy &IRB)
4295 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4296 IRB),
4297 AATags(AATags) {}
4298
4299 /// Emit a leaf load of a single value. This is called at the leaves of the
4300 /// recursive emission to actually load values.
4301 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4302 assert(Ty->isSingleValueType());
4303 // Load the single value and insert it using the indices.
4304 Value *GEP =
4305 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4306 LoadInst *Load =
4307 IRB.CreateAlignedLoad(Ty, Ptr: GEP, Align: Alignment, Name: Name + ".load");
4308
4309 APInt Offset(
4310 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4311 if (AATags &&
4312 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset))
4313 Load->setAAMetadata(
4314 AATags.adjustForAccess(Offset: Offset.getZExtValue(), AccessTy: Load->getType(), DL));
4315 // Record the load so we can generate a fake use for this aggregate
4316 // component.
4317 Components.push_back(Elt: Load);
4318
4319 Agg = IRB.CreateInsertValue(Agg, Val: Load, Idxs: Indices, Name: Name + ".insert");
4320 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4321 }
4322
4323 // Stash the fake uses that use the value generated by this instruction.
4324 void recordFakeUses(LoadInst &LI) {
4325 for (Use &U : LI.uses())
4326 if (auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser()))
4327 if (II->getIntrinsicID() == Intrinsic::fake_use)
4328 FakeUses.push_back(Elt: II);
4329 }
4330
4331 // Replace all fake uses of the aggregate with a series of fake uses, one
4332 // for each split component.
4333 void emitFakeUses() {
4334 for (Instruction *I : FakeUses) {
4335 IRB.SetInsertPoint(I);
4336 for (auto *V : Components)
4337 IRB.CreateIntrinsic(ID: Intrinsic::fake_use, Args: {V});
4338 I->eraseFromParent();
4339 }
4340 }
4341 };
4342
4343 bool visitLoadInst(LoadInst &LI) {
4344 assert(LI.getPointerOperand() == *U);
4345 if (!LI.isSimple() || LI.getType()->isSingleValueType())
4346 return false;
4347
4348 // We have an aggregate being loaded, split it apart.
4349 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4350 LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4351 getAdjustedAlignment(I: &LI, Offset: 0), DL, IRB);
4352 Splitter.recordFakeUses(LI);
4353 Value *V = PoisonValue::get(T: LI.getType());
4354 Splitter.emitSplitOps(Ty: LI.getType(), Agg&: V, Name: LI.getName() + ".fca");
4355 Splitter.emitFakeUses();
4356 Visited.erase(Ptr: &LI);
4357 LI.replaceAllUsesWith(V);
4358 LI.eraseFromParent();
4359 return true;
4360 }
4361
4362 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4363 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4364 AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4365 const DataLayout &DL, IRBuilderTy &IRB)
4366 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4367 DL, IRB),
4368 AATags(AATags), AggStore(AggStore) {}
4369 AAMDNodes AATags;
4370 StoreInst *AggStore;
4371 /// Emit a leaf store of a single value. This is called at the leaves of the
4372 /// recursive emission to actually produce stores.
4373 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4374 assert(Ty->isSingleValueType());
4375 // Extract the single value and store it using the indices.
4376 //
4377 // The gep and extractvalue values are factored out of the CreateStore
4378 // call to make the output independent of the argument evaluation order.
4379 Value *ExtractValue =
4380 IRB.CreateExtractValue(Agg, Idxs: Indices, Name: Name + ".extract");
4381 Value *InBoundsGEP =
4382 IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4383 StoreInst *Store =
4384 IRB.CreateAlignedStore(Val: ExtractValue, Ptr: InBoundsGEP, Align: Alignment);
4385
4386 APInt Offset(
4387 DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), 0);
4388 GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset);
4389 if (AATags) {
4390 Store->setAAMetadata(AATags.adjustForAccess(
4391 Offset: Offset.getZExtValue(), AccessTy: ExtractValue->getType(), DL));
4392 }
4393
4394 // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4395 // If we cannot (because there's an intervening non-const or unbounded
4396 // gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4397 // this instruction.
4398 Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets();
4399 if (auto *OldAI = dyn_cast<AllocaInst>(Val: Base)) {
4400 uint64_t SizeInBits =
4401 DL.getTypeSizeInBits(Ty: Store->getValueOperand()->getType());
4402 migrateDebugInfo(OldAlloca: OldAI, /*IsSplit*/ true, OldAllocaOffsetInBits: Offset.getZExtValue() * 8,
4403 SliceSizeInBits: SizeInBits, OldInst: AggStore, Inst: Store,
4404 Dest: Store->getPointerOperand(), Value: Store->getValueOperand(),
4405 DL);
4406 } else {
4407 assert(at::getDVRAssignmentMarkers(Store).empty() &&
4408 "AT: unexpected debug.assign linked to store through "
4409 "unbounded GEP");
4410 }
4411 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4412 }
4413 };
4414
4415 bool visitStoreInst(StoreInst &SI) {
4416 if (!SI.isSimple() || SI.getPointerOperand() != *U)
4417 return false;
4418 Value *V = SI.getValueOperand();
4419 if (V->getType()->isSingleValueType())
4420 return false;
4421
4422 // We have an aggregate being stored, split it apart.
4423 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4424 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4425 getAdjustedAlignment(I: &SI, Offset: 0), DL, IRB);
4426 Splitter.emitSplitOps(Ty: V->getType(), Agg&: V, Name: V->getName() + ".fca");
4427 Visited.erase(Ptr: &SI);
4428 // The stores replacing SI each have markers describing fragments of the
4429 // assignment so delete the assignment markers linked to SI.
4430 at::deleteAssignmentMarkers(Inst: &SI);
4431 SI.eraseFromParent();
4432 return true;
4433 }
4434
4435 bool visitBitCastInst(BitCastInst &BC) {
4436 enqueueUsers(I&: BC);
4437 return false;
4438 }
4439
4440 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4441 enqueueUsers(I&: ASC);
4442 return false;
4443 }
4444
4445 // Unfold gep (select cond, ptr1, ptr2), idx
4446 // => select cond, gep(ptr1, idx), gep(ptr2, idx)
4447 // and gep ptr, (select cond, idx1, idx2)
4448 // => select cond, gep(ptr, idx1), gep(ptr, idx2)
4449 // We also allow for i1 zext indices, which are equivalent to selects.
4450 bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4451 // Check whether the GEP has exactly one select operand and all indices
4452 // will become constant after the transform.
4453 Instruction *Sel = dyn_cast<SelectInst>(Val: GEPI.getPointerOperand());
4454 for (Value *Op : GEPI.indices()) {
4455 if (auto *SI = dyn_cast<SelectInst>(Val: Op)) {
4456 if (Sel)
4457 return false;
4458
4459 Sel = SI;
4460 if (!isa<ConstantInt>(Val: SI->getTrueValue()) ||
4461 !isa<ConstantInt>(Val: SI->getFalseValue()))
4462 return false;
4463 continue;
4464 }
4465 if (auto *ZI = dyn_cast<ZExtInst>(Val: Op)) {
4466 if (Sel)
4467 return false;
4468 Sel = ZI;
4469 if (!ZI->getSrcTy()->isIntegerTy(BitWidth: 1))
4470 return false;
4471 continue;
4472 }
4473
4474 if (!isa<ConstantInt>(Val: Op))
4475 return false;
4476 }
4477
4478 if (!Sel)
4479 return false;
4480
4481 LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4482 dbgs() << " original: " << *Sel << "\n";
4483 dbgs() << " " << GEPI << "\n";);
4484
4485 auto GetNewOps = [&](Value *SelOp) {
4486 SmallVector<Value *> NewOps;
4487 for (Value *Op : GEPI.operands())
4488 if (Op == Sel)
4489 NewOps.push_back(Elt: SelOp);
4490 else
4491 NewOps.push_back(Elt: Op);
4492 return NewOps;
4493 };
4494
4495 Value *Cond, *True, *False;
4496 Instruction *MDFrom = nullptr;
4497 if (auto *SI = dyn_cast<SelectInst>(Val: Sel)) {
4498 Cond = SI->getCondition();
4499 True = SI->getTrueValue();
4500 False = SI->getFalseValue();
4501 if (!ProfcheckDisableMetadataFixes)
4502 MDFrom = SI;
4503 } else {
4504 Cond = Sel->getOperand(i: 0);
4505 True = ConstantInt::get(Ty: Sel->getType(), V: 1);
4506 False = ConstantInt::get(Ty: Sel->getType(), V: 0);
4507 }
4508 SmallVector<Value *> TrueOps = GetNewOps(True);
4509 SmallVector<Value *> FalseOps = GetNewOps(False);
4510
4511 IRB.SetInsertPoint(&GEPI);
4512 GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4513
4514 Type *Ty = GEPI.getSourceElementType();
4515 Value *NTrue = IRB.CreateGEP(Ty, Ptr: TrueOps[0], IdxList: ArrayRef(TrueOps).drop_front(),
4516 Name: True->getName() + ".sroa.gep", NW);
4517
4518 Value *NFalse =
4519 IRB.CreateGEP(Ty, Ptr: FalseOps[0], IdxList: ArrayRef(FalseOps).drop_front(),
4520 Name: False->getName() + ".sroa.gep", NW);
4521
4522 Value *NSel = MDFrom
4523 ? IRB.CreateSelect(C: Cond, True: NTrue, False: NFalse,
4524 Name: Sel->getName() + ".sroa.sel", MDFrom)
4525 : IRB.CreateSelectWithUnknownProfile(
4526 C: Cond, True: NTrue, False: NFalse, DEBUG_TYPE,
4527 Name: Sel->getName() + ".sroa.sel");
4528 Visited.erase(Ptr: &GEPI);
4529 GEPI.replaceAllUsesWith(V: NSel);
4530 GEPI.eraseFromParent();
4531 Instruction *NSelI = cast<Instruction>(Val: NSel);
4532 Visited.insert(Ptr: NSelI);
4533 enqueueUsers(I&: *NSelI);
4534
4535 LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4536 dbgs() << " " << *NFalse << "\n";
4537 dbgs() << " " << *NSel << "\n";);
4538
4539 return true;
4540 }
4541
4542 // Unfold gep (phi ptr1, ptr2), idx
4543 // => phi ((gep ptr1, idx), (gep ptr2, idx))
4544 // and gep ptr, (phi idx1, idx2)
4545 // => phi ((gep ptr, idx1), (gep ptr, idx2))
4546 bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4547 // To prevent infinitely expanding recursive phis, bail if the GEP pointer
4548 // operand (looking through the phi if it is the phi we want to unfold) is
4549 // an instruction besides a static alloca.
4550 PHINode *Phi = dyn_cast<PHINode>(Val: GEPI.getPointerOperand());
4551 auto IsInvalidPointerOperand = [](Value *V) {
4552 if (!isa<Instruction>(Val: V))
4553 return false;
4554 if (auto *AI = dyn_cast<AllocaInst>(Val: V))
4555 return !AI->isStaticAlloca();
4556 return true;
4557 };
4558 if (Phi) {
4559 if (any_of(Range: Phi->operands(), P: IsInvalidPointerOperand))
4560 return false;
4561 } else {
4562 if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4563 return false;
4564 }
4565 // Check whether the GEP has exactly one phi operand (including the pointer
4566 // operand) and all indices will become constant after the transform.
4567 for (Value *Op : GEPI.indices()) {
4568 if (auto *SI = dyn_cast<PHINode>(Val: Op)) {
4569 if (Phi)
4570 return false;
4571
4572 Phi = SI;
4573 if (!all_of(Range: Phi->incoming_values(),
4574 P: [](Value *V) { return isa<ConstantInt>(Val: V); }))
4575 return false;
4576 continue;
4577 }
4578
4579 if (!isa<ConstantInt>(Val: Op))
4580 return false;
4581 }
4582
4583 if (!Phi)
4584 return false;
4585
4586 LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4587 dbgs() << " original: " << *Phi << "\n";
4588 dbgs() << " " << GEPI << "\n";);
4589
4590 auto GetNewOps = [&](Value *PhiOp) {
4591 SmallVector<Value *> NewOps;
4592 for (Value *Op : GEPI.operands())
4593 if (Op == Phi)
4594 NewOps.push_back(Elt: PhiOp);
4595 else
4596 NewOps.push_back(Elt: Op);
4597 return NewOps;
4598 };
4599
4600 IRB.SetInsertPoint(Phi);
4601 PHINode *NewPhi = IRB.CreatePHI(Ty: GEPI.getType(), NumReservedValues: Phi->getNumIncomingValues(),
4602 Name: Phi->getName() + ".sroa.phi");
4603
4604 Type *SourceTy = GEPI.getSourceElementType();
4605 // We only handle arguments, constants, and static allocas here, so we can
4606 // insert GEPs at the end of the entry block.
4607 IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4608 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4609 Value *Op = Phi->getIncomingValue(i: I);
4610 BasicBlock *BB = Phi->getIncomingBlock(i: I);
4611 Value *NewGEP;
4612 if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) {
4613 NewGEP = NewPhi->getIncomingValue(i: NI);
4614 } else {
4615 SmallVector<Value *> NewOps = GetNewOps(Op);
4616 NewGEP =
4617 IRB.CreateGEP(Ty: SourceTy, Ptr: NewOps[0], IdxList: ArrayRef(NewOps).drop_front(),
4618 Name: Phi->getName() + ".sroa.gep", NW: GEPI.getNoWrapFlags());
4619 }
4620 NewPhi->addIncoming(V: NewGEP, BB);
4621 }
4622
4623 Visited.erase(Ptr: &GEPI);
4624 GEPI.replaceAllUsesWith(V: NewPhi);
4625 GEPI.eraseFromParent();
4626 Visited.insert(Ptr: NewPhi);
4627 enqueueUsers(I&: *NewPhi);
4628
4629 LLVM_DEBUG(dbgs() << " to: ";
4630 for (Value *In
4631 : NewPhi->incoming_values()) dbgs()
4632 << "\n " << *In;
4633 dbgs() << "\n " << *NewPhi << '\n');
4634
4635 return true;
4636 }
4637
4638 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4639 if (unfoldGEPSelect(GEPI))
4640 return true;
4641
4642 if (unfoldGEPPhi(GEPI))
4643 return true;
4644
4645 enqueueUsers(I&: GEPI);
4646 return false;
4647 }
4648
4649 bool visitPHINode(PHINode &PN) {
4650 enqueueUsers(I&: PN);
4651 return false;
4652 }
4653
4654 bool visitSelectInst(SelectInst &SI) {
4655 enqueueUsers(I&: SI);
4656 return false;
4657 }
4658};
4659
4660} // end anonymous namespace
4661
4662/// Strip aggregate type wrapping.
4663///
4664/// This removes no-op aggregate types wrapping an underlying type. It will
4665/// strip as many layers of types as it can without changing either the type
4666/// size or the allocated size.
4667static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
4668 if (Ty->isSingleValueType())
4669 return Ty;
4670
4671 uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4672 uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4673
4674 Type *InnerTy;
4675 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Val: Ty)) {
4676 InnerTy = ArrTy->getElementType();
4677 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4678 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4679 unsigned Index = SL->getElementContainingOffset(FixedOffset: 0);
4680 InnerTy = STy->getElementType(N: Index);
4681 } else {
4682 return Ty;
4683 }
4684
4685 if (AllocSize > DL.getTypeAllocSize(Ty: InnerTy).getFixedValue() ||
4686 TypeSize > DL.getTypeSizeInBits(Ty: InnerTy).getFixedValue())
4687 return Ty;
4688
4689 return stripAggregateTypeWrapping(DL, Ty: InnerTy);
4690}
4691
4692/// Try to find a partition of the aggregate type passed in for a given
4693/// offset and size.
4694///
4695/// This recurses through the aggregate type and tries to compute a subtype
4696/// based on the offset and size. When the offset and size span a sub-section
4697/// of an array, it will even compute a new array type for that sub-section,
4698/// and the same for structs.
4699///
4700/// Note that this routine is very strict and tries to find a partition of the
4701/// type which produces the *exact* right offset and size. It is not forgiving
4702/// when the size or offset cause either end of type-based partition to be off.
4703/// Also, this is a best-effort routine. It is reasonable to give up and not
4704/// return a type if necessary.
4705static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
4706 uint64_t Size) {
4707 if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4708 return stripAggregateTypeWrapping(DL, Ty);
4709 if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() ||
4710 (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4711 return nullptr;
4712
4713 if (isa<ArrayType>(Val: Ty) || isa<VectorType>(Val: Ty)) {
4714 Type *ElementTy;
4715 uint64_t TyNumElements;
4716 if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
4717 ElementTy = AT->getElementType();
4718 TyNumElements = AT->getNumElements();
4719 } else {
4720 // FIXME: This isn't right for vectors with non-byte-sized or
4721 // non-power-of-two sized elements.
4722 auto *VT = cast<FixedVectorType>(Val: Ty);
4723 ElementTy = VT->getElementType();
4724 TyNumElements = VT->getNumElements();
4725 }
4726 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4727 uint64_t NumSkippedElements = Offset / ElementSize;
4728 if (NumSkippedElements >= TyNumElements)
4729 return nullptr;
4730 Offset -= NumSkippedElements * ElementSize;
4731
4732 // First check if we need to recurse.
4733 if (Offset > 0 || Size < ElementSize) {
4734 // Bail if the partition ends in a different array element.
4735 if ((Offset + Size) > ElementSize)
4736 return nullptr;
4737 // Recurse through the element type trying to peel off offset bytes.
4738 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4739 }
4740 assert(Offset == 0);
4741
4742 if (Size == ElementSize)
4743 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4744 assert(Size > ElementSize);
4745 uint64_t NumElements = Size / ElementSize;
4746 if (NumElements * ElementSize != Size)
4747 return nullptr;
4748 return ArrayType::get(ElementType: ElementTy, NumElements);
4749 }
4750
4751 StructType *STy = dyn_cast<StructType>(Val: Ty);
4752 if (!STy)
4753 return nullptr;
4754
4755 const StructLayout *SL = DL.getStructLayout(Ty: STy);
4756
4757 if (SL->getSizeInBits().isScalable())
4758 return nullptr;
4759
4760 if (Offset >= SL->getSizeInBytes())
4761 return nullptr;
4762 uint64_t EndOffset = Offset + Size;
4763 if (EndOffset > SL->getSizeInBytes())
4764 return nullptr;
4765
4766 unsigned Index = SL->getElementContainingOffset(FixedOffset: Offset);
4767 Offset -= SL->getElementOffset(Idx: Index);
4768
4769 Type *ElementTy = STy->getElementType(N: Index);
4770 uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4771 if (Offset >= ElementSize)
4772 return nullptr; // The offset points into alignment padding.
4773
4774 // See if any partition must be contained by the element.
4775 if (Offset > 0 || Size < ElementSize) {
4776 if ((Offset + Size) > ElementSize)
4777 return nullptr;
4778 return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4779 }
4780 assert(Offset == 0);
4781
4782 if (Size == ElementSize)
4783 return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4784
4785 StructType::element_iterator EI = STy->element_begin() + Index,
4786 EE = STy->element_end();
4787 if (EndOffset < SL->getSizeInBytes()) {
4788 unsigned EndIndex = SL->getElementContainingOffset(FixedOffset: EndOffset);
4789 if (Index == EndIndex)
4790 return nullptr; // Within a single element and its padding.
4791
4792 // Don't try to form "natural" types if the elements don't line up with the
4793 // expected size.
4794 // FIXME: We could potentially recurse down through the last element in the
4795 // sub-struct to find a natural end point.
4796 if (SL->getElementOffset(Idx: EndIndex) != EndOffset)
4797 return nullptr;
4798
4799 assert(Index < EndIndex);
4800 EE = STy->element_begin() + EndIndex;
4801 }
4802
4803 // Try to build up a sub-structure.
4804 StructType *SubTy =
4805 StructType::get(Context&: STy->getContext(), Elements: ArrayRef(EI, EE), isPacked: STy->isPacked());
4806 const StructLayout *SubSL = DL.getStructLayout(Ty: SubTy);
4807 if (Size != SubSL->getSizeInBytes())
4808 return nullptr; // The sub-struct doesn't have quite the size needed.
4809
4810 return SubTy;
4811}
4812
4813/// Pre-split loads and stores to simplify rewriting.
4814///
4815/// We want to break up the splittable load+store pairs as much as
4816/// possible. This is important to do as a preprocessing step, as once we
4817/// start rewriting the accesses to partitions of the alloca we lose the
4818/// necessary information to correctly split apart paired loads and stores
4819/// which both point into this alloca. The case to consider is something like
4820/// the following:
4821///
4822/// %a = alloca [12 x i8]
4823/// %gep1 = getelementptr i8, ptr %a, i32 0
4824/// %gep2 = getelementptr i8, ptr %a, i32 4
4825/// %gep3 = getelementptr i8, ptr %a, i32 8
4826/// store float 0.0, ptr %gep1
4827/// store float 1.0, ptr %gep2
4828/// %v = load i64, ptr %gep1
4829/// store i64 %v, ptr %gep2
4830/// %f1 = load float, ptr %gep2
4831/// %f2 = load float, ptr %gep3
4832///
4833/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
4834/// promote everything so we recover the 2 SSA values that should have been
4835/// there all along.
4836///
4837/// \returns true if any changes are made.
4838bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
4839 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
4840
4841 // Track the loads and stores which are candidates for pre-splitting here, in
4842 // the order they first appear during the partition scan. These give stable
4843 // iteration order and a basis for tracking which loads and stores we
4844 // actually split.
4845 SmallVector<LoadInst *, 4> Loads;
4846 SmallVector<StoreInst *, 4> Stores;
4847
4848 // We need to accumulate the splits required of each load or store where we
4849 // can find them via a direct lookup. This is important to cross-check loads
4850 // and stores against each other. We also track the slice so that we can kill
4851 // all the slices that end up split.
4852 struct SplitOffsets {
4853 Slice *S;
4854 std::vector<uint64_t> Splits;
4855 };
4856 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
4857
4858 // Track loads out of this alloca which cannot, for any reason, be pre-split.
4859 // This is important as we also cannot pre-split stores of those loads!
4860 // FIXME: This is all pretty gross. It means that we can be more aggressive
4861 // in pre-splitting when the load feeding the store happens to come from
4862 // a separate alloca. Put another way, the effectiveness of SROA would be
4863 // decreased by a frontend which just concatenated all of its local allocas
4864 // into one big flat alloca. But defeating such patterns is exactly the job
4865 // SROA is tasked with! Sadly, to not have this discrepancy we would have
4866 // change store pre-splitting to actually force pre-splitting of the load
4867 // that feeds it *and all stores*. That makes pre-splitting much harder, but
4868 // maybe it would make it more principled?
4869 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
4870
4871 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
4872 for (auto &P : AS.partitions()) {
4873 for (Slice &S : P) {
4874 Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
4875 if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
4876 // If this is a load we have to track that it can't participate in any
4877 // pre-splitting. If this is a store of a load we have to track that
4878 // that load also can't participate in any pre-splitting.
4879 if (auto *LI = dyn_cast<LoadInst>(Val: I))
4880 UnsplittableLoads.insert(Ptr: LI);
4881 else if (auto *SI = dyn_cast<StoreInst>(Val: I))
4882 if (auto *LI = dyn_cast<LoadInst>(Val: SI->getValueOperand()))
4883 UnsplittableLoads.insert(Ptr: LI);
4884 continue;
4885 }
4886 assert(P.endOffset() > S.beginOffset() &&
4887 "Empty or backwards partition!");
4888
4889 // Determine if this is a pre-splittable slice.
4890 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
4891 assert(!LI->isVolatile() && "Cannot split volatile loads!");
4892
4893 // The load must be used exclusively to store into other pointers for
4894 // us to be able to arbitrarily pre-split it. The stores must also be
4895 // simple to avoid changing semantics.
4896 auto IsLoadSimplyStored = [](LoadInst *LI) {
4897 for (User *LU : LI->users()) {
4898 auto *SI = dyn_cast<StoreInst>(Val: LU);
4899 if (!SI || !SI->isSimple())
4900 return false;
4901 }
4902 return true;
4903 };
4904 if (!IsLoadSimplyStored(LI)) {
4905 UnsplittableLoads.insert(Ptr: LI);
4906 continue;
4907 }
4908
4909 Loads.push_back(Elt: LI);
4910 } else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4911 if (S.getUse() != &SI->getOperandUse(i: SI->getPointerOperandIndex()))
4912 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
4913 continue;
4914 auto *StoredLoad = dyn_cast<LoadInst>(Val: SI->getValueOperand());
4915 if (!StoredLoad || !StoredLoad->isSimple())
4916 continue;
4917 assert(!SI->isVolatile() && "Cannot split volatile stores!");
4918
4919 Stores.push_back(Elt: SI);
4920 } else {
4921 // Other uses cannot be pre-split.
4922 continue;
4923 }
4924
4925 // Record the initial split.
4926 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
4927 auto &Offsets = SplitOffsetsMap[I];
4928 assert(Offsets.Splits.empty() &&
4929 "Should not have splits the first time we see an instruction!");
4930 Offsets.S = &S;
4931 Offsets.Splits.push_back(x: P.endOffset() - S.beginOffset());
4932 }
4933
4934 // Now scan the already split slices, and add a split for any of them which
4935 // we're going to pre-split.
4936 for (Slice *S : P.splitSliceTails()) {
4937 auto SplitOffsetsMapI =
4938 SplitOffsetsMap.find(Val: cast<Instruction>(Val: S->getUse()->getUser()));
4939 if (SplitOffsetsMapI == SplitOffsetsMap.end())
4940 continue;
4941 auto &Offsets = SplitOffsetsMapI->second;
4942
4943 assert(Offsets.S == S && "Found a mismatched slice!");
4944 assert(!Offsets.Splits.empty() &&
4945 "Cannot have an empty set of splits on the second partition!");
4946 assert(Offsets.Splits.back() ==
4947 P.beginOffset() - Offsets.S->beginOffset() &&
4948 "Previous split does not end where this one begins!");
4949
4950 // Record each split. The last partition's end isn't needed as the size
4951 // of the slice dictates that.
4952 if (S->endOffset() > P.endOffset())
4953 Offsets.Splits.push_back(x: P.endOffset() - Offsets.S->beginOffset());
4954 }
4955 }
4956
4957 // We may have split loads where some of their stores are split stores. For
4958 // such loads and stores, we can only pre-split them if their splits exactly
4959 // match relative to their starting offset. We have to verify this prior to
4960 // any rewriting.
4961 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
4962 // Lookup the load we are storing in our map of split
4963 // offsets.
4964 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
4965 // If it was completely unsplittable, then we're done,
4966 // and this store can't be pre-split.
4967 if (UnsplittableLoads.count(Ptr: LI))
4968 return true;
4969
4970 auto LoadOffsetsI = SplitOffsetsMap.find(Val: LI);
4971 if (LoadOffsetsI == SplitOffsetsMap.end())
4972 return false; // Unrelated loads are definitely safe.
4973 auto &LoadOffsets = LoadOffsetsI->second;
4974
4975 // Now lookup the store's offsets.
4976 auto &StoreOffsets = SplitOffsetsMap[SI];
4977
4978 // If the relative offsets of each split in the load and
4979 // store match exactly, then we can split them and we
4980 // don't need to remove them here.
4981 if (LoadOffsets.Splits == StoreOffsets.Splits)
4982 return false;
4983
4984 LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
4985 << " " << *LI << "\n"
4986 << " " << *SI << "\n");
4987
4988 // We've found a store and load that we need to split
4989 // with mismatched relative splits. Just give up on them
4990 // and remove both instructions from our list of
4991 // candidates.
4992 UnsplittableLoads.insert(Ptr: LI);
4993 return true;
4994 });
4995 // Now we have to go *back* through all the stores, because a later store may
4996 // have caused an earlier store's load to become unsplittable and if it is
4997 // unsplittable for the later store, then we can't rely on it being split in
4998 // the earlier store either.
4999 llvm::erase_if(C&: Stores, P: [&UnsplittableLoads](StoreInst *SI) {
5000 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5001 return UnsplittableLoads.count(Ptr: LI);
5002 });
5003 // Once we've established all the loads that can't be split for some reason,
5004 // filter any that made it into our list out.
5005 llvm::erase_if(C&: Loads, P: [&UnsplittableLoads](LoadInst *LI) {
5006 return UnsplittableLoads.count(Ptr: LI);
5007 });
5008
5009 // If no loads or stores are left, there is no pre-splitting to be done for
5010 // this alloca.
5011 if (Loads.empty() && Stores.empty())
5012 return false;
5013
5014 // From here on, we can't fail and will be building new accesses, so rig up
5015 // an IR builder.
5016 IRBuilderTy IRB(&AI);
5017
5018 // Collect the new slices which we will merge into the alloca slices.
5019 SmallVector<Slice, 4> NewSlices;
5020
5021 // Track any allocas we end up splitting loads and stores for so we iterate
5022 // on them.
5023 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
5024
5025 // At this point, we have collected all of the loads and stores we can
5026 // pre-split, and the specific splits needed for them. We actually do the
5027 // splitting in a specific order in order to handle when one of the loads in
5028 // the value operand to one of the stores.
5029 //
5030 // First, we rewrite all of the split loads, and just accumulate each split
5031 // load in a parallel structure. We also build the slices for them and append
5032 // them to the alloca slices.
5033 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
5034 std::vector<LoadInst *> SplitLoads;
5035 const DataLayout &DL = AI.getDataLayout();
5036 for (LoadInst *LI : Loads) {
5037 SplitLoads.clear();
5038
5039 auto &Offsets = SplitOffsetsMap[LI];
5040 unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5041 assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&
5042 "Load must have type size equal to store size");
5043 assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&
5044 "Load must be >= slice size");
5045
5046 uint64_t BaseOffset = Offsets.S->beginOffset();
5047 assert(BaseOffset + SliceSize > BaseOffset &&
5048 "Cannot represent alloca access size using 64-bit integers!");
5049
5050 Instruction *BasePtr = cast<Instruction>(Val: LI->getPointerOperand());
5051 IRB.SetInsertPoint(LI);
5052
5053 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5054
5055 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5056 int Idx = 0, Size = Offsets.Splits.size();
5057 for (;;) {
5058 auto *PartTy = Type::getIntNTy(C&: LI->getContext(), N: PartSize * 8);
5059 auto AS = LI->getPointerAddressSpace();
5060 auto *PartPtrTy = LI->getPointerOperandType();
5061 LoadInst *PLoad = IRB.CreateAlignedLoad(
5062 Ty: PartTy,
5063 Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5064 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5065 PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5066 Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5067 /*IsVolatile*/ isVolatile: false, Name: LI->getName());
5068 PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5069 LLVMContext::MD_access_group});
5070
5071 // Append this load onto the list of split loads so we can find it later
5072 // to rewrite the stores.
5073 SplitLoads.push_back(x: PLoad);
5074
5075 // Now build a new slice for the alloca.
5076 NewSlices.push_back(
5077 Elt: Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5078 &PLoad->getOperandUse(i: PLoad->getPointerOperandIndex()),
5079 /*IsSplittable*/ false));
5080 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5081 << ", " << NewSlices.back().endOffset()
5082 << "): " << *PLoad << "\n");
5083
5084 // See if we've handled all the splits.
5085 if (Idx >= Size)
5086 break;
5087
5088 // Setup the next partition.
5089 PartOffset = Offsets.Splits[Idx];
5090 ++Idx;
5091 PartSize = (Idx < Size ? Offsets.Splits[Idx] : SliceSize) - PartOffset;
5092 }
5093
5094 // Now that we have the split loads, do the slow walk over all uses of the
5095 // load and rewrite them as split stores, or save the split loads to use
5096 // below if the store is going to be split there anyways.
5097 bool DeferredStores = false;
5098 for (User *LU : LI->users()) {
5099 StoreInst *SI = cast<StoreInst>(Val: LU);
5100 if (!Stores.empty() && SplitOffsetsMap.count(Val: SI)) {
5101 DeferredStores = true;
5102 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5103 << "\n");
5104 continue;
5105 }
5106
5107 Value *StoreBasePtr = SI->getPointerOperand();
5108 IRB.SetInsertPoint(SI);
5109 AAMDNodes AATags = SI->getAAMetadata();
5110
5111 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5112
5113 for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5114 LoadInst *PLoad = SplitLoads[Idx];
5115 uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1];
5116 auto *PartPtrTy = SI->getPointerOperandType();
5117
5118 auto AS = SI->getPointerAddressSpace();
5119 StoreInst *PStore = IRB.CreateAlignedStore(
5120 Val: PLoad,
5121 Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5122 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5123 PointerTy: PartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5124 Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5125 /*IsVolatile*/ isVolatile: false);
5126 PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5127 LLVMContext::MD_access_group,
5128 LLVMContext::MD_DIAssignID});
5129
5130 if (AATags)
5131 PStore->setAAMetadata(
5132 AATags.adjustForAccess(Offset: PartOffset, AccessTy: PLoad->getType(), DL));
5133 LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5134 }
5135
5136 // We want to immediately iterate on any allocas impacted by splitting
5137 // this store, and we have to track any promotable alloca (indicated by
5138 // a direct store) as needing to be resplit because it is no longer
5139 // promotable.
5140 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: StoreBasePtr)) {
5141 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5142 Worklist.insert(X: OtherAI);
5143 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5144 Val: StoreBasePtr->stripInBoundsOffsets())) {
5145 Worklist.insert(X: OtherAI);
5146 }
5147
5148 // Mark the original store as dead.
5149 DeadInsts.push_back(Elt: SI);
5150 }
5151
5152 // Save the split loads if there are deferred stores among the users.
5153 if (DeferredStores)
5154 SplitLoadsMap.insert(KV: std::make_pair(x&: LI, y: std::move(SplitLoads)));
5155
5156 // Mark the original load as dead and kill the original slice.
5157 DeadInsts.push_back(Elt: LI);
5158 Offsets.S->kill();
5159 }
5160
5161 // Second, we rewrite all of the split stores. At this point, we know that
5162 // all loads from this alloca have been split already. For stores of such
5163 // loads, we can simply look up the pre-existing split loads. For stores of
5164 // other loads, we split those loads first and then write split stores of
5165 // them.
5166 for (StoreInst *SI : Stores) {
5167 auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5168 IntegerType *Ty = cast<IntegerType>(Val: LI->getType());
5169 assert(Ty->getBitWidth() % 8 == 0);
5170 uint64_t StoreSize = Ty->getBitWidth() / 8;
5171 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
5172
5173 auto &Offsets = SplitOffsetsMap[SI];
5174 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5175 "Slice size should always match load size exactly!");
5176 uint64_t BaseOffset = Offsets.S->beginOffset();
5177 assert(BaseOffset + StoreSize > BaseOffset &&
5178 "Cannot represent alloca access size using 64-bit integers!");
5179
5180 Value *LoadBasePtr = LI->getPointerOperand();
5181 Instruction *StoreBasePtr = cast<Instruction>(Val: SI->getPointerOperand());
5182
5183 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5184
5185 // Check whether we have an already split load.
5186 auto SplitLoadsMapI = SplitLoadsMap.find(Val: LI);
5187 std::vector<LoadInst *> *SplitLoads = nullptr;
5188 if (SplitLoadsMapI != SplitLoadsMap.end()) {
5189 SplitLoads = &SplitLoadsMapI->second;
5190 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
5191 "Too few split loads for the number of splits in the store!");
5192 } else {
5193 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5194 }
5195
5196 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5197 int Idx = 0, Size = Offsets.Splits.size();
5198 for (;;) {
5199 auto *PartTy = Type::getIntNTy(C&: Ty->getContext(), N: PartSize * 8);
5200 auto *LoadPartPtrTy = LI->getPointerOperandType();
5201 auto *StorePartPtrTy = SI->getPointerOperandType();
5202
5203 // Either lookup a split load or create one.
5204 LoadInst *PLoad;
5205 if (SplitLoads) {
5206 PLoad = (*SplitLoads)[Idx];
5207 } else {
5208 IRB.SetInsertPoint(LI);
5209 auto AS = LI->getPointerAddressSpace();
5210 PLoad = IRB.CreateAlignedLoad(
5211 Ty: PartTy,
5212 Ptr: getAdjustedPtr(IRB, DL, Ptr: LoadBasePtr,
5213 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5214 PointerTy: LoadPartPtrTy, NamePrefix: LoadBasePtr->getName() + "."),
5215 Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5216 /*IsVolatile*/ isVolatile: false, Name: LI->getName());
5217 PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5218 LLVMContext::MD_access_group});
5219 }
5220
5221 // And store this partition.
5222 IRB.SetInsertPoint(SI);
5223 auto AS = SI->getPointerAddressSpace();
5224 StoreInst *PStore = IRB.CreateAlignedStore(
5225 Val: PLoad,
5226 Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5227 Offset: APInt(DL.getIndexSizeInBits(AS), PartOffset),
5228 PointerTy: StorePartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5229 Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5230 /*IsVolatile*/ isVolatile: false);
5231 PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5232 LLVMContext::MD_access_group});
5233
5234 // Now build a new slice for the alloca.
5235 NewSlices.push_back(
5236 Elt: Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5237 &PStore->getOperandUse(i: PStore->getPointerOperandIndex()),
5238 /*IsSplittable*/ false));
5239 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5240 << ", " << NewSlices.back().endOffset()
5241 << "): " << *PStore << "\n");
5242 if (!SplitLoads) {
5243 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5244 }
5245
5246 // See if we've finished all the splits.
5247 if (Idx >= Size)
5248 break;
5249
5250 // Setup the next partition.
5251 PartOffset = Offsets.Splits[Idx];
5252 ++Idx;
5253 PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset;
5254 }
5255
5256 // We want to immediately iterate on any allocas impacted by splitting
5257 // this load, which is only relevant if it isn't a load of this alloca and
5258 // thus we didn't already split the loads above. We also have to keep track
5259 // of any promotable allocas we split loads on as they can no longer be
5260 // promoted.
5261 if (!SplitLoads) {
5262 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: LoadBasePtr)) {
5263 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5264 ResplitPromotableAllocas.insert(Ptr: OtherAI);
5265 Worklist.insert(X: OtherAI);
5266 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5267 Val: LoadBasePtr->stripInBoundsOffsets())) {
5268 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5269 Worklist.insert(X: OtherAI);
5270 }
5271 }
5272
5273 // Mark the original store as dead now that we've split it up and kill its
5274 // slice. Note that we leave the original load in place unless this store
5275 // was its only use. It may in turn be split up if it is an alloca load
5276 // for some other alloca, but it may be a normal load. This may introduce
5277 // redundant loads, but where those can be merged the rest of the optimizer
5278 // should handle the merging, and this uncovers SSA splits which is more
5279 // important. In practice, the original loads will almost always be fully
5280 // split and removed eventually, and the splits will be merged by any
5281 // trivial CSE, including instcombine.
5282 if (LI->hasOneUse()) {
5283 assert(*LI->user_begin() == SI && "Single use isn't this store!");
5284 DeadInsts.push_back(Elt: LI);
5285 }
5286 DeadInsts.push_back(Elt: SI);
5287 Offsets.S->kill();
5288 }
5289
5290 // Remove the killed slices that have ben pre-split.
5291 llvm::erase_if(C&: AS, P: [](const Slice &S) { return S.isDead(); });
5292
5293 // Insert our new slices. This will sort and merge them into the sorted
5294 // sequence.
5295 AS.insert(NewSlices);
5296
5297 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5298#ifndef NDEBUG
5299 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5300 LLVM_DEBUG(AS.print(dbgs(), I, " "));
5301#endif
5302
5303 // Finally, don't try to promote any allocas that new require re-splitting.
5304 // They have already been added to the worklist above.
5305 PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5306
5307 return true;
5308}
5309
5310/// Try to canonicalize a homogeneous struct partition to a vector type.
5311///
5312/// We can do this if all the elements of the struct are the same and the
5313/// corresponding vector has the same byte-level layout. This can sometimes
5314/// eliminate allocas because structs cannot get promoted to LLVM values, but
5315/// vectors can.
5316///
5317/// We only apply this transformation when all users of the partition are memory
5318/// intrinsics. Otherwise, if there is a load or store of some other type to the
5319/// partition, SROA would select that type.
5320///
5321/// Applying this transformation too early may hinder memcpyopt, which may
5322/// generate better code when eliminating allocas. For example, see
5323/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5324/// this before memcpyopt can initialize previously uninitialized memory when
5325/// the alloca gets promoted to an SSA value. For another example, see
5326/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5327/// this before memcpyopt can result in promoting an alloca so that we load a
5328/// temporary value instead of copying the temporary value into memory, whereas
5329/// memcpyopt eliminates the temporary altogether.
5330///
5331/// As such, we only apply this transformation after memcpyopt has run. We gate
5332/// this transformation by the "AggregateToVector" pass option.
5333static FixedVectorType *tryCanonicalizeStructToVector(StructType *STy,
5334 Partition &P,
5335 const DataLayout &DL) {
5336 unsigned NumElts = STy->getNumElements();
5337
5338 Type *EltTy = STy->getElementType(N: 0);
5339 if (!llvm::all_equal(Range: STy->elements()))
5340 return nullptr;
5341
5342 bool IsIntegralPointerTy =
5343 EltTy->isPointerTy() && !DL.isNonIntegralPointerType(Ty: EltTy);
5344 if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5345 !IsIntegralPointerTy)
5346 return nullptr;
5347
5348 // Ensure the struct is tightly packed so that the bit-layout is the same as
5349 // the corresponding vector. For example, this prevents a miscompile for
5350 // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5351 // tightly packed elements and trailing padding.
5352 if (DL.getTypeSizeInBits(Ty: EltTy) != DL.getTypeAllocSizeInBits(Ty: EltTy))
5353 return nullptr;
5354
5355 auto *VTy = FixedVectorType::get(ElementType: EltTy, NumElts);
5356 TypeSize StructSize = DL.getStructLayout(Ty: STy)->getSizeInBytes();
5357 TypeSize VectorSize = DL.getTypeStoreSize(Ty: VTy);
5358 // After ruling out per-element padding, make sure a vector load/store
5359 // covers the same number of bytes as the struct layout.
5360 if (StructSize != VectorSize)
5361 return nullptr;
5362
5363 auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5364 if (S.isDead())
5365 return true;
5366 auto *U = S.getUse();
5367 if (!U)
5368 return true;
5369
5370 User *Usr = U->getUser();
5371 if (isa<LifetimeIntrinsic>(Val: Usr) || isa<DbgInfoIntrinsic>(Val: Usr))
5372 return true;
5373
5374 return isa<MemIntrinsic>(Val: Usr);
5375 };
5376
5377 for (const Slice &S : P)
5378 if (!IsIgnorableOrMemIntrinsicSlice(S))
5379 return nullptr;
5380
5381 for (const Slice *S : P.splitSliceTails())
5382 if (!IsIgnorableOrMemIntrinsicSlice(*S))
5383 return nullptr;
5384
5385 return VTy;
5386}
5387
5388/// Select a partition type for an alloca partition.
5389///
5390/// Try to compute a friendly type for this partition of the alloca. This
5391/// won't always succeed, in which case we fall back to a legal integer type
5392/// or an i8 array of an appropriate size.
5393///
5394/// \returns A tuple with the following elements:
5395/// - PartitionType: The computed type for this partition.
5396/// - IsIntegerWideningViable: True if integer widening promotion is used.
5397/// - VectorType: The vector type if vector promotion is used, otherwise
5398/// nullptr.
5399static std::tuple<Type *, bool, VectorType *>
5400selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
5401 LLVMContext &C, bool AggregateToVector) {
5402 auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5403 VectorType *SelectedVecTy, bool SelectedIntWidening) {
5404 LLVM_DEBUG({
5405 dbgs() << "selectPartitionType path=" << Path
5406 << " func=" << AI.getFunction()->getName() << " alloca=";
5407 if (AI.hasName())
5408 dbgs() << AI.getName();
5409 else
5410 dbgs() << "<unnamed>";
5411 dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5412 << ") size=" << P.size();
5413 if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5414 dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5415 if (SelectedTy)
5416 dbgs() << " chosen=" << *SelectedTy;
5417 if (SelectedVecTy)
5418 dbgs() << " vec=" << *SelectedVecTy;
5419 dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5420 });
5421 };
5422 // First check if the partition is viable for vector promotion.
5423 //
5424 // We prefer vector promotion over integer widening promotion when:
5425 // - The vector element type is a floating-point type.
5426 // - All the loads/stores to the alloca are vector loads/stores to the
5427 // entire alloca or load/store a single element of the vector.
5428 //
5429 // Otherwise when there is an integer vector with mixed type loads/stores we
5430 // prefer integer widening promotion because it's more likely the user is
5431 // doing bitwise arithmetic and we generate better code.
5432 VectorType *VecTy =
5433 isVectorPromotionViable(P, DL, VScale: AI.getFunction()->getVScaleValue());
5434 // If the vector element type is a floating-point type, we prefer vector
5435 // promotion. If the vector has one element, let the below code select
5436 // whether we promote with the vector or scalar.
5437 if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5438 VecTy->getElementCount().getFixedValue() > 1) {
5439 LogSelection("direct-fp-vecty", VecTy, VecTy, false);
5440 return {VecTy, false, VecTy};
5441 }
5442
5443 // Check if there is a common type that all slices of the partition use that
5444 // spans the partition.
5445 auto [CommonUseTy, LargestIntTy] =
5446 findCommonType(B: P.begin(), E: P.end(), EndOffset: P.endOffset());
5447 if (CommonUseTy) {
5448 TypeSize CommonUseSize = DL.getTypeAllocSize(Ty: CommonUseTy);
5449 if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5450 // We prefer vector promotion here because if vector promotion is viable
5451 // and there is a common type used, then it implies the second listed
5452 // condition for preferring vector promotion is true.
5453 if (VecTy) {
5454 LogSelection("common-type-vecty", VecTy, VecTy, false);
5455 return {VecTy, false, VecTy};
5456 }
5457 bool IntWiden = isIntegerWideningViable(P, AllocaTy: CommonUseTy, DL);
5458 LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
5459 return {CommonUseTy, IntWiden, nullptr};
5460 }
5461 }
5462
5463 // Can we find an appropriate subtype in the original allocated
5464 // type?
5465 if (Type *TypePartitionTy = getTypePartition(DL, Ty: AI.getAllocatedType(),
5466 Offset: P.beginOffset(), Size: P.size())) {
5467 // If the partition is an integer array that can be spanned by a legal
5468 // integer type, prefer to represent it as a legal integer type because
5469 // it's more likely to be promotable.
5470 if (TypePartitionTy->isArrayTy() &&
5471 TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5472 DL.isLegalInteger(Width: P.size() * 8))
5473 TypePartitionTy = Type::getIntNTy(C, N: P.size() * 8);
5474 // There was no common type used, so we prefer integer widening promotion.
5475 if (isIntegerWideningViable(P, AllocaTy: TypePartitionTy, DL)) {
5476 LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
5477 return {TypePartitionTy, true, nullptr};
5478 }
5479 if (VecTy) {
5480 LogSelection("type-partition-vecty", VecTy, VecTy, false);
5481 return {VecTy, false, VecTy};
5482 }
5483 // If we couldn't promote with TypePartitionTy, try with the largest
5484 // integer type used.
5485 if (LargestIntTy &&
5486 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size() &&
5487 isIntegerWideningViable(P, AllocaTy: LargestIntTy, DL)) {
5488 LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
5489 return {LargestIntTy, true, nullptr};
5490 }
5491
5492 // Try homogeneous struct to vector canonicalization when requested. Running
5493 // this too early can hide memcpy chains from MemCpyOpt.
5494 if (AggregateToVector) {
5495 if (auto *STy = dyn_cast<StructType>(Val: TypePartitionTy)) {
5496 if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5497 LogSelection("struct-fallback-vecty", VTy, nullptr, false);
5498 return {VTy, false, nullptr};
5499 }
5500 }
5501 }
5502
5503 // Fallback to TypePartitionTy and we probably won't promote.
5504 LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
5505 return {TypePartitionTy, false, nullptr};
5506 }
5507
5508 // Select the largest integer type used if it spans the partition.
5509 if (LargestIntTy &&
5510 DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size()) {
5511 LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
5512 return {LargestIntTy, false, nullptr};
5513 }
5514
5515 // Select a legal integer type if it spans the partition.
5516 if (DL.isLegalInteger(Width: P.size() * 8)) {
5517 Type *IntTy = Type::getIntNTy(C, N: P.size() * 8);
5518 LogSelection("legal-int-fallback", IntTy, nullptr, false);
5519 return {IntTy, false, nullptr};
5520 }
5521
5522 // Fallback to an i8 array.
5523 Type *ArrayTy = ArrayType::get(ElementType: Type::getInt8Ty(C), NumElements: P.size());
5524 LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
5525 return {ArrayTy, false, nullptr};
5526}
5527
5528/// Rewrite an alloca partition's users.
5529///
5530/// This routine drives both of the rewriting goals of the SROA pass. It tries
5531/// to rewrite uses of an alloca partition to be conducive for SSA value
5532/// promotion. If the partition needs a new, more refined alloca, this will
5533/// build that new alloca, preserving as much type information as possible, and
5534/// rewrite the uses of the old alloca to point at the new one and have the
5535/// appropriate new offsets. It also evaluates how successful the rewrite was
5536/// at enabling promotion and if it was successful queues the alloca to be
5537/// promoted.
5538std::pair<AllocaInst *, uint64_t>
5539SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5540 const DataLayout &DL = AI.getDataLayout();
5541 // Select the type for the new alloca that spans the partition.
5542 auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5543 selectPartitionType(P, DL, AI, C&: *C, AggregateToVector);
5544
5545 // Check for the case where we're going to rewrite to a new alloca of the
5546 // exact same type as the original, and with the same access offsets. In that
5547 // case, re-use the existing alloca, but still run through the rewriter to
5548 // perform phi and select speculation.
5549 // P.beginOffset() can be non-zero even with the same type in a case with
5550 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5551 AllocaInst *NewAI;
5552 if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == 0) {
5553 NewAI = &AI;
5554 // FIXME: We should be able to bail at this point with "nothing changed".
5555 // FIXME: We might want to defer PHI speculation until after here.
5556 // FIXME: return nullptr;
5557 } else {
5558 // Make sure the alignment is compatible with P.beginOffset().
5559 const Align Alignment = commonAlignment(A: AI.getAlign(), Offset: P.beginOffset());
5560 // If we will get at least this much alignment from the type alone, leave
5561 // the alloca's alignment unconstrained.
5562 const bool IsUnconstrained = Alignment <= DL.getABITypeAlign(Ty: PartitionTy);
5563 NewAI = new AllocaInst(
5564 PartitionTy, AI.getAddressSpace(), nullptr,
5565 IsUnconstrained ? DL.getPrefTypeAlign(Ty: PartitionTy) : Alignment,
5566 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()),
5567 AI.getIterator());
5568 // Copy the old AI debug location over to the new one.
5569 NewAI->setDebugLoc(AI.getDebugLoc());
5570 ++NumNewAllocas;
5571 }
5572
5573 LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5574 << "," << P.endOffset() << ") to: " << *NewAI << "\n");
5575
5576 // Track the high watermark on the worklist as it is only relevant for
5577 // promoted allocas. We will reset it to this point if the alloca is not in
5578 // fact scheduled for promotion.
5579 unsigned PPWOldSize = PostPromotionWorklist.size();
5580 unsigned NumUses = 0;
5581 SmallSetVector<PHINode *, 8> PHIUsers;
5582 SmallSetVector<SelectInst *, 8> SelectUsers;
5583
5584 AllocaSliceRewriter Rewriter(
5585 DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5586 IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5587 bool Promotable = true;
5588 // Check whether we can have tree-structured merge.
5589 if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5590 NumUses += DeletedValues->size() + 1;
5591 for (Value *V : *DeletedValues)
5592 DeadInsts.push_back(Elt: V);
5593 } else {
5594 for (Slice *S : P.splitSliceTails()) {
5595 Promotable &= Rewriter.visit(I: S);
5596 ++NumUses;
5597 }
5598 for (Slice &S : P) {
5599 Promotable &= Rewriter.visit(I: &S);
5600 ++NumUses;
5601 }
5602 }
5603
5604 NumAllocaPartitionUses += NumUses;
5605 MaxUsesPerAllocaPartition.updateMax(V: NumUses);
5606
5607 // Now that we've processed all the slices in the new partition, check if any
5608 // PHIs or Selects would block promotion.
5609 for (PHINode *PHI : PHIUsers)
5610 if (!isSafePHIToSpeculate(PN&: *PHI)) {
5611 Promotable = false;
5612 PHIUsers.clear();
5613 SelectUsers.clear();
5614 break;
5615 }
5616
5617 SmallVector<std::pair<SelectInst *, RewriteableMemOps>, 2>
5618 NewSelectsToRewrite;
5619 NewSelectsToRewrite.reserve(N: SelectUsers.size());
5620 for (SelectInst *Sel : SelectUsers) {
5621 std::optional<RewriteableMemOps> Ops =
5622 isSafeSelectToSpeculate(SI&: *Sel, PreserveCFG);
5623 if (!Ops) {
5624 Promotable = false;
5625 PHIUsers.clear();
5626 SelectUsers.clear();
5627 NewSelectsToRewrite.clear();
5628 break;
5629 }
5630 NewSelectsToRewrite.emplace_back(Args: std::make_pair(x&: Sel, y&: *Ops));
5631 }
5632
5633 if (Promotable) {
5634 for (Use *U : AS.getDeadUsesIfPromotable()) {
5635 auto *OldInst = dyn_cast<Instruction>(Val: U->get());
5636 Value::dropDroppableUse(U&: *U);
5637 if (OldInst)
5638 if (isInstructionTriviallyDead(I: OldInst))
5639 DeadInsts.push_back(Elt: OldInst);
5640 }
5641 if (PHIUsers.empty() && SelectUsers.empty()) {
5642 // Promote the alloca.
5643 PromotableAllocas.insert(X: NewAI);
5644 } else {
5645 // If we have either PHIs or Selects to speculate, add them to those
5646 // worklists and re-queue the new alloca so that we promote in on the
5647 // next iteration.
5648 SpeculatablePHIs.insert_range(R&: PHIUsers);
5649 SelectsToRewrite.reserve(NumEntries: SelectsToRewrite.size() +
5650 NewSelectsToRewrite.size());
5651 for (auto &&KV : llvm::make_range(
5652 x: std::make_move_iterator(i: NewSelectsToRewrite.begin()),
5653 y: std::make_move_iterator(i: NewSelectsToRewrite.end())))
5654 SelectsToRewrite.insert(KV: std::move(KV));
5655 Worklist.insert(X: NewAI);
5656 }
5657 } else {
5658 // Drop any post-promotion work items if promotion didn't happen.
5659 while (PostPromotionWorklist.size() > PPWOldSize)
5660 PostPromotionWorklist.pop_back();
5661
5662 // We couldn't promote and we didn't create a new partition, nothing
5663 // happened.
5664 if (NewAI == &AI)
5665 return {nullptr, 0};
5666
5667 // If we can't promote the alloca, iterate on it to check for new
5668 // refinements exposed by splitting the current alloca. Don't iterate on an
5669 // alloca which didn't actually change and didn't get promoted.
5670 Worklist.insert(X: NewAI);
5671 }
5672
5673 return {NewAI, DL.getTypeSizeInBits(Ty: PartitionTy).getFixedValue()};
5674}
5675
5676// There isn't a shared interface to get the "address" parts out of a
5677// dbg.declare and dbg.assign, so provide some wrappers.
5678bool isKillAddress(const DbgVariableRecord *DVR) {
5679 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5680 return DVR->isKillAddress();
5681 return DVR->isKillLocation();
5682}
5683
5684const DIExpression *getAddressExpression(const DbgVariableRecord *DVR) {
5685 if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5686 return DVR->getAddressExpression();
5687 return DVR->getExpression();
5688}
5689
5690/// Create or replace an existing fragment in a DIExpression with \p Frag.
5691/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5692/// operation, add \p BitExtractOffset to the offset part.
5693///
5694/// Returns the new expression, or nullptr if this fails (see details below).
5695///
5696/// This function is similar to DIExpression::createFragmentExpression except
5697/// for 3 important distinctions:
5698/// 1. The new fragment isn't relative to an existing fragment.
5699/// 2. It assumes the computed location is a memory location. This means we
5700/// don't need to perform checks that creating the fragment preserves the
5701/// expression semantics.
5702/// 3. Existing extract_bits are modified independently of fragment changes
5703/// using \p BitExtractOffset. A change to the fragment offset or size
5704/// may affect a bit extract. But a bit extract offset can change
5705/// independently of the fragment dimensions.
5706///
5707/// Returns the new expression, or nullptr if one couldn't be created.
5708/// Ideally this is only used to signal that a bit-extract has become
5709/// zero-sized (and thus the new debug record has no size and can be
5710/// dropped), however, it fails for other reasons too - see the FIXME below.
5711///
5712/// FIXME: To keep the change that introduces this function NFC it bails
5713/// in some situations unecessarily, e.g. when fragment and bit extract
5714/// sizes differ.
5715static DIExpression *createOrReplaceFragment(const DIExpression *Expr,
5716 DIExpression::FragmentInfo Frag,
5717 int64_t BitExtractOffset) {
5718 SmallVector<uint64_t, 8> Ops;
5719 bool HasFragment = false;
5720 bool HasBitExtract = false;
5721
5722 for (auto &Op : Expr->expr_ops()) {
5723 if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5724 HasFragment = true;
5725 continue;
5726 }
5727 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val: Op)) {
5728 HasBitExtract = true;
5729 int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5730 int64_t ExtractSizeInBits = Extract.getSizeInBits();
5731
5732 // DIExpression::createFragmentExpression doesn't know how to handle
5733 // a fragment that is smaller than the extract. Copy the behaviour
5734 // (bail) to avoid non-NFC changes.
5735 // FIXME: Don't do this.
5736 if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5737 return nullptr;
5738
5739 assert(BitExtractOffset <= 0);
5740 int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5741
5742 // DIExpression::createFragmentExpression doesn't know what to do
5743 // if the new extract starts "outside" the existing one. Copy the
5744 // behaviour (bail) to avoid non-NFC changes.
5745 // FIXME: Don't do this.
5746 if (AdjustedOffset < 0)
5747 return nullptr;
5748
5749 Ops.push_back(Elt: Op.getOp());
5750 Ops.push_back(Elt: std::max<int64_t>(a: 0, b: AdjustedOffset));
5751 Ops.push_back(Elt: ExtractSizeInBits);
5752 continue;
5753 }
5754 Op.appendToVector(V&: Ops);
5755 }
5756
5757 // Unsupported by createFragmentExpression, so don't support it here yet to
5758 // preserve NFC-ness.
5759 if (HasFragment && HasBitExtract)
5760 return nullptr;
5761
5762 if (!HasBitExtract) {
5763 Ops.push_back(Elt: dwarf::DW_OP_LLVM_fragment);
5764 Ops.push_back(Elt: Frag.OffsetInBits);
5765 Ops.push_back(Elt: Frag.SizeInBits);
5766 }
5767 return DIExpression::get(Context&: Expr->getContext(), Elements: Ops);
5768}
5769
5770/// Insert a new DbgRecord.
5771/// \p Orig Original to copy record type, debug loc and variable from, and
5772/// additionally value and value expression for dbg_assign records.
5773/// \p NewAddr Location's new base address.
5774/// \p NewAddrExpr New expression to apply to address.
5775/// \p BeforeInst Insert position.
5776/// \p NewFragment New fragment (absolute, non-relative).
5777/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
5778static void
5779insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr,
5780 DIExpression *NewAddrExpr, Instruction *BeforeInst,
5781 std::optional<DIExpression::FragmentInfo> NewFragment,
5782 int64_t BitExtractAdjustment) {
5783 (void)DIB;
5784
5785 // A dbg_assign puts fragment info in the value expression only. The address
5786 // expression has already been built: NewAddrExpr. A dbg_declare puts the
5787 // new fragment info into NewAddrExpr (as it only has one expression).
5788 DIExpression *NewFragmentExpr =
5789 Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
5790 if (NewFragment)
5791 NewFragmentExpr = createOrReplaceFragment(Expr: NewFragmentExpr, Frag: *NewFragment,
5792 BitExtractOffset: BitExtractAdjustment);
5793 if (!NewFragmentExpr)
5794 return;
5795
5796 if (Orig->isDbgDeclare()) {
5797 DbgVariableRecord *DVR = DbgVariableRecord::createDVRDeclare(
5798 Address: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5799 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5800 Here: BeforeInst->getIterator());
5801 return;
5802 }
5803
5804 if (Orig->isDbgValue()) {
5805 DbgVariableRecord *DVR = DbgVariableRecord::createDbgVariableRecord(
5806 Location: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5807 // Drop debug information if the expression doesn't start with a
5808 // DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
5809 // describes the address of alloca rather than the value inside the alloca.
5810 if (!NewFragmentExpr->startsWithDeref())
5811 DVR->setKillAddress();
5812 BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5813 Here: BeforeInst->getIterator());
5814 return;
5815 }
5816
5817 // Apply a DIAssignID to the store if it doesn't already have it.
5818 if (!NewAddr->hasMetadata(KindID: LLVMContext::MD_DIAssignID)) {
5819 NewAddr->setMetadata(KindID: LLVMContext::MD_DIAssignID,
5820 Node: DIAssignID::getDistinct(Context&: NewAddr->getContext()));
5821 }
5822
5823 DbgVariableRecord *NewAssign = DbgVariableRecord::createLinkedDVRAssign(
5824 LinkedInstr: NewAddr, Val: Orig->getValue(), Variable: Orig->getVariable(), Expression: NewFragmentExpr, Address: NewAddr,
5825 AddressExpression: NewAddrExpr, DI: Orig->getDebugLoc());
5826 LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
5827 (void)NewAssign;
5828}
5829
5830/// Walks the slices of an alloca and form partitions based on them,
5831/// rewriting each of their uses.
5832bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
5833 if (AS.begin() == AS.end())
5834 return false;
5835
5836 unsigned NumPartitions = 0;
5837 bool Changed = false;
5838 const DataLayout &DL = AI.getModule()->getDataLayout();
5839
5840 // First try to pre-split loads and stores.
5841 Changed |= presplitLoadsAndStores(AI, AS);
5842
5843 // Now that we have identified any pre-splitting opportunities,
5844 // mark loads and stores unsplittable except for the following case.
5845 // We leave a slice splittable if all other slices are disjoint or fully
5846 // included in the slice, such as whole-alloca loads and stores.
5847 // If we fail to split these during pre-splitting, we want to force them
5848 // to be rewritten into a partition.
5849 bool IsSorted = true;
5850
5851 uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
5852 const uint64_t MaxBitVectorSize = 1024;
5853 if (AllocaSize <= MaxBitVectorSize) {
5854 // If a byte boundary is included in any load or store, a slice starting or
5855 // ending at the boundary is not splittable.
5856 SmallBitVector SplittableOffset(AllocaSize + 1, true);
5857 for (Slice &S : AS)
5858 for (unsigned O = S.beginOffset() + 1;
5859 O < S.endOffset() && O < AllocaSize; O++)
5860 SplittableOffset.reset(Idx: O);
5861
5862 for (Slice &S : AS) {
5863 if (!S.isSplittable())
5864 continue;
5865
5866 if ((S.beginOffset() > AllocaSize || SplittableOffset[S.beginOffset()]) &&
5867 (S.endOffset() > AllocaSize || SplittableOffset[S.endOffset()]))
5868 continue;
5869
5870 if (isa<LoadInst>(Val: S.getUse()->getUser()) ||
5871 isa<StoreInst>(Val: S.getUse()->getUser())) {
5872 S.makeUnsplittable();
5873 IsSorted = false;
5874 }
5875 }
5876 } else {
5877 // We only allow whole-alloca splittable loads and stores
5878 // for a large alloca to avoid creating too large BitVector.
5879 for (Slice &S : AS) {
5880 if (!S.isSplittable())
5881 continue;
5882
5883 if (S.beginOffset() == 0 && S.endOffset() >= AllocaSize)
5884 continue;
5885
5886 if (isa<LoadInst>(Val: S.getUse()->getUser()) ||
5887 isa<StoreInst>(Val: S.getUse()->getUser())) {
5888 S.makeUnsplittable();
5889 IsSorted = false;
5890 }
5891 }
5892 }
5893
5894 if (!IsSorted)
5895 llvm::stable_sort(Range&: AS);
5896
5897 /// Describes the allocas introduced by rewritePartition in order to migrate
5898 /// the debug info.
5899 struct Fragment {
5900 AllocaInst *Alloca;
5901 uint64_t Offset;
5902 uint64_t Size;
5903 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
5904 : Alloca(AI), Offset(O), Size(S) {}
5905 };
5906 SmallVector<Fragment, 4> Fragments;
5907
5908 // Rewrite each partition.
5909 for (auto &P : AS.partitions()) {
5910 auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
5911 if (NewAI) {
5912 Changed = true;
5913 if (NewAI != &AI) {
5914 uint64_t SizeOfByte = 8;
5915 // Don't include any padding.
5916 uint64_t Size = std::min(a: ActiveBits, b: P.size() * SizeOfByte);
5917 Fragments.push_back(
5918 Elt: Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
5919 }
5920 }
5921 ++NumPartitions;
5922 }
5923
5924 NumAllocaPartitions += NumPartitions;
5925 MaxPartitionsPerAlloca.updateMax(V: NumPartitions);
5926
5927 // Migrate debug information from the old alloca to the new alloca(s)
5928 // and the individual partitions.
5929 auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
5930 // Can't overlap with undef memory.
5931 if (isKillAddress(DVR: DbgVariable))
5932 return;
5933
5934 const Value *DbgPtr = DbgVariable->getAddress();
5935 DIExpression::FragmentInfo VarFrag =
5936 DbgVariable->getFragmentOrEntireVariable();
5937 // Get the address expression constant offset if one exists and the ops
5938 // that come after it.
5939 int64_t CurrentExprOffsetInBytes = 0;
5940 SmallVector<uint64_t> PostOffsetOps;
5941 if (!getAddressExpression(DVR: DbgVariable)
5942 ->extractLeadingOffset(OffsetInBytes&: CurrentExprOffsetInBytes, RemainingOps&: PostOffsetOps))
5943 return; // Couldn't interpret this DIExpression - drop the var.
5944
5945 // Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
5946 int64_t ExtractOffsetInBits = 0;
5947 for (auto Op : getAddressExpression(DVR: DbgVariable)->expr_ops()) {
5948 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val&: Op)) {
5949 ExtractOffsetInBits = Extract.getOffsetInBits();
5950 break;
5951 }
5952 }
5953
5954 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
5955 for (auto Fragment : Fragments) {
5956 int64_t OffsetFromLocationInBits;
5957 std::optional<DIExpression::FragmentInfo> NewDbgFragment;
5958 // Find the variable fragment that the new alloca slice covers.
5959 // Drop debug info for this variable fragment if we can't compute an
5960 // intersect between it and the alloca slice.
5961 if (!DIExpression::calculateFragmentIntersect(
5962 DL, SliceStart: &AI, SliceOffsetInBits: Fragment.Offset, SliceSizeInBits: Fragment.Size, DbgPtr,
5963 DbgPtrOffsetInBits: CurrentExprOffsetInBytes * 8, DbgExtractOffsetInBits: ExtractOffsetInBits, VarFrag,
5964 Result&: NewDbgFragment, OffsetFromLocationInBits))
5965 continue; // Do not migrate this fragment to this slice.
5966
5967 // Zero sized fragment indicates there's no intersect between the variable
5968 // fragment and the alloca slice. Skip this slice for this variable
5969 // fragment.
5970 if (NewDbgFragment && !NewDbgFragment->SizeInBits)
5971 continue; // Do not migrate this fragment to this slice.
5972
5973 // No fragment indicates DbgVariable's variable or fragment exactly
5974 // overlaps the slice; copy its fragment (or nullopt if there isn't one).
5975 if (!NewDbgFragment)
5976 NewDbgFragment = DbgVariable->getFragment();
5977
5978 // Reduce the new expression offset by the bit-extract offset since
5979 // we'll be keeping that.
5980 int64_t OffestFromNewAllocaInBits =
5981 OffsetFromLocationInBits - ExtractOffsetInBits;
5982 // We need to adjust an existing bit extract if the offset expression
5983 // can't eat the slack (i.e., if the new offset would be negative).
5984 int64_t BitExtractOffset =
5985 std::min<int64_t>(a: 0, b: OffestFromNewAllocaInBits);
5986 // The magnitude of a negative value indicates the number of bits into
5987 // the existing variable fragment that the memory region begins. The new
5988 // variable fragment already excludes those bits - the new DbgPtr offset
5989 // only needs to be applied if it's positive.
5990 OffestFromNewAllocaInBits =
5991 std::max(a: int64_t(0), b: OffestFromNewAllocaInBits);
5992
5993 // Rebuild the expression:
5994 // {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
5995 // Add NewDbgFragment later, because dbg.assigns don't want it in the
5996 // address expression but the value expression instead.
5997 DIExpression *NewExpr = DIExpression::get(Context&: AI.getContext(), Elements: PostOffsetOps);
5998 if (OffestFromNewAllocaInBits > 0) {
5999 int64_t OffsetInBytes = (OffestFromNewAllocaInBits + 7) / 8;
6000 NewExpr = DIExpression::prepend(Expr: NewExpr, /*flags=*/Flags: 0, Offset: OffsetInBytes);
6001 }
6002
6003 // Remove any existing intrinsics on the new alloca describing
6004 // the variable fragment.
6005 auto RemoveOne = [DbgVariable](auto *OldDII) {
6006 auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
6007 return LHS->getVariable() == RHS->getVariable() &&
6008 LHS->getDebugLoc()->getInlinedAt() ==
6009 RHS->getDebugLoc()->getInlinedAt();
6010 };
6011 if (SameVariableFragment(OldDII, DbgVariable))
6012 OldDII->eraseFromParent();
6013 };
6014 for_each(Range: findDVRDeclares(V: Fragment.Alloca), F: RemoveOne);
6015 for_each(Range: findDVRValues(V: Fragment.Alloca), F: RemoveOne);
6016 insertNewDbgInst(DIB, Orig: DbgVariable, NewAddr: Fragment.Alloca, NewAddrExpr: NewExpr, BeforeInst: &AI,
6017 NewFragment: NewDbgFragment, BitExtractAdjustment: BitExtractOffset);
6018 }
6019 };
6020
6021 // Migrate debug information from the old alloca to the new alloca(s)
6022 // and the individual partitions.
6023 for_each(Range: findDVRDeclares(V: &AI), F: MigrateOne);
6024 for_each(Range: findDVRValues(V: &AI), F: MigrateOne);
6025 for_each(Range: at::getDVRAssignmentMarkers(Inst: &AI), F: MigrateOne);
6026
6027 return Changed;
6028}
6029
6030/// Clobber a use with poison, deleting the used value if it becomes dead.
6031void SROA::clobberUse(Use &U) {
6032 Value *OldV = U;
6033 // Replace the use with an poison value.
6034 U = PoisonValue::get(T: OldV->getType());
6035
6036 // Check for this making an instruction dead. We have to garbage collect
6037 // all the dead instructions to ensure the uses of any alloca end up being
6038 // minimal.
6039 if (Instruction *OldI = dyn_cast<Instruction>(Val: OldV))
6040 if (isInstructionTriviallyDead(I: OldI)) {
6041 DeadInsts.push_back(Elt: OldI);
6042 }
6043}
6044
6045/// A basic LoadAndStorePromoter that does not remove store nodes.
6046class BasicLoadAndStorePromoter : public LoadAndStorePromoter {
6047public:
6048 BasicLoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S,
6049 Type *ZeroType)
6050 : LoadAndStorePromoter(Insts, S), ZeroType(ZeroType) {}
6051 bool shouldDelete(Instruction *I) const override {
6052 return !isa<StoreInst>(Val: I) && !isa<AllocaInst>(Val: I);
6053 }
6054
6055 Value *getValueToUseForAlloca(Instruction *I) const override {
6056 return UndefValue::get(T: ZeroType);
6057 }
6058
6059private:
6060 Type *ZeroType;
6061};
6062
6063bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6064 // Look through each "partition", looking for slices with the same start/end
6065 // that do not overlap with any before them. The slices are sorted by
6066 // increasing beginOffset. We don't use AS.partitions(), as it will use a more
6067 // sophisticated algorithm that takes splittable slices into account.
6068 LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6069 bool AllSameAndValid = true;
6070 Type *PartitionType = nullptr;
6071 SmallVector<Instruction *> Insts;
6072 uint64_t BeginOffset = 0;
6073 uint64_t EndOffset = 0;
6074
6075 auto Flush = [&]() {
6076 if (AllSameAndValid && !Insts.empty()) {
6077 LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6078 << EndOffset << ")\n");
6079 SmallVector<PHINode *, 4> NewPHIs;
6080 SSAUpdater SSA(&NewPHIs);
6081 Insts.push_back(Elt: &AI);
6082 BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6083 Promoter.run(Insts);
6084 }
6085 AllSameAndValid = true;
6086 PartitionType = nullptr;
6087 Insts.clear();
6088 };
6089
6090 for (Slice &S : AS) {
6091 auto *User = cast<Instruction>(Val: S.getUse()->getUser());
6092 if (isAssumeLikeIntrinsic(I: User)) {
6093 LLVM_DEBUG({
6094 dbgs() << "Ignoring slice: ";
6095 AS.print(dbgs(), &S);
6096 });
6097 continue;
6098 }
6099 if (S.beginOffset() >= EndOffset) {
6100 Flush();
6101 BeginOffset = S.beginOffset();
6102 EndOffset = S.endOffset();
6103 } else if (S.beginOffset() != BeginOffset || S.endOffset() != EndOffset) {
6104 if (AllSameAndValid) {
6105 LLVM_DEBUG({
6106 dbgs() << "Slice does not match range [" << BeginOffset << ", "
6107 << EndOffset << ")";
6108 AS.print(dbgs(), &S);
6109 });
6110 AllSameAndValid = false;
6111 }
6112 EndOffset = std::max(a: EndOffset, b: S.endOffset());
6113 continue;
6114 }
6115
6116 if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
6117 Type *UserTy = LI->getType();
6118 // LoadAndStorePromoter requires all the types to be the same.
6119 if (!LI->isSimple() || (PartitionType && UserTy != PartitionType))
6120 AllSameAndValid = false;
6121 PartitionType = UserTy;
6122 Insts.push_back(Elt: User);
6123 } else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
6124 Type *UserTy = SI->getValueOperand()->getType();
6125 if (!SI->isSimple() || (PartitionType && UserTy != PartitionType))
6126 AllSameAndValid = false;
6127 PartitionType = UserTy;
6128 Insts.push_back(Elt: User);
6129 } else {
6130 AllSameAndValid = false;
6131 }
6132 }
6133
6134 Flush();
6135 return true;
6136}
6137
6138/// Analyze an alloca for SROA.
6139///
6140/// This analyzes the alloca to ensure we can reason about it, builds
6141/// the slices of the alloca, and then hands it off to be split and
6142/// rewritten as needed.
6143std::pair<bool /*Changed*/, bool /*CFGChanged*/>
6144SROA::runOnAlloca(AllocaInst &AI) {
6145 bool Changed = false;
6146 bool CFGChanged = false;
6147
6148 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6149 ++NumAllocasAnalyzed;
6150
6151 // Special case dead allocas, as they're trivial.
6152 if (AI.use_empty()) {
6153 AI.eraseFromParent();
6154 Changed = true;
6155 return {Changed, CFGChanged};
6156 }
6157 const DataLayout &DL = AI.getDataLayout();
6158
6159 // Skip alloca forms that this analysis can't handle.
6160 std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6161 if (AI.isArrayAllocation() || !Size || Size->isScalable() || Size->isZero())
6162 return {Changed, CFGChanged};
6163
6164 // First, split any FCA loads and stores touching this alloca to promote
6165 // better splitting and promotion opportunities.
6166 IRBuilderTy IRB(&AI);
6167 AggLoadStoreRewriter AggRewriter(DL, IRB);
6168 Changed |= AggRewriter.rewrite(I&: AI);
6169
6170 // Build the slices using a recursive instruction-visiting builder.
6171 AllocaSlices AS(DL, AI);
6172 LLVM_DEBUG(AS.print(dbgs()));
6173 if (AS.isEscaped())
6174 return {Changed, CFGChanged};
6175
6176 if (AS.isEscapedReadOnly()) {
6177 Changed |= propagateStoredValuesToLoads(AI, AS);
6178 return {Changed, CFGChanged};
6179 }
6180
6181 // Delete all the dead users of this alloca before splitting and rewriting it.
6182 for (Instruction *DeadUser : AS.getDeadUsers()) {
6183 // Free up everything used by this instruction.
6184 for (Use &DeadOp : DeadUser->operands())
6185 clobberUse(U&: DeadOp);
6186
6187 // Now replace the uses of this instruction.
6188 DeadUser->replaceAllUsesWith(V: PoisonValue::get(T: DeadUser->getType()));
6189
6190 // And mark it for deletion.
6191 DeadInsts.push_back(Elt: DeadUser);
6192 Changed = true;
6193 }
6194 for (Use *DeadOp : AS.getDeadOperands()) {
6195 clobberUse(U&: *DeadOp);
6196 Changed = true;
6197 }
6198
6199 // No slices to split. Leave the dead alloca for a later pass to clean up.
6200 if (AS.begin() == AS.end())
6201 return {Changed, CFGChanged};
6202
6203 Changed |= splitAlloca(AI, AS);
6204
6205 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6206 while (!SpeculatablePHIs.empty())
6207 speculatePHINodeLoads(IRB, PN&: *SpeculatablePHIs.pop_back_val());
6208
6209 LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6210 auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6211 while (!RemainingSelectsToRewrite.empty()) {
6212 const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6213 CFGChanged |=
6214 rewriteSelectInstMemOps(SI&: *K, Ops: V, IRB, DTU: PreserveCFG ? nullptr : DTU);
6215 }
6216
6217 return {Changed, CFGChanged};
6218}
6219
6220/// Delete the dead instructions accumulated in this run.
6221///
6222/// Recursively deletes the dead instructions we've accumulated. This is done
6223/// at the very end to maximize locality of the recursive delete and to
6224/// minimize the problems of invalidated instruction pointers as such pointers
6225/// are used heavily in the intermediate stages of the algorithm.
6226///
6227/// We also record the alloca instructions deleted here so that they aren't
6228/// subsequently handed to mem2reg to promote.
6229bool SROA::deleteDeadInstructions(
6230 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6231 bool Changed = false;
6232 while (!DeadInsts.empty()) {
6233 Instruction *I = dyn_cast_or_null<Instruction>(Val: DeadInsts.pop_back_val());
6234 if (!I)
6235 continue;
6236 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6237
6238 // If the instruction is an alloca, find the possible dbg.declare connected
6239 // to it, and remove it too. We must do this before calling RAUW or we will
6240 // not be able to find it.
6241 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: I)) {
6242 DeletedAllocas.insert(Ptr: AI);
6243 for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6244 OldDII->eraseFromParent();
6245 }
6246
6247 at::deleteAssignmentMarkers(Inst: I);
6248 I->replaceAllUsesWith(V: UndefValue::get(T: I->getType()));
6249
6250 for (Use &Operand : I->operands())
6251 if (Instruction *U = dyn_cast<Instruction>(Val&: Operand)) {
6252 // Zero out the operand and see if it becomes trivially dead.
6253 Operand = nullptr;
6254 if (isInstructionTriviallyDead(I: U))
6255 DeadInsts.push_back(Elt: U);
6256 }
6257
6258 ++NumDeleted;
6259 I->eraseFromParent();
6260 Changed = true;
6261 }
6262 return Changed;
6263}
6264/// Promote the allocas, using the best available technique.
6265///
6266/// This attempts to promote whatever allocas have been identified as viable in
6267/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6268/// This function returns whether any promotion occurred.
6269bool SROA::promoteAllocas() {
6270 if (PromotableAllocas.empty())
6271 return false;
6272
6273 if (SROASkipMem2Reg) {
6274 LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6275 } else {
6276 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6277 NumPromoted += PromotableAllocas.size();
6278 PromoteMemToReg(Allocas: PromotableAllocas.getArrayRef(), DT&: DTU->getDomTree(), AC);
6279 }
6280
6281 PromotableAllocas.clear();
6282 return true;
6283}
6284
6285std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
6286 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6287
6288 const DataLayout &DL = F.getDataLayout();
6289 BasicBlock &EntryBB = F.getEntryBlock();
6290 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(x: EntryBB.end());
6291 I != E; ++I) {
6292 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val&: I)) {
6293 std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6294 if (Size && Size->isScalable() && isAllocaPromotable(AI))
6295 PromotableAllocas.insert(X: AI);
6296 else
6297 Worklist.insert(X: AI);
6298 }
6299 }
6300
6301 bool Changed = false;
6302 bool CFGChanged = false;
6303 // A set of deleted alloca instruction pointers which should be removed from
6304 // the list of promotable allocas.
6305 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
6306
6307 do {
6308 while (!Worklist.empty()) {
6309 auto [IterationChanged, IterationCFGChanged] =
6310 runOnAlloca(AI&: *Worklist.pop_back_val());
6311 Changed |= IterationChanged;
6312 CFGChanged |= IterationCFGChanged;
6313
6314 Changed |= deleteDeadInstructions(DeletedAllocas);
6315
6316 // Remove the deleted allocas from various lists so that we don't try to
6317 // continue processing them.
6318 if (!DeletedAllocas.empty()) {
6319 Worklist.set_subtract(DeletedAllocas);
6320 PostPromotionWorklist.set_subtract(DeletedAllocas);
6321 PromotableAllocas.set_subtract(DeletedAllocas);
6322 DeletedAllocas.clear();
6323 }
6324 }
6325
6326 Changed |= promoteAllocas();
6327
6328 Worklist = PostPromotionWorklist;
6329 PostPromotionWorklist.clear();
6330 } while (!Worklist.empty());
6331
6332 assert((!CFGChanged || Changed) && "Can not only modify the CFG.");
6333 assert((!CFGChanged || !PreserveCFG) &&
6334 "Should not have modified the CFG when told to preserve it.");
6335
6336 if (Changed && isAssignmentTrackingEnabled(M: *F.getParent())) {
6337 for (auto &BB : F) {
6338 RemoveRedundantDbgInstrs(BB: &BB);
6339 }
6340 }
6341
6342 return {Changed, CFGChanged};
6343}
6344
6345PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
6346 DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
6347 AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
6348 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6349 auto [Changed, CFGChanged] =
6350 SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6351 if (!Changed)
6352 return PreservedAnalyses::all();
6353 PreservedAnalyses PA;
6354 if (!CFGChanged)
6355 PA.preserveSet<CFGAnalyses>();
6356 PA.preserve<DominatorTreeAnalysis>();
6357 return PA;
6358}
6359
6360void SROAPass::printPipeline(
6361 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6362 static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
6363 OS, MapClassName2PassName);
6364 OS << '<'
6365 << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6366 : "modify-cfg");
6367 if (Options.AggregateToVector)
6368 OS << ";aggregate-to-vector";
6369 OS << '>';
6370}
6371
6372SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
6373
6374namespace {
6375
6376/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6377class SROALegacyPass : public FunctionPass {
6378 SROAOptions Options;
6379
6380public:
6381 static char ID;
6382
6383 SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
6384 : FunctionPass(ID), Options(Options) {
6385 initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
6386 }
6387
6388 bool runOnFunction(Function &F) override {
6389 if (skipFunction(F))
6390 return false;
6391
6392 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6393 AssumptionCache &AC =
6394 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6395 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6396 auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6397 return Changed;
6398 }
6399
6400 void getAnalysisUsage(AnalysisUsage &AU) const override {
6401 AU.addRequired<AssumptionCacheTracker>();
6402 AU.addRequired<DominatorTreeWrapperPass>();
6403 AU.addPreserved<GlobalsAAWrapperPass>();
6404 AU.addPreserved<DominatorTreeWrapperPass>();
6405 }
6406
6407 StringRef getPassName() const override { return "SROA"; }
6408};
6409
6410} // end anonymous namespace
6411
6412char SROALegacyPass::ID = 0;
6413
6414FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
6415 return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
6416 : SROAOptions::ModifyCFG,
6417 AggregateToVector));
6418}
6419
6420INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6421 "Scalar Replacement Of Aggregates", false, false)
6422INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6423INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
6424INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
6425 false, false)
6426