SROA.cpp source code [llvm_projects/llvm/lib/Transforms/Scalar/SROA.cpp]

1	//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	/// \file
9	/// This transformation implements the well known scalar replacement of
10	/// aggregates transformation. It tries to identify promotable elements of an
11	/// aggregate alloca, and promote them to registers. It will also try to
12	/// convert uses of an element (or set of elements) of an alloca into a vector
13	/// or bitfield-style integer scalar if appropriate.
14	///
15	/// It works to do this with minimal slicing of the alloca so that regions
16	/// which are merely transferred in and out of external memory remain unchanged
17	/// and are not decomposed to scalar code.
18	///
19	/// Because this also performs alloca promotion, it can be thought of as also
20	/// serving the purpose of SSA formation. The algorithm iterates on the
21	/// function until all opportunities for promotion have been realized.
22	///
23	//===----------------------------------------------------------------------===//
24
25	#include "llvm/Transforms/Scalar/SROA.h"
26	#include "ScalarOptions.h"
27	#include "llvm/ADT/APInt.h"
28	#include "llvm/ADT/ArrayRef.h"
29	#include "llvm/ADT/DenseMap.h"
30	#include "llvm/ADT/MapVector.h"
31	#include "llvm/ADT/PointerIntPair.h"
32	#include "llvm/ADT/STLExtras.h"
33	#include "llvm/ADT/SetVector.h"
34	#include "llvm/ADT/SmallPtrSet.h"
35	#include "llvm/ADT/SmallVector.h"
36	#include "llvm/ADT/SparseBitVector.h"
37	#include "llvm/ADT/Statistic.h"
38	#include "llvm/ADT/StringRef.h"
39	#include "llvm/ADT/Twine.h"
40	#include "llvm/ADT/iterator.h"
41	#include "llvm/ADT/iterator_range.h"
42	#include "llvm/Analysis/AssumptionCache.h"
43	#include "llvm/Analysis/DomTreeUpdater.h"
44	#include "llvm/Analysis/GlobalsModRef.h"
45	#include "llvm/Analysis/Loads.h"
46	#include "llvm/Analysis/PtrUseVisitor.h"
47	#include "llvm/Analysis/ValueTracking.h"
48	#include "llvm/Analysis/VectorUtils.h"
49	#include "llvm/IR/BasicBlock.h"
50	#include "llvm/IR/Constant.h"
51	#include "llvm/IR/ConstantFolder.h"
52	#include "llvm/IR/Constants.h"
53	#include "llvm/IR/DIBuilder.h"
54	#include "llvm/IR/DataLayout.h"
55	#include "llvm/IR/DebugInfo.h"
56	#include "llvm/IR/DebugInfoMetadata.h"
57	#include "llvm/IR/DerivedTypes.h"
58	#include "llvm/IR/Dominators.h"
59	#include "llvm/IR/Function.h"
60	#include "llvm/IR/GlobalAlias.h"
61	#include "llvm/IR/IRBuilder.h"
62	#include "llvm/IR/InstVisitor.h"
63	#include "llvm/IR/Instruction.h"
64	#include "llvm/IR/Instructions.h"
65	#include "llvm/IR/IntrinsicInst.h"
66	#include "llvm/IR/LLVMContext.h"
67	#include "llvm/IR/Metadata.h"
68	#include "llvm/IR/Module.h"
69	#include "llvm/IR/Operator.h"
70	#include "llvm/IR/PassManager.h"
71	#include "llvm/IR/Type.h"
72	#include "llvm/IR/Use.h"
73	#include "llvm/IR/User.h"
74	#include "llvm/IR/Value.h"
75	#include "llvm/IR/ValueHandle.h"
76	#include "llvm/InitializePasses.h"
77	#include "llvm/Pass.h"
78	#include "llvm/Support/Casting.h"
79	#include "llvm/Support/Compiler.h"
80	#include "llvm/Support/Debug.h"
81	#include "llvm/Support/ErrorHandling.h"
82	#include "llvm/Support/raw_ostream.h"
83	#include "llvm/Transforms/Scalar.h"
84	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
85	#include "llvm/Transforms/Utils/Local.h"
86	#include "llvm/Transforms/Utils/PromoteMemToReg.h"
87	#include "llvm/Transforms/Utils/SSAUpdater.h"
88	#include <algorithm>
89	#include <cassert>
90	#include <cstddef>
91	#include <cstdint>
92	#include <cstring>
93	#include <iterator>
94	#include <string>
95	#include <tuple>
96	#include <utility>
97	#include <variant>
98	#include <vector>
99
100	using namespace llvm;
101
102	#define DEBUG_TYPE "sroa"
103
104	STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105	STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106	STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107	STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108	STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109	STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110	STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111	STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112	STATISTIC(NumLoadsPredicated,
113	"Number of loads rewritten into predicated loads to allow promotion");
114	STATISTIC(
115	NumStoresPredicated,
116	"Number of stores rewritten into predicated stores to allow promotion");
117	STATISTIC(NumDeleted, "Number of instructions deleted");
118	STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120	namespace {
121
122	class AllocaSliceRewriter;
123	class AllocaSlices;
124	class Partition;
125	class Slice;
126
127	class SelectHandSpeculativity {
128	unsigned char Storage = `0`; // None are speculatable by default.
129	using TrueVal = Bitfield::Element<bool, `0`, `1`>; // Low 0'th bit.
130	using FalseVal = Bitfield::Element<bool, `1`, `1`>; // Low 1'th bit.
131	public:
132	SelectHandSpeculativity() = default;
133	SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
134	bool isSpeculatable(bool isTrueVal) const;
135	bool areAllSpeculatable() const;
136	bool areAnySpeculatable() const;
137	bool areNoneSpeculatable() const;
138	// For interop as int half of PointerIntPair.
139	explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
140	explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
141	};
142	static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
143
144	using PossiblySpeculatableLoad =
145	PointerIntPair<LoadInst *, `2`, SelectHandSpeculativity>;
146	using UnspeculatableStore = StoreInst *;
147	using RewriteableMemOp =
148	std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
149	using RewriteableMemOps = SmallVector<RewriteableMemOp, `2`>;
150	using InstructionSliceMap =
151	SmallDenseMap<Instruction , SmallPtrSet<Slice , `8`>, `8`>;
152
153	/// An optimization pass providing Scalar Replacement of Aggregates.
154	///
155	/// This pass takes allocations which can be completely analyzed (that is, they
156	/// don't escape) and tries to turn them into scalar SSA values. There are
157	/// a few steps to this process.
158	///
159	/// 1) It takes allocations of aggregates and analyzes the ways in which they
160	/// are used to try to split them into smaller allocations, ideally of
161	/// a single scalar data type. It will split up memcpy and memset accesses
162	/// as necessary and try to isolate individual scalar accesses.
163	/// 2) It will transform accesses into forms which are suitable for SSA value
164	/// promotion. This can be replacing a memset with a scalar store of an
165	/// integer value, or it can involve speculating operations on a PHI or
166	/// select to be a PHI or select of the results.
167	/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
168	/// onto insert and extract operations on a vector value, and convert them to
169	/// this form. By doing so, it will enable promotion of vector aggregates to
170	/// SSA vector values.
171	class SROA {
172	LLVMContext *const C;
173	DomTreeUpdater *const DTU;
174	AssumptionCache *const AC;
175	const bool PreserveCFG;
176	const bool AggregateToVector;
177
178	/// Worklist of alloca instructions to simplify.
179	///
180	/// Each alloca in the function is added to this. Each new alloca formed gets
181	/// added to it as well to recursively simplify unless that alloca can be
182	/// directly promoted. Finally, each time we rewrite a use of an alloca other
183	/// the one being actively rewritten, we add it back onto the list if not
184	/// already present to ensure it is re-visited.
185	SmallSetVector<AllocaInst *, `16`> Worklist;
186
187	/// A collection of instructions to delete.
188	/// We try to batch deletions to simplify code and make things a bit more
189	/// efficient. We also make sure there is no dangling pointers.
190	SmallVector<WeakVH, `8`> DeadInsts;
191
192	/// Post-promotion worklist.
193	///
194	/// Sometimes we discover an alloca which has a high probability of becoming
195	/// viable for SROA after a round of promotion takes place. In those cases,
196	/// the alloca is enqueued here for re-processing.
197	///
198	/// Note that we have to be very careful to clear allocas out of this list in
199	/// the event they are deleted.
200	SmallSetVector<AllocaInst *, `16`> PostPromotionWorklist;
201
202	/// A collection of alloca instructions we can directly promote.
203	SetVector<AllocaInst , SmallVector<AllocaInst >,
204	SmallPtrSet<AllocaInst *, `16`>, `16`>
205	PromotableAllocas;
206
207	/// A worklist of PHIs to speculate prior to promoting allocas.
208	///
209	/// All of these PHIs have been checked for the safety of speculation and by
210	/// being speculated will allow promoting allocas currently in the promotable
211	/// queue.
212	SmallSetVector<PHINode *, `8`> SpeculatablePHIs;
213
214	/// A worklist of PHIs whose stores should be predicated onto incoming edges.
215	///
216	/// All of these PHIs have been checked so that rewriting their stores will
217	/// allow the corresponding allocas to be promoted. Edges which require
218	/// splitting are only present when CFG modification is allowed.
219	SmallSetVector<PHINode *, `8`> PHIsWithStoreToRewrite;
220
221	/// A worklist of select instructions to rewrite prior to promoting
222	/// allocas.
223	SmallMapVector<SelectInst *, RewriteableMemOps, `8`> SelectsToRewrite;
224
225	/// Select instructions that use an alloca and are subsequently loaded can be
226	/// rewritten to load both input pointers and then select between the result,
227	/// allowing the load of the alloca to be promoted.
228	/// From this:
229	/// %P2 = select i1 %cond, ptr %Alloca, ptr %Other
230	/// %V = load <type>, ptr %P2
231	/// to:
232	/// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
233	/// %V2 = load <type>, ptr %Other
234	/// %V = select i1 %cond, <type> %V1, <type> %V2
235	///
236	/// We can do this to a select if its only uses are loads
237	/// and if either the operand to the select can be loaded unconditionally,
238	/// or if we are allowed to perform CFG modifications.
239	static std::optional<RewriteableMemOps>
240	isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
241
242	public:
243	SROA(LLVMContext C, DomTreeUpdater DTU, AssumptionCache *AC,
244	SROAOptions Options)
245	: C(C), DTU(DTU), AC(AC),
246	PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
247	AggregateToVector(Options.AggregateToVector) {}
248
249	/// Main run method used by both the SROAPass and by the legacy pass.
250	std::pair<bool /Changed/, bool /CFGChanged/> runSROA(Function &F);
251
252	private:
253	friend class AllocaSliceRewriter;
254
255	bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
256	std::pair<AllocaInst *, uint64_t>
257	rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
258	bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
259	bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
260	std::pair<bool /Changed/, bool /CFGChanged/> runOnAlloca(AllocaInst &AI);
261	void clobberUse(Use &U);
262	bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
263	bool promoteAllocas();
264	};
265
266	} // end anonymous namespace
267
268	/// Calculate the fragment of a variable to use when slicing a store
269	/// based on the slice dimensions, existing fragment, and base storage
270	/// fragment.
271	/// Results:
272	/// UseFrag - Use Target as the new fragment.
273	/// UseNoFrag - The new slice already covers the whole variable.
274	/// Skip - The new alloca slice doesn't include this variable.
275	/// FIXME: Can we use calculateFragmentIntersect instead?
276	namespace {
277	enum FragCalcResult { UseFrag, UseNoFrag, Skip };
278	}
279	static FragCalcResult
280	calculateFragment(DILocalVariable *Variable,
281	uint64_t NewStorageSliceOffsetInBits,
282	uint64_t NewStorageSliceSizeInBits,
283	std::optional<DIExpression::FragmentInfo> StorageFragment,
284	std::optional<DIExpression::FragmentInfo> CurrentFragment,
285	DIExpression::FragmentInfo &Target) {
286	// If the base storage describes part of the variable apply the offset and
287	// the size constraint.
288	if (StorageFragment) {
289	Target.SizeInBits =
290	std::min(a: NewStorageSliceSizeInBits, b: StorageFragment ->SizeInBits);
291	Target.OffsetInBits =
292	NewStorageSliceOffsetInBits + StorageFragment ->OffsetInBits;
293	} else {
294	Target.SizeInBits = NewStorageSliceSizeInBits;
295	Target.OffsetInBits = NewStorageSliceOffsetInBits;
296	}
297
298	// If this slice extracts the entirety of an independent variable from a
299	// larger alloca, do not produce a fragment expression, as the variable is
300	// not fragmented.
301	if (!CurrentFragment) {
302	if (auto Size = Variable->getSizeInBits()) {
303	// Treat the current fragment as covering the whole variable.
304	CurrentFragment = DIExpression::FragmentInfo(*Size, `0`);
305	if (Target == CurrentFragment)
306	return UseNoFrag;
307	}
308	}
309
310	// No additional work to do if there isn't a fragment already, or there is
311	// but it already exactly describes the new assignment.
312	if (!CurrentFragment \|\| *CurrentFragment == Target)
313	return UseFrag;
314
315	// Reject the target fragment if it doesn't fit wholly within the current
316	// fragment. TODO: We could instead chop up the target to fit in the case of
317	// a partial overlap.
318	if (Target.startInBits() < CurrentFragment ->startInBits() \|\|
319	Target.endInBits() > CurrentFragment ->endInBits())
320	return Skip;
321
322	// Target fits within the current fragment, return it.
323	return UseFrag;
324	}
325
326	static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
327	return DebugVariable (DVR->getVariable(), std::nullopt,
328	DVR->getDebugLoc().getInlinedAt());
329	}
330
331	/// Find linked dbg.assign and generate a new one with the correct
332	/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
333	/// value component is copied from the old dbg.assign to the new.
334	/// \param OldAlloca Alloca for the variable before splitting.
335	/// \param IsSplit True if the store (not necessarily alloca)
336	/// is being split.
337	/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
338	/// \param SliceSizeInBits New number of bits being written to.
339	/// \param OldInst Instruction that is being split.
340	/// \param Inst New instruction performing this part of the
341	/// split store.
342	/// \param Dest Store destination.
343	/// \param Value Stored value.
344	/// \param DL Datalayout.
345	static void migrateDebugInfo(AllocaInst OldAlloca, bool* IsSplit,
346	uint64_t OldAllocaOffsetInBits,
347	uint64_t SliceSizeInBits, Instruction *OldInst,
348	Instruction Inst, Value Dest, Value *Value,
349	const DataLayout &DL) {
350	// If we want allocas to be migrated using this helper then we need to ensure
351	// that the BaseFragments map code still works. A simple solution would be
352	// to choose to always clone alloca dbg_assigns (rather than sometimes
353	// "stealing" them).
354	assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
355
356	auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(Inst: OldInst);
357	// Nothing to do if OldInst has no linked dbg.assign intrinsics.
358	if (DVRAssignMarkerRange.empty())
359	return;
360
361	LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
362	LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
363	LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
364	LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
365	<< "\n");
366	LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
367	LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
368	LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
369	LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
370	if (Value)
371	LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
372
373	/// Map of aggregate variables to their fragment associated with OldAlloca.
374	DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>>
375	BaseFragments;
376	for (auto *DVR : at::getDVRAssignmentMarkers(Inst: OldAlloca))
377	BaseFragments [getAggregateVariable(DVR)] =
378	DVR->getExpression()->getFragmentInfo();
379
380	// The new inst needs a DIAssignID unique metadata tag (if OldInst has
381	// one). It shouldn't already have one: assert this assumption.
382	assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
383	DIAssignID NewID = nullptr*;
384	auto &Ctx = Inst->getContext();
385	DIBuilder DIB(OldInst->getModule(), /AllowUnresolved/* false);
386	assert(OldAlloca->isStaticAlloca());
387
388	auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
389	LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
390	<< "\n");
391	auto *Expr = DbgAssign->getExpression();
392	bool SetKillLocation = false;
393
394	if (IsSplit) {
395	std::optional<DIExpression::FragmentInfo> BaseFragment;
396	{
397	auto R = BaseFragments.find(Val: getAggregateVariable(DVR: DbgAssign));
398	if (R == BaseFragments.end())
399	return;
400	BaseFragment = R ->second;
401	}
402	std::optional<DIExpression::FragmentInfo> CurrentFragment =
403	Expr->getFragmentInfo();
404	DIExpression::FragmentInfo NewFragment;
405	FragCalcResult Result = calculateFragment(
406	Variable: DbgAssign->getVariable(), NewStorageSliceOffsetInBits: OldAllocaOffsetInBits, NewStorageSliceSizeInBits: SliceSizeInBits,
407	StorageFragment: BaseFragment, CurrentFragment, Target&: NewFragment);
408
409	if (Result == Skip)
410	return;
411	if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
412	if (CurrentFragment) {
413	// Rewrite NewFragment to be relative to the existing one (this is
414	// what createFragmentExpression wants). CalculateFragment has
415	// already resolved the size for us. FIXME: Should it return the
416	// relative fragment too?
417	NewFragment.OffsetInBits -= CurrentFragment ->OffsetInBits;
418	}
419	// Add the new fragment info to the existing expression if possible.
420	if (auto E = DIExpression::createFragmentExpression(
421	Expr, OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits)) {
422	Expr = *E;
423	} else {
424	// Otherwise, add the new fragment info to an empty expression and
425	// discard the value component of this dbg.assign as the value cannot
426	// be computed with the new fragment.
427	Expr = *DIExpression::createFragmentExpression(
428	Expr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
429	OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits);
430	SetKillLocation = true;
431	}
432	}
433	}
434
435	// If we haven't created a DIAssignID ID do that now and attach it to Inst.
436	if (!NewID) {
437	NewID = DIAssignID::getDistinct(Context&: Ctx);
438	Inst->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: NewID);
439	}
440
441	DbgVariableRecord *NewAssign;
442	if (IsSplit) {
443	::Value *NewValue = Value ? Value : DbgAssign->getValue();
444	NewAssign = cast<DbgVariableRecord>(Val: DIB.insertDbgAssign(
445	LinkedInstr: Inst, Val: NewValue, SrcVar: DbgAssign->getVariable(), ValExpr: Expr, Addr: Dest,
446	AddrExpr: DIExpression::get(Context&: Expr->getContext(), Elements: {}), DL: DbgAssign->getDebugLoc()));
447	} else {
448	// The store is not split, simply steal the existing dbg_assign.
449	NewAssign = DbgAssign;
450	NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
451	NewAssign->setAddress(Dest);
452	if (Value)
453	NewAssign->replaceVariableLocationOp(OpIdx: `0u`, NewValue: Value);
454	assert(Expr == NewAssign->getExpression());
455	}
456
457	// If we've updated the value but the original dbg.assign has an arglist
458	// then kill it now - we can't use the requested new value.
459	// We can't replace the DIArgList with the new value as it'd leave
460	// the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
461	// an arglist). And we can't keep the DIArgList in case the linked store
462	// is being split - in which case the DIArgList + expression may no longer
463	// be computing the correct value.
464	// This should be a very rare situation as it requires the value being
465	// stored to differ from the dbg.assign (i.e., the value has been
466	// represented differently in the debug intrinsic for some reason).
467	SetKillLocation \|=
468	Value && (DbgAssign->hasArgList() \|\|
469	!DbgAssign->getExpression()->isSingleLocationExpression());
470	if (SetKillLocation)
471	NewAssign->setKillLocation();
472
473	// We could use more precision here at the cost of some additional (code)
474	// complexity - if the original dbg.assign was adjacent to its store, we
475	// could position this new dbg.assign adjacent to its store rather than the
476	// old dbg.assgn. That would result in interleaved dbg.assigns rather than
477	// what we get now:
478	// split store !1
479	// split store !2
480	// dbg.assign !1
481	// dbg.assign !2
482	// This (current behaviour) results results in debug assignments being
483	// noted as slightly offset (in code) from the store. In practice this
484	// should have little effect on the debugging experience due to the fact
485	// that all the split stores should get the same line number.
486	if (NewAssign != DbgAssign) {
487	NewAssign->moveBefore(MoveBefore: DbgAssign->getIterator());
488	NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
489	}
490	LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
491	};
492
493	for_each(Range&: DVRAssignMarkerRange, F: MigrateDbgAssign);
494	}
495
496	namespace {
497
498	/// A custom IRBuilder inserter which prefixes all names, but only in
499	/// Assert builds.
500	class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
501	std::string Prefix;
502
503	Twine getNameWithPrefix(const Twine &Name) const {
504	return Name.isTriviallyEmpty() ? Name : Prefix + Name;
505	}
506
507	public:
508	void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
509
510	void InsertHelper(Instruction I, const* Twine &Name,
511	BasicBlock::iterator InsertPt) const {
512	IRBuilderDefaultInserter::InsertHelper(I, Name: getNameWithPrefix(Name),
513	InsertPt);
514	}
515	};
516
517	/// Provide a type for IRBuilder that drops names in release builds.
518	using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
519
520	/// A used slice of an alloca.
521	///
522	/// This structure represents a slice of an alloca used by some instruction. It
523	/// stores both the begin and end offsets of this use, a pointer to the use
524	/// itself, and a flag indicating whether we can classify the use as splittable
525	/// or not when forming partitions of the alloca.
526	class Slice {
527	/// The beginning offset of the range.
528	uint64_t BeginOffset = `0`;
529
530	/// The ending offset, not included in the range.
531	uint64_t EndOffset = `0`;
532
533	/// Storage for both the use of this slice and whether it can be
534	/// split.
535	PointerIntPair<Use , `1`, bool*> UseAndIsSplittable;
536
537	public:
538	Slice() = default;
539
540	Slice(uint64_t BeginOffset, uint64_t EndOffset, Use U, bool* IsSplittable)
541	: BeginOffset(BeginOffset), EndOffset(EndOffset),
542	UseAndIsSplittable (U, IsSplittable) {}
543
544	uint64_t beginOffset() const { return BeginOffset; }
545	uint64_t endOffset() const { return EndOffset; }
546
547	bool isSplittable() const { return UseAndIsSplittable.getInt(); }
548	void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
549
550	Use getUse() const* { return UseAndIsSplittable.getPointer(); }
551
552	bool isDead() const { return getUse() == nullptr; }
553	void kill() { UseAndIsSplittable.setPointer(nullptr); }
554
555	/// Support for ordering ranges.
556	///
557	/// This provides an ordering over ranges such that start offsets are
558	/// always increasing, and within equal start offsets, the end offsets are
559	/// decreasing. Thus the spanning range comes first in a cluster with the
560	/// same start position.
561	bool operator<(const Slice &RHS) const {
562	if (beginOffset() < RHS.beginOffset())
563	return true;
564	if (beginOffset() > RHS.beginOffset())
565	return false;
566	if (isSplittable() != RHS.isSplittable())
567	return !isSplittable();
568	if (endOffset() > RHS.endOffset())
569	return true;
570	return false;
571	}
572
573	/// Support comparison with a single offset to allow binary searches.
574	[[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
575	return LHS.beginOffset() < RHSOffset;
576	}
577	[[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
578	return LHSOffset < RHS.beginOffset();
579	}
580
581	bool operator==(const Slice &RHS) const {
582	return isSplittable() == RHS.isSplittable() &&
583	beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
584	}
585	bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
586	};
587
588	/// Representation of the alloca slices.
589	///
590	/// This class represents the slices of an alloca which are formed by its
591	/// various uses. If a pointer escapes, we can't fully build a representation
592	/// for the slices used and we reflect that in this structure. The uses are
593	/// stored, sorted by increasing beginning offset and with unsplittable slices
594	/// starting at a particular offset before splittable slices.
595	class AllocaSlices {
596	public:
597	/// Construct the slices of a particular alloca.
598	AllocaSlices(const DataLayout &DL, AllocaInst &AI);
599
600	/// Test whether a pointer to the allocation escapes our analysis.
601	///
602	/// If this is true, the slices are never fully built and should be
603	/// ignored.
604	bool isEscaped() const { return PointerEscapingInstr; }
605	bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
606
607	/// Support for iterating over the slices.
608	/// @{
609	using iterator = SmallVectorImpl<Slice>::iterator;
610	using range = iterator_range<iterator>;
611
612	iterator begin() { return Slices.begin(); }
613	iterator end() { return Slices.end(); }
614
615	using const_iterator = SmallVectorImpl<Slice>::const_iterator;
616	using const_range = iterator_range<const_iterator>;
617
618	const_iterator begin() const { return Slices.begin(); }
619	const_iterator end() const { return Slices.end(); }
620	/// @}
621
622	/// Erase a range of slices.
623	void erase(iterator Start, iterator Stop) { Slices.erase(CS: Start, CE: Stop); }
624
625	/// Insert new slices for this alloca.
626	///
627	/// This moves the slices into the alloca's slices collection, and re-sorts
628	/// everything so that the usual ordering properties of the alloca's slices
629	/// hold.
630	void insert(ArrayRef<Slice> NewSlices) {
631	int OldSize = Slices.size();
632	Slices.append(in_start: NewSlices.begin(), in_end: NewSlices.end());
633	auto SliceI = Slices.begin() + OldSize;
634	std::stable_sort(first: SliceI, last: Slices.end());
635	std::inplace_merge(first: Slices.begin(), middle: SliceI, last: Slices.end());
636	}
637
638	// Forward declare the iterator and range accessor for walking the
639	// partitions.
640	class partition_iterator;
641	iterator_range<partition_iterator> partitions();
642
643	/// Access the dead users for this alloca.
644	ArrayRef<Instruction > getDeadUsers() const* { return DeadUsers; }
645
646	/// Access Uses that should be dropped if the alloca is promotable.
647	ArrayRef<Use > getDeadUsesIfPromotable() const* {
648	return DeadUseIfPromotable;
649	}
650
651	/// Access the dead operands referring to this alloca.
652	///
653	/// These are operands which have cannot actually be used to refer to the
654	/// alloca as they are outside its range and the user doesn't correct for
655	/// that. These mostly consist of PHI node inputs and the like which we just
656	/// need to replace with undef.
657	ArrayRef<Use > getDeadOperands() const* { return DeadOperands; }
658
659	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
660	void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
661	void printSlice(raw_ostream &OS, const_iterator I,
662	StringRef Indent = " ") const;
663	void printUse(raw_ostream &OS, const_iterator I,
664	StringRef Indent = " ") const;
665	void print(raw_ostream &OS) const;
666	void dump(const_iterator I) const;
667	void dump() const;
668	#endif
669
670	private:
671	template <typename DerivedT, typename RetT = void> class BuilderBase;
672	class SliceBuilder;
673
674	friend class AllocaSlices::SliceBuilder;
675
676	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
677	/// Handle to alloca instruction to simplify method interfaces.
678	AllocaInst &AI;
679	#endif
680
681	/// The instruction responsible for this alloca not having a known set
682	/// of slices.
683	///
684	/// When an instruction (potentially) escapes the pointer to the alloca, we
685	/// store a pointer to that here and abort trying to form slices of the
686	/// alloca. This will be null if the alloca slices are analyzed successfully.
687	Instruction *PointerEscapingInstr;
688	Instruction *PointerEscapingInstrReadOnly;
689
690	/// The slices of the alloca.
691	///
692	/// We store a vector of the slices formed by uses of the alloca here. This
693	/// vector is sorted by increasing begin offset, and then the unsplittable
694	/// slices before the splittable ones. See the Slice inner class for more
695	/// details.
696	SmallVector<Slice, `8`> Slices;
697
698	/// Instructions which will become dead if we rewrite the alloca.
699	///
700	/// Note that these are not separated by slice. This is because we expect an
701	/// alloca to be completely rewritten or not rewritten at all. If rewritten,
702	/// all these instructions can simply be removed and replaced with poison as
703	/// they come from outside of the allocated space.
704	SmallVector<Instruction *, `8`> DeadUsers;
705
706	/// Uses which will become dead if can promote the alloca.
707	SmallVector<Use *, `8`> DeadUseIfPromotable;
708
709	/// Operands which will become dead if we rewrite the alloca.
710	///
711	/// These are operands that in their particular use can be replaced with
712	/// poison when we rewrite the alloca. These show up in out-of-bounds inputs
713	/// to PHI nodes and the like. They aren't entirely dead (there might be
714	/// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
715	/// want to swap this particular input for poison to simplify the use lists of
716	/// the alloca.
717	SmallVector<Use *, `8`> DeadOperands;
718	};
719
720	/// A partition of the slices.
721	///
722	/// An ephemeral representation for a range of slices which can be viewed as
723	/// a partition of the alloca. This range represents a span of the alloca's
724	/// memory which cannot be split, and provides access to all of the slices
725	/// overlapping some part of the partition.
726	///
727	/// Objects of this type are produced by traversing the alloca's slices, but
728	/// are only ephemeral and not persistent.
729	class Partition {
730	private:
731	friend class AllocaSlices;
732	friend class AllocaSlices::partition_iterator;
733
734	using iterator = AllocaSlices::iterator;
735
736	/// The beginning and ending offsets of the alloca for this
737	/// partition.
738	uint64_t BeginOffset = `0`, EndOffset = `0`;
739
740	/// The start and end iterators of this partition.
741	iterator SI, SJ;
742
743	/// A collection of split slice tails overlapping the partition.
744	SmallVector<Slice *, `4`> SplitTails;
745
746	/// Raw constructor builds an empty partition starting and ending at
747	/// the given iterator.
748	Partition(iterator SI) : SI(SI), SJ(SI) {}
749
750	public:
751	/// The start offset of this partition.
752	///
753	/// All of the contained slices start at or after this offset.
754	uint64_t beginOffset() const { return BeginOffset; }
755
756	/// The end offset of this partition.
757	///
758	/// All of the contained slices end at or before this offset.
759	uint64_t endOffset() const { return EndOffset; }
760
761	/// The size of the partition.
762	///
763	/// Note that this can never be zero.
764	uint64_t size() const {
765	assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
766	return EndOffset - BeginOffset;
767	}
768
769	/// Test whether this partition contains no slices, and merely spans
770	/// a region occupied by split slices.
771	bool empty() const { return SI == SJ; }
772
773	/// \name Iterate slices that start within the partition.
774	/// These may be splittable or unsplittable. They have a begin offset >= the
775	/// partition begin offset.
776	/// @{
777	// FIXME: We should probably define a "concat_iterator" helper and use that
778	// to stitch together pointee_iterators over the split tails and the
779	// contiguous iterators of the partition. That would give a much nicer
780	// interface here. We could then additionally expose filtered iterators for
781	// split, unsplit, and unsplittable splices based on the usage patterns.
782	iterator begin() const { return SI; }
783	iterator end() const { return SJ; }
784	/// @}
785
786	/// Get the sequence of split slice tails.
787	///
788	/// These tails are of slices which start before this partition but are
789	/// split and overlap into the partition. We accumulate these while forming
790	/// partitions.
791	ArrayRef<Slice > splitSliceTails() const* { return SplitTails; }
792	};
793
794	} // end anonymous namespace
795
796	/// An iterator over partitions of the alloca's slices.
797	///
798	/// This iterator implements the core algorithm for partitioning the alloca's
799	/// slices. It is a forward iterator as we don't support backtracking for
800	/// efficiency reasons, and re-use a single storage area to maintain the
801	/// current set of split slices.
802	///
803	/// It is templated on the slice iterator type to use so that it can operate
804	/// with either const or non-const slice iterators.
805	class AllocaSlices::partition_iterator
806	: public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
807	Partition> {
808	friend class AllocaSlices;
809
810	/// Most of the state for walking the partitions is held in a class
811	/// with a nice interface for examining them.
812	Partition P;
813
814	/// We need to keep the end of the slices to know when to stop.
815	AllocaSlices::iterator SE;
816
817	/// We also need to keep track of the maximum split end offset seen.
818	/// FIXME: Do we really?
819	uint64_t MaxSplitSliceEndOffset = `0`;
820
821	/// Sets the partition to be empty at given iterator, and sets the
822	/// end iterator.
823	partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
824	: P (SI), SE(SE) {
825	// If not already at the end, advance our state to form the initial
826	// partition.
827	if (SI != SE)
828	advance();
829	}
830
831	/// Advance the iterator to the next partition.
832	///
833	/// Requires that the iterator not be at the end of the slices.
834	void advance() {
835	assert((P.SI != SE \|\| !P.SplitTails.empty()) &&
836	"Cannot advance past the end of the slices!");
837
838	// Clear out any split uses which have ended.
839	if (!P.SplitTails.empty()) {
840	if (P.EndOffset >= MaxSplitSliceEndOffset) {
841	// If we've finished all splits, this is easy.
842	P.SplitTails.clear();
843	MaxSplitSliceEndOffset = `0`;
844	} else {
845	// Remove the uses which have ended in the prior partition. This
846	// cannot change the max split slice end because we just checked that
847	// the prior partition ended prior to that max.
848	llvm::erase_if(C&: P.SplitTails,
849	P: [&](Slice S) { return* S->endOffset() <= P.EndOffset; });
850	assert(llvm::any_of(P.SplitTails,
851	[&](Slice *S) {
852	return S->endOffset() == MaxSplitSliceEndOffset;
853	}) &&
854	"Could not find the current max split slice offset!");
855	assert(llvm::all_of(P.SplitTails,
856	[&](Slice *S) {
857	return S->endOffset() <= MaxSplitSliceEndOffset;
858	}) &&
859	"Max split slice end offset is not actually the max!");
860	}
861	}
862
863	// If P.SI is already at the end, then we've cleared the split tail and
864	// now have an end iterator.
865	if (P.SI == SE) {
866	assert(P.SplitTails.empty() && "Failed to clear the split slices!");
867	return;
868	}
869
870	// If we had a non-empty partition previously, set up the state for
871	// subsequent partitions.
872	if (P.SI != P.SJ) {
873	// Accumulate all the splittable slices which started in the old
874	// partition into the split list.
875	for (Slice &S : P)
876	if (S.isSplittable() && S.endOffset() > P.EndOffset) {
877	P.SplitTails.push_back(Elt: &S);
878	MaxSplitSliceEndOffset =
879	std::max(a: S.endOffset(), b: MaxSplitSliceEndOffset);
880	}
881
882	// Start from the end of the previous partition.
883	P.SI = P.SJ;
884
885	// If P.SI is now at the end, we at most have a tail of split slices.
886	if (P.SI == SE) {
887	P.BeginOffset = P.EndOffset;
888	P.EndOffset = MaxSplitSliceEndOffset;
889	return;
890	}
891
892	// If the we have split slices and the next slice is after a gap and is
893	// not splittable immediately form an empty partition for the split
894	// slices up until the next slice begins.
895	if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
896	!P.SI->isSplittable()) {
897	P.BeginOffset = P.EndOffset;
898	P.EndOffset = P.SI->beginOffset();
899	return;
900	}
901	}
902
903	// OK, we need to consume new slices. Set the end offset based on the
904	// current slice, and step SJ past it. The beginning offset of the
905	// partition is the beginning offset of the next slice unless we have
906	// pre-existing split slices that are continuing, in which case we begin
907	// at the prior end offset.
908	P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
909	P.EndOffset = P.SI->endOffset();
910	++P.SJ;
911
912	// There are two strategies to form a partition based on whether the
913	// partition starts with an unsplittable slice or a splittable slice.
914	if (!P.SI->isSplittable()) {
915	// When we're forming an unsplittable region, it must always start at
916	// the first slice and will extend through its end.
917	assert(P.BeginOffset == P.SI->beginOffset());
918
919	// Form a partition including all of the overlapping slices with this
920	// unsplittable slice.
921	while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
922	if (!P.SJ->isSplittable())
923	P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
924	++P.SJ;
925	}
926
927	// We have a partition across a set of overlapping unsplittable
928	// partitions.
929	return;
930	}
931
932	// If we're starting with a splittable slice, then we need to form
933	// a synthetic partition spanning it and any other overlapping splittable
934	// splices.
935	assert(P.SI->isSplittable() && "Forming a splittable partition!");
936
937	// Collect all of the overlapping splittable slices.
938	while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
939	P.SJ->isSplittable()) {
940	P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
941	++P.SJ;
942	}
943
944	// Back upiP.EndOffset if we ended the span early when encountering an
945	// unsplittable slice. This synthesizes the early end offset of
946	// a partition spanning only splittable slices.
947	if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
948	assert(!P.SJ->isSplittable());
949	P.EndOffset = P.SJ->beginOffset();
950	}
951	}
952
953	public:
954	bool operator==(const partition_iterator &RHS) const {
955	assert(SE == RHS.SE &&
956	"End iterators don't match between compared partition iterators!");
957
958	// The observed positions of partitions is marked by the P.SI iterator and
959	// the emptiness of the split slices. The latter is only relevant when
960	// P.SI == SE, as the end iterator will additionally have an empty split
961	// slices list, but the prior may have the same P.SI and a tail of split
962	// slices.
963	if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
964	assert(P.SJ == RHS.P.SJ &&
965	"Same set of slices formed two different sized partitions!");
966	assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
967	"Same slice position with differently sized non-empty split "
968	"slice tails!");
969	return true;
970	}
971	return false;
972	}
973
974	partition_iterator &operator++() {
975	advance();
976	return *this;
977	}
978
979	Partition &operator() { return* P; }
980	};
981
982	/// A forward range over the partitions of the alloca's slices.
983	///
984	/// This accesses an iterator range over the partitions of the alloca's
985	/// slices. It computes these partitions on the fly based on the overlapping
986	/// offsets of the slices and the ability to split them. It will visit "empty"
987	/// partitions to cover regions of the alloca only accessed via split
988	/// slices.
989	iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
990	return make_range(x: partition_iterator (begin(), end()),
991	y: partition_iterator (end(), end()));
992	}
993
994	static Value *foldSelectInst(SelectInst &SI) {
995	// If the condition being selected on is a constant or the same value is
996	// being selected between, fold the select. Yes this does (rarely) happen
997	// early on.
998	if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: SI.getCondition()))
999	return SI.getOperand(i_nocapture: `1` + CI->isZero());
1000	if (SI.getOperand(i_nocapture: `1`) == SI.getOperand(i_nocapture: `2`))
1001	return SI.getOperand(i_nocapture: `1`);
1002
1003	return nullptr;
1004	}
1005
1006	/// A helper that folds a PHI node or a select.
1007	static Value *foldPHINodeOrSelectInst(Instruction &I) {
1008	if (PHINode *PN = dyn_cast<PHINode>(Val: &I)) {
1009	// If PN merges together the same value, return that value.
1010	return PN->hasConstantValue();
1011	}
1012	return foldSelectInst(SI&: cast<SelectInst>(Val&: I));
1013	}
1014
1015	/// Builder for the alloca slices.
1016	///
1017	/// This class builds a set of alloca slices by recursively visiting the uses
1018	/// of an alloca and making a slice for each load and store at each offset.
1019	class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1020	friend class PtrUseVisitor<SliceBuilder>;
1021	friend class InstVisitor<SliceBuilder>;
1022
1023	using Base = PtrUseVisitor<SliceBuilder>;
1024
1025	const uint64_t AllocSize;
1026	AllocaSlices &AS;
1027
1028	SmallDenseMap<Instruction , unsigned*> MemTransferSliceMap;
1029	SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
1030
1031	/// Set to de-duplicate dead instructions found in the use walk.
1032	SmallPtrSet<Instruction *, `4`> VisitedDeadInsts;
1033
1034	public:
1035	SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1036	: PtrUseVisitor<SliceBuilder>(DL),
1037	AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1038
1039	private:
1040	void markAsDead(Instruction &I) {
1041	if (VisitedDeadInsts.insert(Ptr: &I).second)
1042	AS.DeadUsers.push_back(Elt: &I);
1043	}
1044
1045	void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1046	bool IsSplittable = false) {
1047	// Completely skip uses which have a zero size or start either before or
1048	// past the end of the allocation.
1049	if (Size == `0` \|\| Offset.uge(RHS: AllocSize)) {
1050	LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1051	<< Offset
1052	<< " which has zero size or starts outside of the "
1053	<< AllocSize << " byte alloca:\n"
1054	<< " alloca: " << AS.AI << "\n"
1055	<< " use: " << I << "\n");
1056	return markAsDead(I);
1057	}
1058
1059	uint64_t BeginOffset = Offset.getZExtValue();
1060	uint64_t EndOffset = BeginOffset + Size;
1061
1062	// Clamp the end offset to the end of the allocation. Note that this is
1063	// formulated to handle even the case where "BeginOffset + Size" overflows.
1064	// This may appear superficially to be something we could ignore entirely,
1065	// but that is not so! There may be widened loads or PHI-node uses where
1066	// some instructions are dead but not others. We can't completely ignore
1067	// them, and so have to record at least the information here.
1068	assert(AllocSize >= BeginOffset); // Established above.
1069	if (Size > AllocSize - BeginOffset) {
1070	LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1071	<< Offset << " to remain within the " << AllocSize
1072	<< " byte alloca:\n"
1073	<< " alloca: " << AS.AI << "\n"
1074	<< " use: " << I << "\n");
1075	EndOffset = AllocSize;
1076	}
1077
1078	AS.Slices.push_back(Elt: Slice (BeginOffset, EndOffset, U, IsSplittable));
1079	}
1080
1081	void visitBitCastInst(BitCastInst &BC) {
1082	if (BC.use_empty())
1083	return markAsDead(I&: BC);
1084
1085	return Base::visitBitCastInst(BC);
1086	}
1087
1088	void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1089	if (ASC.use_empty())
1090	return markAsDead(I&: ASC);
1091
1092	return Base::visitAddrSpaceCastInst(ASC);
1093	}
1094
1095	void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1096	if (GEPI.use_empty())
1097	return markAsDead(I&: GEPI);
1098
1099	return Base::visitGetElementPtrInst(GEPI);
1100	}
1101
1102	void handleLoadOrStore(Type Ty, Instruction &I, const* APInt &Offset,
1103	uint64_t Size, bool IsVolatile) {
1104	// We allow splitting of non-volatile loads and stores where the type is an
1105	// integer type. These may be used to implement 'memcpy' or other "transfer
1106	// of bits" patterns.
1107	bool IsSplittable =
1108	Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1109
1110	insertUse(I, Offset, Size, IsSplittable);
1111	}
1112
1113	void visitLoadInst(LoadInst &LI) {
1114	assert((!LI.isSimple() \|\| LI.getType()->isSingleValueType()) &&
1115	"All simple FCA loads should have been pre-split");
1116
1117	// If there is a load with an unknown offset, we can still perform store
1118	// to load forwarding for other known-offset loads.
1119	if (!IsOffsetKnown)
1120	return PI.setEscapedReadOnly(&LI);
1121
1122	TypeSize Size = DL.getTypeStoreSize(Ty: LI.getType());
1123	if (Size.isScalable()) {
1124	unsigned VScale = LI.getFunction()->getVScaleValue();
1125	if (!VScale)
1126	return PI.setAborted(&LI);
1127
1128	Size = TypeSize::getFixed(ExactSize: Size.getKnownMinValue() * VScale);
1129	}
1130
1131	return handleLoadOrStore(Ty: LI.getType(), I&: LI, Offset, Size: Size.getFixedValue(),
1132	IsVolatile: LI.isVolatile());
1133	}
1134
1135	void visitStoreInst(StoreInst &SI) {
1136	Value *ValOp = SI.getValueOperand();
1137	if (ValOp == *U)
1138	return PI.setEscapedAndAborted(&SI);
1139	if (!IsOffsetKnown)
1140	return PI.setAborted(&SI);
1141
1142	TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValOp->getType());
1143	if (StoreSize.isScalable()) {
1144	unsigned VScale = SI.getFunction()->getVScaleValue();
1145	if (!VScale)
1146	return PI.setAborted(&SI);
1147
1148	StoreSize = TypeSize::getFixed(ExactSize: StoreSize.getKnownMinValue() * VScale);
1149	}
1150
1151	uint64_t Size = StoreSize.getFixedValue();
1152
1153	// If this memory access can be shown to statically* extend outside the*
1154	// bounds of the allocation, it's behavior is undefined, so simply
1155	// ignore it. Note that this is more strict than the generic clamping
1156	// behavior of insertUse. We also try to handle cases which might run the
1157	// risk of overflow.
1158	// FIXME: We should instead consider the pointer to have escaped if this
1159	// function is being instrumented for addressing bugs or race conditions.
1160	if (Size > AllocSize \|\| Offset.ugt(RHS: AllocSize - Size)) {
1161	LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1162	<< Offset << " which extends past the end of the "
1163	<< AllocSize << " byte alloca:\n"
1164	<< " alloca: " << AS.AI << "\n"
1165	<< " use: " << SI << "\n");
1166	return markAsDead(I&: SI);
1167	}
1168
1169	assert((!SI.isSimple() \|\| ValOp->getType()->isSingleValueType()) &&
1170	"All simple FCA stores should have been pre-split");
1171	handleLoadOrStore(Ty: ValOp->getType(), I&: SI, Offset, Size, IsVolatile: SI.isVolatile());
1172	}
1173
1174	void visitMemSetInst(MemSetInst &II) {
1175	assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1176	ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1177	if ((Length && Length->getValue() == `0`) \|\|
1178	(IsOffsetKnown && Offset.uge(RHS: AllocSize)))
1179	// Zero-length mem transfer intrinsics can be ignored entirely.
1180	return markAsDead(I&: II);
1181
1182	if (!IsOffsetKnown)
1183	return PI.setAborted(&II);
1184
1185	insertUse(I&: II, Offset,
1186	Size: Length ? Length->getLimitedValue()
1187	: AllocSize - Offset.getLimitedValue(),
1188	IsSplittable: (bool)Length);
1189	}
1190
1191	void visitMemTransferInst(MemTransferInst &II) {
1192	ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1193	if (Length && Length->getValue() == `0`)
1194	// Zero-length mem transfer intrinsics can be ignored entirely.
1195	return markAsDead(I&: II);
1196
1197	// Because we can visit these intrinsics twice, also check to see if the
1198	// first time marked this instruction as dead. If so, skip it.
1199	if (VisitedDeadInsts.count(Ptr: &II))
1200	return;
1201
1202	if (!IsOffsetKnown)
1203	return PI.setAborted(&II);
1204
1205	// This side of the transfer is completely out-of-bounds, and so we can
1206	// nuke the entire transfer. However, we also need to nuke the other side
1207	// if already added to our partitions.
1208	// FIXME: Yet another place we really should bypass this when
1209	// instrumenting for ASan.
1210	if (Offset.uge(RHS: AllocSize)) {
1211	auto MTPI = MemTransferSliceMap.find(Val: &II);
1212	if (MTPI != MemTransferSliceMap.end())
1213	AS.Slices [MTPI ->second].kill();
1214	return markAsDead(I&: II);
1215	}
1216
1217	uint64_t RawOffset = Offset.getLimitedValue();
1218	uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1219
1220	// Check for the special case where the same exact value is used for both
1221	// source and dest.
1222	if (U == II.getRawDest() && U == II.getRawSource()) {
1223	// For non-volatile transfers this is a no-op.
1224	if (!II.isVolatile())
1225	return markAsDead(I&: II);
1226
1227	return insertUse(I&: II, Offset, Size, /IsSplittable=/false);
1228	}
1229
1230	// If we have seen both source and destination for a mem transfer, then
1231	// they both point to the same alloca.
1232	bool Inserted;
1233	SmallDenseMap<Instruction , unsigned*>::iterator MTPI;
1234	std::tie(args&: MTPI, args&: Inserted) =
1235	MemTransferSliceMap.insert(KV: std::make_pair(x: &II, y: AS.Slices.size()));
1236	unsigned PrevIdx = MTPI ->second;
1237	if (!Inserted) {
1238	Slice &PrevP = AS.Slices [PrevIdx];
1239
1240	// Check if the begin offsets match and this is a non-volatile transfer.
1241	// In that case, we can completely elide the transfer.
1242	if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1243	PrevP.kill();
1244	return markAsDead(I&: II);
1245	}
1246
1247	// Otherwise we have an offset transfer within the same alloca. We can't
1248	// split those.
1249	PrevP.makeUnsplittable();
1250	}
1251
1252	// Insert the use now that we've fixed up the splittable nature.
1253	insertUse(I&: II, Offset, Size, /IsSplittable=/Inserted && Length);
1254
1255	// Check that we ended up with a valid index in the map.
1256	assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1257	"Map index doesn't point back to a slice with this user.");
1258	}
1259
1260	// Disable SRoA for any intrinsics except for lifetime invariants.
1261	// FIXME: What about debug intrinsics? This matches old behavior, but
1262	// doesn't make sense.
1263	void visitIntrinsicInst(IntrinsicInst &II) {
1264	if (II.isDroppable()) {
1265	AS.DeadUseIfPromotable.push_back(Elt: U);
1266	return;
1267	}
1268
1269	if (!IsOffsetKnown)
1270	return PI.setAborted(&II);
1271
1272	if (II.isLifetimeStartOrEnd()) {
1273	insertUse(I&: II, Offset, Size: AllocSize, IsSplittable: true);
1274	return;
1275	}
1276
1277	Base::visitIntrinsicInst(II);
1278	}
1279
1280	Instruction hasUnsafePHIOrSelectUse(Instruction Root, uint64_t &Size) {
1281	// We consider any PHI or select that results in a direct load or store of
1282	// the same offset to be a viable use for slicing purposes. These uses
1283	// are considered unsplittable and the size is the maximum loaded or stored
1284	// size.
1285	SmallPtrSet<Instruction *, `4`> Visited;
1286	SmallVector<std::pair<Instruction , Instruction >, `4`> Uses;
1287	Visited.insert(Ptr: Root);
1288	Uses.push_back(Elt: std::make_pair(x: cast<Instruction>(Val&: *U), y&: Root));
1289	const DataLayout &DL = Root->getDataLayout();
1290	// If there are no loads or stores, the access is dead. We mark that as
1291	// a size zero access.
1292	Size = `0`;
1293	do {
1294	Instruction I, UsedI;
1295	std::tie(args&: UsedI, args&: I) = Uses.pop_back_val();
1296
1297	if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1298	TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
1299	if (LoadSize.isScalable()) {
1300	PI.setAborted(LI);
1301	return nullptr;
1302	}
1303	Size = std::max(a: Size, b: LoadSize.getFixedValue());
1304	continue;
1305	}
1306	if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1307	Value *Op = SI->getOperand(i_nocapture: `0`);
1308	if (Op == UsedI)
1309	return SI;
1310	TypeSize StoreSize = DL.getTypeStoreSize(Ty: Op->getType());
1311	if (StoreSize.isScalable()) {
1312	PI.setAborted(SI);
1313	return nullptr;
1314	}
1315	Size = std::max(a: Size, b: StoreSize.getFixedValue());
1316	continue;
1317	}
1318
1319	if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1320	if (!GEP->hasAllZeroIndices())
1321	return GEP;
1322	} else if (!isa<BitCastInst>(Val: I) && !isa<PHINode>(Val: I) &&
1323	!isa<SelectInst>(Val: I) && !isa<AddrSpaceCastInst>(Val: I)) {
1324	return I;
1325	}
1326
1327	for (User *U : I->users())
1328	if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
1329	Uses.push_back(Elt: std::make_pair(x&: I, y: cast<Instruction>(Val: U)));
1330	} while (!Uses.empty());
1331
1332	return nullptr;
1333	}
1334
1335	void visitPHINodeOrSelectInst(Instruction &I) {
1336	assert(isa<PHINode>(I) \|\| isa<SelectInst>(I));
1337	if (I.use_empty())
1338	return markAsDead(I);
1339
1340	// If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1341	// instructions in this BB, which may be required during rewriting. Bail out
1342	// on these cases.
1343	if (isa<PHINode>(Val: I) && !I.getParent()->hasInsertionPt())
1344	return PI.setAborted(&I);
1345
1346	// TODO: We could use simplifyInstruction here to fold PHINodes and
1347	// SelectInsts. However, doing so requires to change the current
1348	// dead-operand-tracking mechanism. For instance, suppose neither loading
1349	// from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1350	// trap either. However, if we simply replace %U with undef using the
1351	// current dead-operand-tracking mechanism, "load (select undef, undef,
1352	// %other)" may trap because the select may return the first operand
1353	// "undef".
1354	if (Value *Result = foldPHINodeOrSelectInst(I)) {
1355	if (Result == *U)
1356	// If the result of the constant fold will be the pointer, recurse
1357	// through the PHI/select as if we had RAUW'ed it.
1358	enqueueUsers(I);
1359	else
1360	// Otherwise the operand to the PHI/select is dead, and we can replace
1361	// it with poison.
1362	AS.DeadOperands.push_back(Elt: U);
1363
1364	return;
1365	}
1366
1367	if (!IsOffsetKnown)
1368	return PI.setAborted(&I);
1369
1370	// See if we already have computed info on this node.
1371	uint64_t &Size = PHIOrSelectSizes [&I];
1372	if (!Size) {
1373	// This is a new PHI/Select, check for an unsafe use of it.
1374	if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(Root: &I, Size))
1375	return PI.setAborted(UnsafeI);
1376	}
1377
1378	// For PHI and select operands outside the alloca, we can't nuke the entire
1379	// phi or select -- the other side might still be relevant, so we special
1380	// case them here and use a separate structure to track the operands
1381	// themselves which should be replaced with poison.
1382	// FIXME: This should instead be escaped in the event we're instrumenting
1383	// for address sanitization.
1384	if (Offset.uge(RHS: AllocSize)) {
1385	AS.DeadOperands.push_back(Elt: U);
1386	return;
1387	}
1388
1389	insertUse(I, Offset, Size);
1390	}
1391
1392	void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(I&: PN); }
1393
1394	void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(I&: SI); }
1395
1396	/// Disable SROA entirely if there are unhandled users of the alloca.
1397	void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1398
1399	void visitCallBase(CallBase &CB) {
1400	// If the call operand is read-only and only does a read-only or address
1401	// capture, then we mark it as EscapedReadOnly.
1402	if (CB.isDataOperand(U) &&
1403	!capturesFullProvenance(CC: CB.getCaptureInfo(OpNo: U->getOperandNo())) &&
1404	CB.onlyReadsMemory(OpNo: U->getOperandNo())) {
1405	PI.setEscapedReadOnly(&CB);
1406	return;
1407	}
1408
1409	Base::visitCallBase(CB);
1410	}
1411	};
1412
1413	AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1414	:
1415	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1416	AI(AI),
1417	#endif
1418	PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1419	SliceBuilder PB(DL, AI, *this);
1420	SliceBuilder::PtrInfo PtrI = PB.visitPtr(I&: AI);
1421	if (PtrI.isEscaped() \|\| PtrI.isAborted()) {
1422	// FIXME: We should sink the escape vs. abort info into the caller nicely,
1423	// possibly by just storing the PtrInfo in the AllocaSlices.
1424	PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1425	: PtrI.getAbortingInst();
1426	assert(PointerEscapingInstr && "Did not track a bad instruction");
1427	return;
1428	}
1429	PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1430
1431	llvm::erase_if(C&: Slices, P: [](const Slice &S) { return S.isDead(); });
1432
1433	// Sort the uses. This arranges for the offsets to be in ascending order,
1434	// and the sizes to be in descending order.
1435	llvm::stable_sort(Range&: Slices);
1436	}
1437
1438	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1439
1440	void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1441	StringRef Indent) const {
1442	printSlice(OS, I, Indent);
1443	OS << "\n";
1444	printUse(OS, I, Indent);
1445	}
1446
1447	void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1448	StringRef Indent) const {
1449	OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1450	<< " slice #" << (I - begin())
1451	<< (I->isSplittable() ? " (splittable)" : "");
1452	}
1453
1454	void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1455	StringRef Indent) const {
1456	OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1457	}
1458
1459	void AllocaSlices::print(raw_ostream &OS) const {
1460	if (PointerEscapingInstr) {
1461	OS << "Can't analyze slices for alloca: " << AI << "\n"
1462	<< " A pointer to this alloca escaped by:\n"
1463	<< " " << *PointerEscapingInstr << "\n";
1464	return;
1465	}
1466
1467	if (PointerEscapingInstrReadOnly)
1468	OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1469
1470	OS << "Slices of alloca: " << AI << "\n";
1471	for (const_iterator I = begin(), E = end(); I != E; ++I)
1472	print(OS, I);
1473	}
1474
1475	LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1476	print(dbgs(), I);
1477	}
1478	LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1479
1480	#endif // !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1481
1482	/// Find a common load/store type used through a pointer PHI or select.
1483	///
1484	/// Look through a PHI or select to see if all of its users are loads or stores
1485	/// of one common type. Whether those accesses can be speculated does not affect
1486	/// the type they use and is checked separately when attempting promotion.
1487	static Type *findCommonTypeThroughPHIOrSelect(Instruction &I) {
1488	assert((isa<PHINode, SelectInst>(I)) && "expected a PHI or select");
1489	Type Ty = nullptr*;
1490
1491	for (User *U : I.users()) {
1492	Type UserTy = nullptr*;
1493	if (auto *LI = dyn_cast<LoadInst>(Val: U))
1494	UserTy = LI->getType();
1495	else if (auto *Store = dyn_cast<StoreInst>(Val: U))
1496	// Slice building rejects stores of the PHI-or-select-derived pointer, so
1497	// it must be the store's pointer operand here.
1498	UserTy = Store->getValueOperand()->getType();
1499
1500	if (!UserTy \|\| (Ty && Ty != UserTy))
1501	return nullptr;
1502	Ty = UserTy;
1503	}
1504
1505	return Ty;
1506	}
1507
1508	/// Walk the range of a partitioning looking for a common type to cover this
1509	/// sequence of slices.
1510	static std::pair<Type , IntegerType >
1511	findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1512	uint64_t EndOffset) {
1513	Type Ty = nullptr*;
1514	bool TyIsCommon = true;
1515	IntegerType ITy = nullptr*;
1516
1517	// Note that we need to look at every* alloca slice's Use to ensure we*
1518	// always get consistent results regardless of the order of slices.
1519	for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1520	Use *U = I->getUse();
1521	if (isa<IntrinsicInst>(Val: *U->getUser()))
1522	continue;
1523	if (I->beginOffset() != B->beginOffset() \|\| I->endOffset() != EndOffset)
1524	continue;
1525
1526	Type UserTy = nullptr*;
1527	if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
1528	UserTy = LI->getType();
1529	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
1530	UserTy = SI->getValueOperand()->getType();
1531	} else if (isa<PHINode, SelectInst>(Val: U->getUser())) {
1532	UserTy =
1533	findCommonTypeThroughPHIOrSelect(I&: *cast<Instruction>(Val: U->getUser()));
1534	}
1535
1536	if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(Val: UserTy)) {
1537	// If the type is larger than the partition, skip it. We only encounter
1538	// this for split integer operations where we want to use the type of the
1539	// entity causing the split. Also skip if the type is not a byte width
1540	// multiple.
1541	if (UserITy->getBitWidth() % `8` != `0` \|\|
1542	UserITy->getBitWidth() / `8` > (EndOffset - B->beginOffset()))
1543	continue;
1544
1545	// Track the largest bitwidth integer type used in this way in case there
1546	// is no common type.
1547	if (!ITy \|\| ITy->getBitWidth() < UserITy->getBitWidth())
1548	ITy = UserITy;
1549	}
1550
1551	// To avoid depending on the order of slices, Ty and TyIsCommon must not
1552	// depend on types skipped above.
1553	if (!UserTy \|\| (Ty && Ty != UserTy))
1554	TyIsCommon = false; // Give up on anything but an iN type.
1555	else
1556	Ty = UserTy;
1557	}
1558
1559	return {TyIsCommon ? Ty : nullptr, ITy};
1560	}
1561
1562	/// PHI instructions that use an alloca and are subsequently loaded can be
1563	/// rewritten to load both input pointers in the pred blocks and then PHI the
1564	/// results, allowing the load of the alloca to be promoted.
1565	/// From this:
1566	/// %P2 = phi [i32 %Alloca, i32* %Other]*
1567	/// %V = load i32 %P2*
1568	/// to:
1569	/// %V1 = load i32 %Alloca -> will be mem2reg'd*
1570	/// ...
1571	/// %V2 = load i32 %Other*
1572	/// ...
1573	/// %V = phi [i32 %V1, i32 %V2]
1574	///
1575	/// We can do this to a PHI if its only uses are loads and if any loads moved
1576	/// across other outgoing edges can be executed unconditionally.
1577	///
1578	/// FIXME: This should be hoisted into a generic utility, likely in
1579	/// Transforms/Util/Local.h
1580	static bool isSafePHIToSpeculate(PHINode &PN) {
1581	const DataLayout &DL = PN.getDataLayout();
1582
1583	// For now, we can only do this promotion if the load is in the same block
1584	// as the PHI, and if there are no stores between the phi and load.
1585	// TODO: Allow recursive phi users.
1586	BasicBlock *BB = PN.getParent();
1587	Align MaxAlign;
1588	uint64_t APWidth = DL.getIndexTypeSizeInBits(Ty: PN.getType());
1589	Type LoadType = nullptr*;
1590	for (User *U : PN.users()) {
1591	LoadInst *LI = dyn_cast<LoadInst>(Val: U);
1592	if (!LI \|\| !LI->isSimple())
1593	return false;
1594
1595	// For now we only allow loads in the same block as the PHI. This is
1596	// a common case that happens when instcombine merges two loads through
1597	// a PHI.
1598	if (LI->getParent() != BB)
1599	return false;
1600
1601	if (LoadType) {
1602	if (LoadType != LI->getType())
1603	return false;
1604	} else {
1605	LoadType = LI->getType();
1606	}
1607
1608	// Ensure that there are no instructions between the PHI and the load that
1609	// could store.
1610	for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1611	if (BBI ->mayWriteToMemory())
1612	return false;
1613
1614	MaxAlign = std::max(a: MaxAlign, b: LI->getAlign());
1615	}
1616
1617	if (!LoadType)
1618	return false;
1619
1620	APInt LoadSize =
1621	APInt (APWidth, DL.getTypeStoreSize(Ty: LoadType).getFixedValue());
1622
1623	// We can only transform this if it is safe to push the loads into the
1624	// predecessor blocks. The only thing to watch out for is that we can't put
1625	// a possibly trapping load in the predecessor if it is a critical edge.
1626	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1627	Instruction *TI = PN.getIncomingBlock(i: Idx)->getTerminator();
1628	Value *InVal = PN.getIncomingValue(i: Idx);
1629
1630	// If the value is produced by the terminator of the predecessor (an
1631	// invoke) or it has side-effects, there is no valid place to put a load
1632	// in the predecessor.
1633	if (TI == InVal \|\| TI->mayHaveSideEffects())
1634	return false;
1635
1636	// If the predecessor has a single successor, then the edge isn't
1637	// critical.
1638	if (TI->getNumSuccessors() == `1`)
1639	continue;
1640
1641	// If this pointer is always safe to load, or if we can prove that there
1642	// is already a load in the block, then we can move the load to the pred
1643	// block.
1644	if (isSafeToLoadUnconditionally(V: InVal, Alignment: MaxAlign, Size: LoadSize,
1645	SQ: SimplifyQuery (DL, TI)))
1646	continue;
1647
1648	return false;
1649	}
1650
1651	return true;
1652	}
1653
1654	/// Check whether a single store through PN can be moved onto each incoming
1655	/// edge.
1656	static StoreInst getPHIStoreToRewrite(PHINode &PN, bool* PreserveCFG,
1657	DominatorTree &DT) {
1658	// TODO: Support multiple stores and mixed load/store users.
1659	// TODO: Look through other instructions, such as other phis or addrspacecasts
1660	if (!PN.hasOneUse())
1661	return nullptr;
1662
1663	auto *SI = dyn_cast<StoreInst>(Val: PN.user_back());
1664	if (!SI \|\| SI->getPointerOperand() != &PN)
1665	return nullptr;
1666
1667	if (SI->isVolatile())
1668	return nullptr;
1669
1670	BasicBlock *BB = PN.getParent();
1671	// TODO: Allow a harmless prefix between the PHIs and the store.
1672	if (&*BB->getFirstNonPHIOrDbg() != SI)
1673	return nullptr;
1674
1675	Value *StoredValue = SI->getValueOperand();
1676	SmallPtrSet<BasicBlock *, `4`> SeenPreds;
1677	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1678	BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1679
1680	// Only one store is needed for duplicate edges from the same predecessor.
1681	if (!SeenPreds.insert(Ptr: Pred).second)
1682	continue;
1683
1684	// TODO: Support other terminators.
1685	Instruction *TI = Pred->getTerminator();
1686	if (!isa<CondBrInst, UncondBrInst>(Val: TI))
1687	return nullptr;
1688
1689	if (Pred == BB)
1690	return nullptr;
1691
1692	// TODO: If StoredValue is another PHI in BB, use its corresponding
1693	// incoming value instead of requiring it to dominate every predecessor.
1694	if (!DT.dominates(Def: StoredValue, User: TI))
1695	return nullptr;
1696
1697	if (TI->getNumSuccessors() == `1`)
1698	continue;
1699
1700	// If the predecessor has more than one successor, then we will need to
1701	// split it so that we can insert the store only on the path to this BB.
1702	if (PreserveCFG \|\| !BB->canSplitPredecessors())
1703	return nullptr;
1704	}
1705
1706	return SI;
1707	}
1708
1709	static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1710	LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1711
1712	LoadInst *SomeLoad = cast<LoadInst>(Val: PN.user_back());
1713	Type *LoadTy = SomeLoad->getType();
1714	IRB.SetInsertPoint(&PN);
1715	PHINode *NewPN = IRB.CreatePHI(Ty: LoadTy, NumReservedValues: PN.getNumIncomingValues(),
1716	Name: PN.getName() + ".sroa.speculated");
1717
1718	// Get the AA tags and alignment to use from one of the loads. It does not
1719	// matter which one we get and if any differ.
1720	AAMDNodes AATags = SomeLoad->getAAMetadata();
1721	Align Alignment = SomeLoad->getAlign();
1722
1723	// Rewrite all loads of the PN to use the new PHI.
1724	while (!PN.use_empty()) {
1725	LoadInst *LI = cast<LoadInst>(Val: PN.user_back());
1726	LI->replaceAllUsesWith(V: NewPN);
1727	LI->eraseFromParent();
1728	}
1729
1730	// Inject loads into all of the pred blocks.
1731	DenseMap<BasicBlock , Value > InjectedLoads;
1732	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1733	BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1734	Value *InVal = PN.getIncomingValue(i: Idx);
1735
1736	// A PHI node is allowed to have multiple (duplicated) entries for the same
1737	// basic block, as long as the value is the same. So if we already injected
1738	// a load in the predecessor, then we should reuse the same load for all
1739	// duplicated entries.
1740	if (Value *V = InjectedLoads.lookup(Val: Pred)) {
1741	NewPN->addIncoming(V, BB: Pred);
1742	continue;
1743	}
1744
1745	Instruction *TI = Pred->getTerminator();
1746	IRB.SetInsertPoint(TI);
1747
1748	LoadInst *Load = IRB.CreateAlignedLoad(
1749	Ty: LoadTy, Ptr: InVal, Align: Alignment,
1750	Name: (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1751	++NumLoadsSpeculated;
1752	if (AATags)
1753	Load->setAAMetadata(AATags);
1754	NewPN->addIncoming(V: Load, BB: Pred);
1755	InjectedLoads [Pred] = Load;
1756	}
1757
1758	LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1759	PN.eraseFromParent();
1760	}
1761
1762	/// Move a store through a pointer PHI onto each of the PHI's incoming edges.
1763	/// Returns whether this required modifying the CFG.
1764	static bool rewritePHINodeStore(PHINode &PN, StoreInst &SI, DomTreeUpdater &DTU,
1765	SmallSetVector<AllocaInst *, `16`> &Worklist) {
1766	LLVM_DEBUG(dbgs() << " original: " << PN << "\n"
1767	<< " " << SI << "\n");
1768
1769	// Splitting one edge rewrites all PHIs in the destination block. Snapshot
1770	// the original predecessor/value pairs before making any CFG changes.
1771	SmallVector<std::pair<BasicBlock , Value >, `4`> IncomingValues;
1772	SmallPtrSet<BasicBlock *, `4`> SeenPreds;
1773	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1774	BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1775	if (!SeenPreds.insert(Ptr: Pred).second)
1776	continue;
1777	Value *InVal = PN.getIncomingValue(i: Idx);
1778	IncomingValues.emplace_back(Args&: Pred, Args&: InVal);
1779
1780	// Revisit every alloca exposed by removing the pointer PHI.
1781	if (auto *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: InVal)))
1782	Worklist.insert(X: AI);
1783	}
1784
1785	bool CFGChanged = false;
1786	BasicBlock *BB = PN.getParent();
1787	for (auto [Pred, InVal] : IncomingValues) {
1788	BasicBlock *StoreBB = Pred;
1789	if (Pred->getTerminator()->getNumSuccessors() != `1`) {
1790	StoreBB = SplitBlockPredecessors(BB, Preds: {Pred}, Suffix: ".sroa.store", DTU: &DTU);
1791	assert(StoreBB && "store edge was not checked for splitting");
1792	CFGChanged = true;
1793	}
1794
1795	auto *NewStore = cast<StoreInst>(Val: SI.clone());
1796	NewStore->setOperand(i_nocapture: StoreInst::getPointerOperandIndex(), Val_nocapture: InVal);
1797	NewStore->insertBefore(InsertPos: StoreBB->getTerminator()->getIterator());
1798	++NumStoresPredicated;
1799	LLVM_DEBUG(dbgs() << " to: " << *NewStore << "\n");
1800	}
1801
1802	SI.eraseFromParent();
1803	PN.eraseFromParent();
1804	return CFGChanged;
1805	}
1806
1807	SelectHandSpeculativity &
1808	SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1809	if (isTrueVal)
1810	Bitfield::set<SelectHandSpeculativity::TrueVal>(Packed&: Storage, Value: true);
1811	else
1812	Bitfield::set<SelectHandSpeculativity::FalseVal>(Packed&: Storage, Value: true);
1813	return *this;
1814	}
1815
1816	bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1817	return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Packed: Storage)
1818	: Bitfield::get<SelectHandSpeculativity::FalseVal>(Packed: Storage);
1819	}
1820
1821	bool SelectHandSpeculativity::areAllSpeculatable() const {
1822	return isSpeculatable(/isTrueVal=/true) &&
1823	isSpeculatable(/isTrueVal=/false);
1824	}
1825
1826	bool SelectHandSpeculativity::areAnySpeculatable() const {
1827	return isSpeculatable(/isTrueVal=/true) \|\|
1828	isSpeculatable(/isTrueVal=/false);
1829	}
1830	bool SelectHandSpeculativity::areNoneSpeculatable() const {
1831	return !areAnySpeculatable();
1832	}
1833
1834	static SelectHandSpeculativity
1835	isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1836	assert(LI.isSimple() && "Only for simple loads");
1837	SelectHandSpeculativity Spec;
1838
1839	const DataLayout &DL = SI.getDataLayout();
1840	for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1841	if (isSafeToLoadUnconditionally(V: Value, Ty: LI.getType(), Alignment: LI.getAlign(),
1842	SQ: SimplifyQuery (DL, &LI)))
1843	Spec.setAsSpeculatable(/isTrueVal=/Value == SI.getTrueValue());
1844	else if (PreserveCFG)
1845	return Spec;
1846
1847	return Spec;
1848	}
1849
1850	std::optional<RewriteableMemOps>
1851	SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1852	RewriteableMemOps Ops;
1853
1854	for (User *U : SI.users()) {
1855	if (auto *Store = dyn_cast<StoreInst>(Val: U)) {
1856	// Note that atomic stores can be transformed; atomic semantics do not
1857	// have any meaning for a local alloca. Stores are not speculatable,
1858	// however, so if we can't turn it into a predicated store, we are done.
1859	if (Store->isVolatile() \|\| PreserveCFG)
1860	return {}; // Give up on this `select`.
1861	Ops.emplace_back(Args&: Store);
1862	continue;
1863	}
1864
1865	auto *LI = dyn_cast<LoadInst>(Val: U);
1866
1867	// Note that atomic loads can be transformed;
1868	// atomic semantics do not have any meaning for a local alloca.
1869	if (!LI \|\| LI->isVolatile())
1870	return {}; // Give up on this `select`.
1871
1872	PossiblySpeculatableLoad Load(LI);
1873	if (!LI->isSimple()) {
1874	// If the `load` is not simple, we can't speculatively execute it,
1875	// but we could handle this via a CFG modification. But can we?
1876	if (PreserveCFG)
1877	return {}; // Give up on this `select`.
1878	Ops.emplace_back(Args&: Load);
1879	continue;
1880	}
1881
1882	SelectHandSpeculativity Spec =
1883	isSafeLoadOfSelectToSpeculate(LI&: *LI, SI, PreserveCFG);
1884	if (PreserveCFG && !Spec.areAllSpeculatable())
1885	return {}; // Give up on this `select`.
1886
1887	Load.setInt(Spec);
1888	Ops.emplace_back(Args&: Load);
1889	}
1890
1891	return Ops;
1892	}
1893
1894	static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1895	IRBuilderTy &IRB) {
1896	LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1897
1898	Value *TV = SI.getTrueValue();
1899	Value *FV = SI.getFalseValue();
1900	// Replace the given load of the select with a select of two loads.
1901
1902	assert(LI.isSimple() && "We only speculate simple loads");
1903
1904	IRB.SetInsertPoint(&LI);
1905
1906	LoadInst *TL =
1907	IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: TV, Align: LI.getAlign(),
1908	Name: LI.getName() + ".sroa.speculate.load.true");
1909	LoadInst *FL =
1910	IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: FV, Align: LI.getAlign(),
1911	Name: LI.getName() + ".sroa.speculate.load.false");
1912	NumLoadsSpeculated += `2`;
1913
1914	// Transfer alignment and AA info if present.
1915	TL->setAlignment(LI.getAlign());
1916	FL->setAlignment(LI.getAlign());
1917
1918	AAMDNodes Tags = LI.getAAMetadata();
1919	if (Tags) {
1920	TL->setAAMetadata(Tags);
1921	FL->setAAMetadata(Tags);
1922	}
1923
1924	Value *V = IRB.CreateSelect(C: SI.getCondition(), True: TL, False: FL,
1925	Name: LI.getName() + ".sroa.speculated", MDFrom: &SI);
1926
1927	LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1928	LI.replaceAllUsesWith(V);
1929	}
1930
1931	template <typename T>
1932	static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1933	SelectHandSpeculativity Spec,
1934	DomTreeUpdater &DTU) {
1935	assert((isa<LoadInst>(I) \|\| isa<StoreInst>(I)) && "Only for load and store!");
1936	LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1937	BasicBlock *Head = I.getParent();
1938	Instruction ThenTerm = nullptr*;
1939	Instruction ElseTerm = nullptr*;
1940	if (Spec.areNoneSpeculatable())
1941	SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1942	SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU);
1943	else {
1944	SplitBlockAndInsertIfThen(SI.getCondition(), &I, /Unreachable=/false,
1945	SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU,
1946	/LI=/nullptr, /ThenBlock=/nullptr);
1947	if (Spec.isSpeculatable(/isTrueVal=/true))
1948	cast<CondBrInst>(Val: Head->getTerminator())->swapSuccessors();
1949	}
1950	auto *HeadBI = cast<CondBrInst>(Val: Head->getTerminator());
1951	Spec = {}; // Do not use `Spec` beyond this point.
1952	BasicBlock *Tail = I.getParent();
1953	Tail->setName(Head->getName() + ".cont");
1954	PHINode *PN;
1955	if (isa<LoadInst>(I))
1956	PN = PHINode::Create(Ty: I.getType(), NumReservedValues: `2`, NameStr: "", InsertBefore: I.getIterator());
1957	for (BasicBlock *SuccBB : successors(BB: Head)) {
1958	bool IsThen = SuccBB == HeadBI->getSuccessor(i: `0`);
1959	int SuccIdx = IsThen ? `0` : `1`;
1960	auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1961	auto &CondMemOp = cast<T>(*I.clone());
1962	if (NewMemOpBB != Head) {
1963	NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1964	if (isa<LoadInst>(I))
1965	++NumLoadsPredicated;
1966	else
1967	++NumStoresPredicated;
1968	} else {
1969	CondMemOp.dropUBImplyingAttrsAndMetadata();
1970	++NumLoadsSpeculated;
1971	}
1972	CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
1973	Value *Ptr = SI.getOperand(i_nocapture: `1` + SuccIdx);
1974	CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
1975	if (isa<LoadInst>(I)) {
1976	CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
1977	PN->addIncoming(V: &CondMemOp, BB: NewMemOpBB);
1978	} else
1979	LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
1980	}
1981	if (isa<LoadInst>(I)) {
1982	PN->takeName(V: &I);
1983	LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
1984	I.replaceAllUsesWith(PN);
1985	}
1986	}
1987
1988	static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I,
1989	SelectHandSpeculativity Spec,
1990	DomTreeUpdater &DTU) {
1991	if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1992	rewriteMemOpOfSelect(SI&: SelInst, I&: *LI, Spec, DTU);
1993	else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1994	rewriteMemOpOfSelect(SI&: SelInst, I&: *SI, Spec, DTU);
1995	else
1996	llvm_unreachable_internal(msg: "Only for load and store.");
1997	}
1998
1999	static bool rewriteSelectInstMemOps(SelectInst &SI,
2000	const RewriteableMemOps &Ops,
2001	IRBuilderTy &IRB, DomTreeUpdater *DTU) {
2002	bool CFGChanged = false;
2003	LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
2004
2005	for (const RewriteableMemOp &Op : Ops) {
2006	SelectHandSpeculativity Spec;
2007	Instruction *I;
2008	if (auto *const *US = std::get_if<UnspeculatableStore>(ptr: &Op)) {
2009	I = *US;
2010	} else {
2011	auto PSL = std::get<PossiblySpeculatableLoad>(v: Op);
2012	I = PSL.getPointer();
2013	Spec = PSL.getInt();
2014	}
2015	if (Spec.areAllSpeculatable()) {
2016	speculateSelectInstLoads(SI, LI&: cast<LoadInst>(Val&: *I), IRB);
2017	} else {
2018	assert(DTU && "Should not get here when not allowed to modify the CFG!");
2019	rewriteMemOpOfSelect(SelInst&: SI, I&: I, Spec, DTU&: DTU);
2020	CFGChanged = true;
2021	}
2022	I->eraseFromParent();
2023	}
2024
2025	for (User *U : make_early_inc_range(Range: SI.users()))
2026	cast<BitCastInst>(Val: U)->eraseFromParent();
2027	SI.eraseFromParent();
2028	return CFGChanged;
2029	}
2030
2031	/// Compute an adjusted pointer from Ptr by Offset bytes where the
2032	/// resulting pointer has PointerTy.
2033	static Value getAdjustedPtr(IRBuilderTy &IRB, const* DataLayout &DL, Value *Ptr,
2034	APInt Offset, Type *PointerTy,
2035	const Twine &NamePrefix) {
2036	if (Offset != `0`)
2037	Ptr = IRB.CreateInBoundsPtrAdd(Ptr, Offset: IRB.getInt(AI: Offset),
2038	Name: NamePrefix + "sroa_idx");
2039	return IRB.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: PointerTy,
2040	Name: NamePrefix + "sroa_cast");
2041	}
2042
2043	/// Compute the adjusted alignment for a load or store from an offset.
2044	static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
2045	return commonAlignment(A: getLoadStoreAlignment(I), Offset);
2046	}
2047
2048	/// Test whether we can convert a value from the old to the new type.
2049	///
2050	/// This predicate should be used to guard calls to convertValue in order to
2051	/// ensure that we only try to convert viable values. The strategy is that we
2052	/// will peel off single element struct and array wrappings to get to an
2053	/// underlying value, and convert that value.
2054	static bool canConvertValue(const DataLayout &DL, Type OldTy, Type NewTy,
2055	unsigned VScale = `0`) {
2056	if (OldTy == NewTy)
2057	return true;
2058
2059	// For integer types, we can't handle any bit-width differences. This would
2060	// break both vector conversions with extension and introduce endianness
2061	// issues when in conjunction with loads and stores.
2062	if (isa<IntegerType>(Val: OldTy) && isa<IntegerType>(Val: NewTy)) {
2063	assert(cast<IntegerType>(OldTy)->getBitWidth() !=
2064	cast<IntegerType>(NewTy)->getBitWidth() &&
2065	"We can't have the same bitwidth for different int types");
2066	return false;
2067	}
2068
2069	TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
2070	TypeSize OldSize = DL.getTypeSizeInBits(Ty: OldTy);
2071
2072	if ((isa<ScalableVectorType>(Val: NewTy) && isa<FixedVectorType>(Val: OldTy)) \|\|
2073	(isa<ScalableVectorType>(Val: OldTy) && isa<FixedVectorType>(Val: NewTy))) {
2074	// Conversion is only possible when the size of scalable vectors is known.
2075	if (!VScale)
2076	return false;
2077
2078	// For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
2079	// a single domain (either fixed or scalable). Any additional conversion
2080	// between fixed and scalable types is handled through integer types.
2081	auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
2082	auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
2083
2084	if (isa<ScalableVectorType>(Val: NewTy)) {
2085	if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: NewVTy), EltTy: OldVTy))
2086	return false;
2087
2088	NewSize = TypeSize::getFixed(ExactSize: NewSize.getKnownMinValue() * VScale);
2089	} else {
2090	if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: OldVTy), EltTy: NewVTy))
2091	return false;
2092
2093	OldSize = TypeSize::getFixed(ExactSize: OldSize.getKnownMinValue() * VScale);
2094	}
2095	}
2096
2097	if (NewSize != OldSize)
2098	return false;
2099	if (!NewTy->isSingleValueType() \|\| !OldTy->isSingleValueType())
2100	return false;
2101
2102	// We can convert pointers to integers and vice-versa. Same for vectors
2103	// of pointers and integers.
2104	OldTy = OldTy->getScalarType();
2105	NewTy = NewTy->getScalarType();
2106	if (NewTy->isPointerTy() \|\| OldTy->isPointerTy()) {
2107	if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
2108	unsigned OldAS = OldTy->getPointerAddressSpace();
2109	unsigned NewAS = NewTy->getPointerAddressSpace();
2110	// Convert pointers if they are pointers from the same address space or
2111	// different integral (not non-integral) address spaces with the same
2112	// pointer size.
2113	return OldAS == NewAS \|\|
2114	(!DL.isNonIntegralAddressSpace(AddrSpace: OldAS) &&
2115	!DL.isNonIntegralAddressSpace(AddrSpace: NewAS) &&
2116	DL.getPointerSize(AS: OldAS) == DL.getPointerSize(AS: NewAS));
2117	}
2118
2119	// We can convert integers to integral pointers, but not to non-integral
2120	// pointers.
2121	if (OldTy->isIntegerTy())
2122	return !DL.isNonIntegralPointerType(Ty: NewTy);
2123
2124	// We can convert integral pointers to integers, but non-integral pointers
2125	// need to remain pointers.
2126	if (!DL.isNonIntegralPointerType(Ty: OldTy))
2127	return NewTy->isIntegerTy();
2128
2129	return false;
2130	}
2131
2132	if (OldTy->isTargetExtTy() \|\| NewTy->isTargetExtTy())
2133	return false;
2134
2135	return true;
2136	}
2137
2138	/// Test whether the given slice use can be promoted to a vector.
2139	///
2140	/// This function is called to test each entry in a partition which is slated
2141	/// for a single slice.
2142	static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2143	VectorType *Ty,
2144	uint64_t ElementSize,
2145	const DataLayout &DL,
2146	unsigned VScale) {
2147	// First validate the slice offsets.
2148	uint64_t BeginOffset =
2149	std::max(a: S.beginOffset(), b: P.beginOffset()) - P.beginOffset();
2150	uint64_t BeginIndex = BeginOffset / ElementSize;
2151	if (BeginIndex * ElementSize != BeginOffset \|\|
2152	BeginIndex >= cast<FixedVectorType>(Val: Ty)->getNumElements())
2153	return false;
2154	uint64_t EndOffset = std::min(a: S.endOffset(), b: P.endOffset()) - P.beginOffset();
2155	uint64_t EndIndex = EndOffset / ElementSize;
2156	if (EndIndex * ElementSize != EndOffset \|\|
2157	EndIndex > cast<FixedVectorType>(Val: Ty)->getNumElements())
2158	return false;
2159
2160	assert(EndIndex > BeginIndex && "Empty vector!");
2161	uint64_t NumElements = EndIndex - BeginIndex;
2162	Type *SliceTy = (NumElements == `1`)
2163	? Ty->getElementType()
2164	: FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElements);
2165
2166	Type *SplitIntTy =
2167	Type::getIntNTy(C&: Ty->getContext(), N: NumElements * ElementSize * `8`);
2168
2169	Use *U = S.getUse();
2170
2171	if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2172	if (MI->isVolatile())
2173	return false;
2174	if (!S.isSplittable())
2175	return false; // Skip any unsplittable intrinsics.
2176	if (isa<MemSetInst>(Val: MI)) {
2177	Type SplatTy = Type::getIntNTy(C&: Ty->getContext(), N: ElementSize `8`);
2178	if (!canConvertValue(DL, OldTy: SplatTy, NewTy: Ty->getElementType(), VScale))
2179	return false;
2180	}
2181	} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2182	if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2183	return false;
2184	} else if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2185	if (LI->isVolatile())
2186	return false;
2187	Type *LTy = LI->getType();
2188	// Disable vector promotion when there are loads or stores of an FCA.
2189	if (LTy->isStructTy())
2190	return false;
2191	if (P.beginOffset() > S.beginOffset() \|\| P.endOffset() < S.endOffset()) {
2192	assert(LTy->isIntegerTy());
2193	LTy = SplitIntTy;
2194	}
2195	if (!canConvertValue(DL, OldTy: SliceTy, NewTy: LTy, VScale))
2196	return false;
2197	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2198	if (SI->isVolatile())
2199	return false;
2200	Type *STy = SI->getValueOperand()->getType();
2201	// Disable vector promotion when there are loads or stores of an FCA.
2202	if (STy->isStructTy())
2203	return false;
2204	if (P.beginOffset() > S.beginOffset() \|\| P.endOffset() < S.endOffset()) {
2205	assert(STy->isIntegerTy());
2206	STy = SplitIntTy;
2207	}
2208	if (!canConvertValue(DL, OldTy: STy, NewTy: SliceTy, VScale))
2209	return false;
2210	} else {
2211	return false;
2212	}
2213
2214	return true;
2215	}
2216
2217	/// Test whether any vector type in \p CandidateTys is viable for promotion.
2218	///
2219	/// This implements the necessary checking for \c isVectorPromotionViable over
2220	/// all slices of the alloca for the given VectorType.
2221	static VectorType *
2222	checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
2223	SmallVectorImpl<VectorType *> &CandidateTys,
2224	bool HaveCommonEltTy, Type *CommonEltTy,
2225	bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2226	VectorType CommonVecPtrTy, unsigned* VScale) {
2227	// If we didn't find a vector type, nothing to do here.
2228	if (CandidateTys.empty())
2229	return nullptr;
2230
2231	// Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2232	// then we should choose it, not some other alternative.
2233	// But, we can't perform a no-op pointer address space change via bitcast,
2234	// so if we didn't have a common pointer element type, bail.
2235	if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2236	return nullptr;
2237
2238	// Try to pick the "best" element type out of the choices.
2239	if (!HaveCommonEltTy && HaveVecPtrTy) {
2240	// If there was a pointer element type, there's really only one choice.
2241	CandidateTys.clear();
2242	CandidateTys.push_back(Elt: CommonVecPtrTy);
2243	} else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2244	// Integer-ify vector types.
2245	for (VectorType *&VTy : CandidateTys) {
2246	if (!VTy->getElementType()->isIntegerTy())
2247	VTy = cast<VectorType>(Val: VTy->getWithNewType(EltTy: IntegerType::getIntNTy(
2248	C&: VTy->getContext(), N: VTy->getScalarSizeInBits())));
2249	}
2250
2251	// Rank the remaining candidate vector types. This is easy because we know
2252	// they're all integer vectors. We sort by ascending number of elements.
2253	auto RankVectorTypesComp = [&DL](VectorType RHSTy, VectorType LHSTy) {
2254	(void)DL;
2255	assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2256	DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2257	"Cannot have vector types of different sizes!");
2258	assert(RHSTy->getElementType()->isIntegerTy() &&
2259	"All non-integer types eliminated!");
2260	assert(LHSTy->getElementType()->isIntegerTy() &&
2261	"All non-integer types eliminated!");
2262	return cast<FixedVectorType>(Val: RHSTy)->getNumElements() <
2263	cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2264	};
2265	auto RankVectorTypesEq = [&DL](VectorType RHSTy, VectorType LHSTy) {
2266	(void)DL;
2267	assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2268	DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2269	"Cannot have vector types of different sizes!");
2270	assert(RHSTy->getElementType()->isIntegerTy() &&
2271	"All non-integer types eliminated!");
2272	assert(LHSTy->getElementType()->isIntegerTy() &&
2273	"All non-integer types eliminated!");
2274	return cast<FixedVectorType>(Val: RHSTy)->getNumElements() ==
2275	cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2276	};
2277	llvm::sort(C&: CandidateTys, Comp: RankVectorTypesComp);
2278	CandidateTys.erase(CS: llvm::unique(R&: CandidateTys, P: RankVectorTypesEq),
2279	CE: CandidateTys.end());
2280	} else {
2281	// The only way to have the same element type in every vector type is to
2282	// have the same vector type. Check that and remove all but one.
2283	#ifndef NDEBUG
2284	for (VectorType *VTy : CandidateTys) {
2285	assert(VTy->getElementType() == CommonEltTy &&
2286	"Unaccounted for element type!");
2287	assert(VTy == CandidateTys[`0`] &&
2288	"Different vector types with the same element type!");
2289	}
2290	#endif
2291	CandidateTys.resize(N: `1`);
2292	}
2293
2294	// FIXME: hack. Do we have a named constant for this?
2295	// SDAG SDNode can't have more than 65535 operands.
2296	llvm::erase_if(C&: CandidateTys, P: [](VectorType *VTy) {
2297	return cast<FixedVectorType>(Val: VTy)->getNumElements() >
2298	std::numeric_limits<unsigned short>::max();
2299	});
2300
2301	// Find a vector type viable for promotion by iterating over all slices.
2302	auto VTy = llvm::find_if(Range&: CandidateTys, P: [&](VectorType VTy) -> bool {
2303	uint64_t ElementSize =
2304	DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2305
2306	// While the definition of LLVM vectors is bitpacked, we don't support sizes
2307	// that aren't byte sized.
2308	if (ElementSize % `8`)
2309	return false;
2310	assert((DL.getTypeSizeInBits(VTy).getFixedValue() % `8`) == `0` &&
2311	"vector size not a multiple of element size?");
2312	ElementSize /= `8`;
2313
2314	for (const Slice &S : P)
2315	if (!isVectorPromotionViableForSlice(P, S, Ty: VTy, ElementSize, DL, VScale))
2316	return false;
2317
2318	for (const Slice *S : P.splitSliceTails())
2319	if (!isVectorPromotionViableForSlice(P, S: *S, Ty: VTy, ElementSize, DL, VScale))
2320	return false;
2321
2322	return true;
2323	});
2324	return VTy != CandidateTys.end() ? VTy : nullptr*;
2325	}
2326
2327	static VectorType *createAndCheckVectorTypesForPromotion(
2328	SetVector<Type > &OtherTys, ArrayRef<VectorType > CandidateTysCopy,
2329	function_ref<void(Type *)> CheckCandidateType, Partition &P,
2330	const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2331	bool &HaveCommonEltTy, Type &CommonEltTy, bool* &HaveVecPtrTy,
2332	bool &HaveCommonVecPtrTy, VectorType &CommonVecPtrTy, unsigned* VScale) {
2333	[[maybe_unused]] VectorType *OriginalElt =
2334	CandidateTysCopy.size() ? CandidateTysCopy [`0`] : nullptr;
2335	// Consider additional vector types where the element type size is a
2336	// multiple of load/store element size.
2337	for (Type *Ty : OtherTys) {
2338	if (!VectorType::isValidElementType(ElemTy: Ty))
2339	continue;
2340	unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2341	// Make a copy of CandidateTys and iterate through it, because we
2342	// might append to CandidateTys in the loop.
2343	for (VectorType *const VTy : CandidateTysCopy) {
2344	// The elements in the copy should remain invariant throughout the loop
2345	assert(CandidateTysCopy[`0`] == OriginalElt && "Different Element");
2346	unsigned VectorSize = DL.getTypeSizeInBits(Ty: VTy).getFixedValue();
2347	unsigned ElementSize =
2348	DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2349	if (TypeSize != VectorSize && TypeSize != ElementSize &&
2350	VectorSize % TypeSize == `0`) {
2351	VectorType NewVTy = VectorType::get(ElementType: Ty, NumElements: VectorSize / TypeSize, Scalable: false*);
2352	CheckCandidateType (NewVTy);
2353	}
2354	}
2355	}
2356
2357	return checkVectorTypesForPromotion(
2358	P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2359	HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2360	}
2361
2362	/// Test whether the given alloca partitioning and range of slices can be
2363	/// promoted to a vector.
2364	///
2365	/// This is a quick test to check whether we can rewrite a particular alloca
2366	/// partition (and its newly formed alloca) into a vector alloca with only
2367	/// whole-vector loads and stores such that it could be promoted to a vector
2368	/// SSA value. We only can ensure this for a limited set of operations, and we
2369	/// don't want to do the rewrites unless we are confident that the result will
2370	/// be promotable, so we have an early test here.
2371	static VectorType isVectorPromotionViable(Partition &P, const* DataLayout &DL,
2372	unsigned VScale) {
2373	// Collect the candidate types for vector-based promotion. Also track whether
2374	// we have different element types.
2375	SmallVector<VectorType *, `4`> CandidateTys;
2376	SetVector<Type *> LoadStoreTys;
2377	SetVector<Type *> DeferredTys;
2378	Type CommonEltTy = nullptr*;
2379	VectorType CommonVecPtrTy = nullptr*;
2380	bool HaveVecPtrTy = false;
2381	bool HaveCommonEltTy = true;
2382	bool HaveCommonVecPtrTy = true;
2383	auto CheckCandidateType = [&](Type *Ty) {
2384	if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2385	// Return if bitcast to vectors is different for total size in bits.
2386	if (!CandidateTys.empty()) {
2387	VectorType *V = CandidateTys [`0`];
2388	if (DL.getTypeSizeInBits(Ty: VTy).getFixedValue() !=
2389	DL.getTypeSizeInBits(Ty: V).getFixedValue()) {
2390	CandidateTys.clear();
2391	return;
2392	}
2393	}
2394	CandidateTys.push_back(Elt: VTy);
2395	Type *EltTy = VTy->getElementType();
2396
2397	if (!CommonEltTy)
2398	CommonEltTy = EltTy;
2399	else if (CommonEltTy != EltTy)
2400	HaveCommonEltTy = false;
2401
2402	if (EltTy->isPointerTy()) {
2403	HaveVecPtrTy = true;
2404	if (!CommonVecPtrTy)
2405	CommonVecPtrTy = VTy;
2406	else if (CommonVecPtrTy != VTy)
2407	HaveCommonVecPtrTy = false;
2408	}
2409	}
2410	};
2411
2412	// Put load and store types into a set for de-duplication.
2413	for (const Slice &S : P) {
2414	Type *Ty;
2415	if (auto *LI = dyn_cast<LoadInst>(Val: S.getUse()->getUser()))
2416	Ty = LI->getType();
2417	else if (auto *SI = dyn_cast<StoreInst>(Val: S.getUse()->getUser()))
2418	Ty = SI->getValueOperand()->getType();
2419	else
2420	continue;
2421
2422	auto CandTy = Ty->getScalarType();
2423	if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() \|\|
2424	S.endOffset() != P.endOffset())) {
2425	DeferredTys.insert(X: Ty);
2426	continue;
2427	}
2428
2429	LoadStoreTys.insert(X: Ty);
2430	// Consider any loads or stores that are the exact size of the slice.
2431	if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2432	CheckCandidateType (Ty);
2433	}
2434
2435	SmallVector<VectorType *, `4`> CandidateTysCopy = CandidateTys;
2436	if (auto *VTy = createAndCheckVectorTypesForPromotion(
2437	OtherTys&: LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2438	CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2439	HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2440	return VTy;
2441
2442	CandidateTys.clear();
2443	return createAndCheckVectorTypesForPromotion(
2444	OtherTys&: DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2445	HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2446	CommonVecPtrTy, VScale);
2447	}
2448
2449	/// Test whether a slice of an alloca is valid for integer widening.
2450	///
2451	/// This implements the necessary checking for the \c isIntegerWideningViable
2452	/// test below on a single slice of the alloca.
2453	static bool isIntegerWideningViableForSlice(const Slice &S,
2454	uint64_t AllocBeginOffset,
2455	Type *AllocaTy,
2456	const DataLayout &DL,
2457	bool &WholeAllocaOp) {
2458	uint64_t Size = DL.getTypeStoreSize(Ty: AllocaTy).getFixedValue();
2459
2460	uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2461	uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2462
2463	Use *U = S.getUse();
2464
2465	// Lifetime intrinsics operate over the whole alloca whose sizes are usually
2466	// larger than other load/store slices (RelEnd > Size). But lifetime are
2467	// always promotable and should not impact other slices' promotability of the
2468	// partition.
2469	if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2470	if (II->isLifetimeStartOrEnd() \|\| II->isDroppable())
2471	return true;
2472	}
2473
2474	// We can't reasonably handle cases where the load or store extends past
2475	// the end of the alloca's type and into its padding.
2476	if (RelEnd > Size)
2477	return false;
2478
2479	if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2480	if (LI->isVolatile())
2481	return false;
2482	// We can't handle loads that extend past the allocated memory.
2483	TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
2484	if (!LoadSize.isFixed() \|\| LoadSize.getFixedValue() > Size)
2485	return false;
2486	// So far, AllocaSliceRewriter does not support widening split slice tails
2487	// in rewriteIntegerLoad.
2488	if (S.beginOffset() < AllocBeginOffset)
2489	return false;
2490	// Note that we don't count vector loads or stores as whole-alloca
2491	// operations which enable integer widening because we would prefer to use
2492	// vector widening instead.
2493	if (!isa<VectorType>(Val: LI->getType()) && RelBegin == `0` && RelEnd == Size)
2494	WholeAllocaOp = true;
2495	if (IntegerType *ITy = dyn_cast<IntegerType>(Val: LI->getType())) {
2496	if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2497	return false;
2498	} else if (RelBegin != `0` \|\| RelEnd != Size \|\|
2499	!canConvertValue(DL, OldTy: AllocaTy, NewTy: LI->getType())) {
2500	// Non-integer loads need to be convertible from the alloca type so that
2501	// they are promotable.
2502	return false;
2503	}
2504	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2505	Type *ValueTy = SI->getValueOperand()->getType();
2506	if (SI->isVolatile())
2507	return false;
2508	// We can't handle stores that extend past the allocated memory.
2509	TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValueTy);
2510	if (!StoreSize.isFixed() \|\| StoreSize.getFixedValue() > Size)
2511	return false;
2512	// So far, AllocaSliceRewriter does not support widening split slice tails
2513	// in rewriteIntegerStore.
2514	if (S.beginOffset() < AllocBeginOffset)
2515	return false;
2516	// Note that we don't count vector loads or stores as whole-alloca
2517	// operations which enable integer widening because we would prefer to use
2518	// vector widening instead.
2519	if (!isa<VectorType>(Val: ValueTy) && RelBegin == `0` && RelEnd == Size)
2520	WholeAllocaOp = true;
2521	if (IntegerType *ITy = dyn_cast<IntegerType>(Val: ValueTy)) {
2522	if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2523	return false;
2524	} else if (RelBegin != `0` \|\| RelEnd != Size \|\|
2525	!canConvertValue(DL, OldTy: ValueTy, NewTy: AllocaTy)) {
2526	// Non-integer stores need to be convertible to the alloca type so that
2527	// they are promotable.
2528	return false;
2529	}
2530	} else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2531	if (MI->isVolatile() \|\| !isa<Constant>(Val: MI->getLength()))
2532	return false;
2533	if (!S.isSplittable())
2534	return false; // Skip any unsplittable intrinsics.
2535	} else {
2536	return false;
2537	}
2538
2539	return true;
2540	}
2541
2542	/// Test whether the given alloca partition's integer operations can be
2543	/// widened to promotable ones.
2544	///
2545	/// This is a quick test to check whether we can rewrite the integer loads and
2546	/// stores to a particular alloca into wider loads and stores and be able to
2547	/// promote the resulting alloca.
2548	static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2549	const DataLayout &DL) {
2550	uint64_t SizeInBits = DL.getTypeSizeInBits(Ty: AllocaTy).getFixedValue();
2551	// Don't create integer types larger than the maximum bitwidth.
2552	if (SizeInBits > IntegerType::MAX_INT_BITS)
2553	return false;
2554
2555	// Don't try to handle allocas with bit-padding.
2556	if (SizeInBits != DL.getTypeStoreSizeInBits(Ty: AllocaTy).getFixedValue())
2557	return false;
2558
2559	// We need to ensure that an integer type with the appropriate bitwidth can
2560	// be converted to the alloca type, whatever that is. We don't want to force
2561	// the alloca itself to have an integer type if there is a more suitable one.
2562	Type *IntTy = Type::getIntNTy(C&: AllocaTy->getContext(), N: SizeInBits);
2563	if (!canConvertValue(DL, OldTy: AllocaTy, NewTy: IntTy) \|\|
2564	!canConvertValue(DL, OldTy: IntTy, NewTy: AllocaTy))
2565	return false;
2566
2567	// While examining uses, we ensure that the alloca has a covering load or
2568	// store. We don't want to widen the integer operations only to fail to
2569	// promote due to some other unsplittable entry (which we may make splittable
2570	// later). However, if there are only splittable uses, go ahead and assume
2571	// that we cover the alloca.
2572	// FIXME: We shouldn't consider split slices that happen to start in the
2573	// partition here...
2574	bool WholeAllocaOp = P.empty() && DL.isLegalInteger(Width: SizeInBits);
2575
2576	for (const Slice &S : P)
2577	if (!isIntegerWideningViableForSlice(S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2578	WholeAllocaOp))
2579	return false;
2580
2581	for (const Slice *S : P.splitSliceTails())
2582	if (!isIntegerWideningViableForSlice(S: *S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2583	WholeAllocaOp))
2584	return false;
2585
2586	return WholeAllocaOp;
2587	}
2588
2589	static Value extractInteger(const* DataLayout &DL, IRBuilderTy &IRB, Value *V,
2590	IntegerType *Ty, uint64_t Offset,
2591	const Twine &Name) {
2592	LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2593	IntegerType *IntTy = cast<IntegerType>(Val: V->getType());
2594	assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2595	DL.getTypeStoreSize(IntTy).getFixedValue() &&
2596	"Element extends past full value");
2597	uint64_t ShAmt = `8` * Offset;
2598	if (DL.isBigEndian())
2599	ShAmt = `8` * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2600	DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2601	if (ShAmt) {
2602	V = IRB.CreateLShr(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2603	LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2604	}
2605	assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2606	"Cannot extract to a larger integer!");
2607	if (Ty != IntTy) {
2608	V = IRB.CreateTrunc(V, DestTy: Ty, Name: Name + ".trunc");
2609	LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2610	}
2611	return V;
2612	}
2613
2614	static Value insertInteger(const* DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2615	Value V, uint64_t Offset, const* Twine &Name) {
2616	IntegerType *IntTy = cast<IntegerType>(Val: Old->getType());
2617	IntegerType *Ty = cast<IntegerType>(Val: V->getType());
2618	assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2619	"Cannot insert a larger integer!");
2620	LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2621	if (Ty != IntTy) {
2622	V = IRB.CreateZExt(V, DestTy: IntTy, Name: Name + ".ext");
2623	LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2624	}
2625	assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2626	DL.getTypeStoreSize(IntTy).getFixedValue() &&
2627	"Element store outside of alloca store");
2628	uint64_t ShAmt = `8` * Offset;
2629	if (DL.isBigEndian())
2630	ShAmt = `8` * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2631	DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2632	if (ShAmt) {
2633	V = IRB.CreateShl(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2634	LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2635	}
2636
2637	if (ShAmt \|\| Ty->getBitWidth() < IntTy->getBitWidth()) {
2638	APInt Mask = ~Ty->getMask().zext(width: IntTy->getBitWidth()).shl(shiftAmt: ShAmt);
2639	Old = IRB.CreateAnd(LHS: Old, RHS: Mask, Name: Name + ".mask");
2640	LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2641	V = IRB.CreateOr(LHS: Old, RHS: V, Name: Name + ".insert");
2642	LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2643	}
2644	return V;
2645	}
2646
2647	static Value extractVector(IRBuilderTy &IRB, Value V, unsigned BeginIndex,
2648	unsigned EndIndex, const Twine &Name) {
2649	auto *VecTy = cast<FixedVectorType>(Val: V->getType());
2650	unsigned NumElements = EndIndex - BeginIndex;
2651	assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2652
2653	if (NumElements == VecTy->getNumElements())
2654	return V;
2655
2656	if (NumElements == `1`) {
2657	V = IRB.CreateExtractElement(Vec: V, Idx: BeginIndex, Name: Name + ".extract");
2658	LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2659	return V;
2660	}
2661
2662	auto Mask = llvm::to_vector<`8`>(Range: llvm::seq<int>(Begin: BeginIndex, End: EndIndex));
2663	V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".extract");
2664	LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2665	return V;
2666	}
2667
2668	static Value insertVector(IRBuilderTy &IRB, Value Old, Value *V,
2669	unsigned BeginIndex, const Twine &Name) {
2670	VectorType *VecTy = cast<VectorType>(Val: Old->getType());
2671	assert(VecTy && "Can only insert a vector into a vector");
2672
2673	VectorType *Ty = dyn_cast<VectorType>(Val: V->getType());
2674	if (!Ty) {
2675	// Single element to insert.
2676	V = IRB.CreateInsertElement(Vec: Old, NewElt: V, Idx: BeginIndex, Name: Name + ".insert");
2677	LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2678	return V;
2679	}
2680
2681	unsigned NumSubElements = cast<FixedVectorType>(Val: Ty)->getNumElements();
2682	unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2683
2684	assert(NumSubElements <= NumElements && "Too many elements!");
2685	if (NumSubElements == NumElements) {
2686	assert(V->getType() == VecTy && "Vector type mismatch");
2687	return V;
2688	}
2689	unsigned EndIndex = BeginIndex + NumSubElements;
2690
2691	// When inserting a smaller vector into the larger to store, we first
2692	// use a shuffle vector to widen it with undef elements, and then
2693	// a second shuffle vector to select between the loaded vector and the
2694	// incoming vector.
2695	SmallVector<int, `8`> Mask;
2696	Mask.reserve(N: NumElements);
2697	for (unsigned Idx = `0`; Idx != NumElements; ++Idx)
2698	if (Idx >= BeginIndex && Idx < EndIndex)
2699	Mask.push_back(Elt: Idx - BeginIndex);
2700	else
2701	Mask.push_back(Elt: -`1`);
2702	V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".expand");
2703	LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2704
2705	Mask.clear();
2706	for (unsigned Idx = `0`; Idx != NumElements; ++Idx)
2707	if (Idx >= BeginIndex && Idx < EndIndex)
2708	Mask.push_back(Elt: Idx);
2709	else
2710	Mask.push_back(Elt: Idx + NumElements);
2711	V = IRB.CreateShuffleVector(V1: V, V2: Old, Mask, Name: Name + "blend");
2712	LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2713	return V;
2714	}
2715
2716	/// This function takes two vector values and combines them into a single vector
2717	/// by concatenating their elements. The function handles:
2718	///
2719	/// 1. Element type mismatch: If either vector's element type differs from
2720	/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2721	/// preserving the total bit width (adjusting the number of elements
2722	/// accordingly).
2723	///
2724	/// 2. Size mismatch: After transforming the vectors to have the desired element
2725	/// type, if the two vectors have different numbers of elements, the smaller
2726	/// vector is extended with poison values to match the size of the larger
2727	/// vector before concatenation.
2728	///
2729	/// 3. Concatenation: The vectors are merged using a shuffle operation that
2730	/// places all elements of V0 first, followed by all elements of V1.
2731	///
2732	/// \param V0 The first vector to merge (must be a vector type)
2733	/// \param V1 The second vector to merge (must be a vector type)
2734	/// \param DL The data layout for size calculations
2735	/// \param NewAIEltTy The desired element type for the result vector
2736	/// \param Builder IRBuilder for creating new instructions
2737	/// \return A new vector containing all elements from V0 followed by all
2738	/// elements from V1
2739	static Value mergeTwoVectors(Value V0, Value V1, const* DataLayout &DL,
2740	Type *NewAIEltTy, IRBuilder<> &Builder) {
2741	// V0 and V1 are vectors
2742	// Create a new vector type with combined elements
2743	// Use ShuffleVector to concatenate the vectors
2744	auto *VecType0 = cast<FixedVectorType>(Val: V0->getType());
2745	auto *VecType1 = cast<FixedVectorType>(Val: V1->getType());
2746
2747	// If V0/V1 element types are different from NewAllocaElementType,
2748	// we need to introduce bitcasts before merging them
2749	auto BitcastIfNeeded = [&](Value &V, FixedVectorType &VecType,
2750	const char *DebugName) {
2751	Type *EltType = VecType->getElementType();
2752	if (EltType != NewAIEltTy) {
2753	// Calculate new number of elements to maintain same bit width
2754	unsigned TotalBits =
2755	VecType->getNumElements() * DL.getTypeSizeInBits(Ty: EltType);
2756	unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(Ty: NewAIEltTy);
2757
2758	auto *NewVecType = FixedVectorType::get(ElementType: NewAIEltTy, NumElts: NewNumElts);
2759	V = Builder.CreateBitCast(V, DestTy: NewVecType);
2760	VecType = NewVecType;
2761	LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2762	}
2763	};
2764
2765	BitcastIfNeeded (V0, VecType0, "V0");
2766	BitcastIfNeeded (V1, VecType1, "V1");
2767
2768	unsigned NumElts0 = VecType0->getNumElements();
2769	unsigned NumElts1 = VecType1->getNumElements();
2770
2771	SmallVector<int, `16`> ShuffleMask;
2772
2773	if (NumElts0 == NumElts1) {
2774	for (unsigned i = `0`; i < NumElts0 + NumElts1; ++i)
2775	ShuffleMask.push_back(Elt: i);
2776	} else {
2777	// If two vectors have different sizes, we need to extend
2778	// the smaller vector to the size of the larger vector.
2779	unsigned SmallSize = std::min(a: NumElts0, b: NumElts1);
2780	unsigned LargeSize = std::max(a: NumElts0, b: NumElts1);
2781	bool IsV0Smaller = NumElts0 < NumElts1;
2782	Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2783	SmallVector<int, `16`> ExtendMask;
2784	for (unsigned i = `0`; i < SmallSize; ++i)
2785	ExtendMask.push_back(Elt: i);
2786	for (unsigned i = SmallSize; i < LargeSize; ++i)
2787	ExtendMask.push_back(Elt: PoisonMaskElem);
2788	ExtendedVec = Builder.CreateShuffleVector(
2789	V1: ExtendedVec, V2: PoisonValue::get(T: ExtendedVec->getType()), Mask: ExtendMask);
2790	LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2791	for (unsigned i = `0`; i < NumElts0; ++i)
2792	ShuffleMask.push_back(Elt: i);
2793	for (unsigned i = `0`; i < NumElts1; ++i)
2794	ShuffleMask.push_back(Elt: LargeSize + i);
2795	}
2796
2797	return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
2798	}
2799
2800	namespace {
2801
2802	/// Visitor to rewrite instructions using p particular slice of an alloca
2803	/// to use a new alloca.
2804	///
2805	/// Also implements the rewriting to vector-based accesses when the partition
2806	/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2807	/// lives here.
2808	class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2809	// Befriend the base class so it can delegate to private visit methods.
2810	friend class InstVisitor<AllocaSliceRewriter, bool>;
2811
2812	using Base = InstVisitor<AllocaSliceRewriter, bool>;
2813
2814	const DataLayout &DL;
2815	AllocaSlices &AS;
2816	SROA &Pass;
2817	AllocaInst &OldAI, &NewAI;
2818	const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2819	Type *NewAllocaTy;
2820
2821	// This is a convenience and flag variable that will be null unless the new
2822	// alloca's integer operations should be widened to this integer type due to
2823	// passing isIntegerWideningViable above. If it is non-null, the desired
2824	// integer type will be stored here for easy access during rewriting.
2825	IntegerType *IntTy;
2826
2827	// If we are rewriting an alloca partition which can be written as pure
2828	// vector operations, we stash extra information here. When VecTy is
2829	// non-null, we have some strict guarantees about the rewritten alloca:
2830	// - The new alloca is exactly the size of the vector type here.
2831	// - The accesses all either map to the entire vector or to a single
2832	// element.
2833	// - The set of accessing instructions is only one of those handled above
2834	// in isVectorPromotionViable. Generally these are the same access kinds
2835	// which are promotable via mem2reg.
2836	VectorType *VecTy;
2837	Type *ElementTy;
2838	uint64_t ElementSize;
2839
2840	// The original offset of the slice currently being rewritten relative to
2841	// the original alloca.
2842	uint64_t BeginOffset = `0`;
2843	uint64_t EndOffset = `0`;
2844
2845	// The new offsets of the slice currently being rewritten relative to the
2846	// original alloca.
2847	uint64_t NewBeginOffset = `0`, NewEndOffset = `0`;
2848
2849	uint64_t SliceSize = `0`;
2850	bool IsSplittable = false;
2851	bool IsSplit = false;
2852	Use OldUse = nullptr*;
2853	Instruction OldPtr = nullptr*;
2854
2855	// Track post-rewrite users which are PHI nodes and Selects.
2856	SmallSetVector<PHINode *, `8`> &PHIUsers;
2857	SmallSetVector<SelectInst *, `8`> &SelectUsers;
2858
2859	// Utility IR builder, whose name prefix is setup for each visited use, and
2860	// the insertion point is set to point to the user.
2861	IRBuilderTy IRB;
2862
2863	// Return the new alloca, addrspacecasted if required to avoid changing the
2864	// addrspace of a volatile access.
2865	Value getPtrToNewAI(unsigned* AddrSpace, bool IsVolatile) {
2866	if (!IsVolatile \|\| AddrSpace == NewAI.getType()->getPointerAddressSpace())
2867	return &NewAI;
2868
2869	Type *AccessTy = IRB.getPtrTy(AddrSpace);
2870	return IRB.CreateAddrSpaceCast(V: &NewAI, DestTy: AccessTy);
2871	}
2872
2873	public:
2874	AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2875	AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2876	uint64_t NewAllocaBeginOffset,
2877	uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2878	VectorType *PromotableVecTy,
2879	SmallSetVector<PHINode *, `8`> &PHIUsers,
2880	SmallSetVector<SelectInst *, `8`> &SelectUsers)
2881	: DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2882	NewAllocaBeginOffset(NewAllocaBeginOffset),
2883	NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2884	IntTy(IsIntegerPromotable
2885	? Type::getIntNTy(
2886	C&: NewAI.getContext(),
2887	N: DL.getTypeSizeInBits(Ty: NewAllocaTy).getFixedValue())
2888	: nullptr),
2889	VecTy(PromotableVecTy),
2890	ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2891	ElementSize(VecTy ? DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / `8`
2892	: `0`),
2893	PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2894	IRB (*NewAI.getModule(), ConstantFolder ()) {
2895	if (VecTy) {
2896	assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % `8`) == `0` &&
2897	"Only multiple-of-8 sized vector elements are viable");
2898	++NumVectorized;
2899	}
2900	assert((!IntTy && !VecTy) \|\| (IntTy && !VecTy) \|\| (!IntTy && VecTy));
2901	}
2902
2903	bool visit(AllocaSlices::const_iterator I) {
2904	bool CanSROA = true;
2905	BeginOffset = I->beginOffset();
2906	EndOffset = I->endOffset();
2907	IsSplittable = I->isSplittable();
2908	IsSplit =
2909	BeginOffset < NewAllocaBeginOffset \|\| EndOffset > NewAllocaEndOffset;
2910	LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2911	LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2912	LLVM_DEBUG(dbgs() << "\n");
2913
2914	// Compute the intersecting offset range.
2915	assert(BeginOffset < NewAllocaEndOffset);
2916	assert(EndOffset > NewAllocaBeginOffset);
2917	NewBeginOffset = std::max(a: BeginOffset, b: NewAllocaBeginOffset);
2918	NewEndOffset = std::min(a: EndOffset, b: NewAllocaEndOffset);
2919
2920	SliceSize = NewEndOffset - NewBeginOffset;
2921	LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2922	<< ") NewBegin:(" << NewBeginOffset << ", "
2923	<< NewEndOffset << ") NewAllocaBegin:("
2924	<< NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2925	<< ")\n");
2926	assert(IsSplit \|\| NewBeginOffset == BeginOffset);
2927	OldUse = I->getUse();
2928	OldPtr = cast<Instruction>(Val: OldUse->get());
2929
2930	Instruction *OldUserI = cast<Instruction>(Val: OldUse->getUser());
2931	IRB.SetInsertPoint(OldUserI);
2932	IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2933	// Avoid materializing the name prefix when it is discarded anyway.
2934	if (!IRB.getContext().shouldDiscardValueNames())
2935	IRB.getInserter().SetNamePrefix(Twine (NewAI.getName()) + "." +
2936	Twine (BeginOffset) + ".");
2937
2938	CanSROA &= visit(I: cast<Instruction>(Val: OldUse->getUser()));
2939	if (VecTy \|\| IntTy)
2940	assert(CanSROA);
2941	return CanSROA;
2942	}
2943
2944	/// Attempts to rewrite a partition using tree-structured merge optimization.
2945	///
2946	/// This function handles two patterns. Both produce an O(log n) tree of
2947	/// shufflevectors in place of the linear expand+blend chain that SROA would
2948	/// otherwise emit for each partial store.
2949	///
2950	/// Pattern 1 (stores-only):
2951	/// Multiple non-overlapping partial stores completely fill the alloca
2952	/// and there is exactly one full-width load coming after the stores.
2953	/// The stores are tree-merged into a single vector and stored once.
2954	///
2955	/// Example transformation:
2956	/// Before: (stores do not have to be in order)
2957	/// %alloca = alloca <8 x float>
2958	/// store <2 x float> %val0, ptr %alloca ; offset 0-1
2959	/// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2960	/// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2961	/// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2962	/// %r = load <8 x float>, ptr %alloca
2963	///
2964	/// After: tree of shufflevectors producing <8 x float> directly.
2965	///
2966	/// Pattern 2 (init + RMW, possibly multi-round):
2967	/// A single full-width init store, followed by partial loads and
2968	/// partial stores that read-modify-write the alloca one or more
2969	/// times, optionally followed by a full-width load. The only
2970	/// structural requirement is that the distinct [begin, end) ranges
2971	/// touched by the partial loads and stores, taken together, tile
2972	/// the alloca disjointly.
2973	///
2974	/// We keep a map from each slice range to the SSA value that
2975	/// currently lives there, `SliceValues[r] -> Value`:*
2976	/// - initialize each entry to the corresponding piece of the
2977	/// init store's value (via a shufflevector picking the
2978	/// range's elements out of the init value),
2979	/// - walk partial loads and stores in block order,
2980	/// - for a partial load at range r: RAUW with `SliceValues[r]`,
2981	/// - for a partial store at range r: update `SliceValues[r]` to
2982	/// the stored value and drop the store.
2983	/// At the end, the final `SliceValues[r]` entries are tree-merged
2984	/// (in range order) into a single store to the alloca, and the
2985	/// optional full-width load is replaced by a load of the alloca.
2986	///
2987	/// Because the ranges are disjoint by construction, a store at one
2988	/// range cannot affect another range's tracked value, so a single
2989	/// block-order walk correctly tracks the memory state at each
2990	/// range. The algorithm handles multi-round RMW, partial loads
2991	/// and stores interleaved in any order, read-only slices (the
2992	/// tracked value stays at the init extract), and write-only
2993	/// slices (the tracked value never flows into a load).
2994	///
2995	/// \param P The partition to analyze and potentially rewrite
2996	/// \return An optional vector of values that were deleted during the
2997	/// rewrite, or std::nullopt if the partition cannot be optimized.
2998	std::optional<SmallVector<Value *, `4`>>
2999	rewriteTreeStructuredMerge(Partition &P) {
3000	// No tail slices that overlap with the partition
3001	if (P.splitSliceTails().size() > `0`)
3002	return std::nullopt;
3003
3004	// Structure to hold store information
3005	struct StoreInfo {
3006	StoreInst *Store;
3007	uint64_t BeginOffset;
3008	uint64_t EndOffset;
3009	Value *StoredValue;
3010	StoreInfo(StoreInst SI, uint64_t Begin, uint64_t End, Value Val)
3011	: Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
3012	};
3013	struct LoadInfo {
3014	LoadInst *Load;
3015	uint64_t BeginOffset;
3016	uint64_t EndOffset;
3017	};
3018
3019	SmallVector<StoreInfo, `4`> StoreInfos; // partial stores only
3020	SmallVector<LoadInfo, `4`> LoadInfos; // partial loads only
3021	LoadInst FullLoad = nullptr; // optional full-width load*
3022	StoreInst InitStore = nullptr; // optional full-width init store*
3023
3024	// If the new alloca is a fixed vector type, we use its element type as the
3025	// allocated element type, otherwise we use i8 as the allocated element
3026	Type *AllocatedEltTy =
3027	isa<FixedVectorType>(Val: NewAllocaTy)
3028	? cast<FixedVectorType>(Val: NewAllocaTy)->getElementType()
3029	: Type::getInt8Ty(C&: NewAI.getContext());
3030	unsigned AllocatedEltTySize = DL.getTypeSizeInBits(Ty: AllocatedEltTy);
3031
3032	// Helper to check if a type is
3033	// 1. A fixed vector type
3034	// 2. The element type is not a pointer
3035	// 3. The element type size is byte-aligned
3036	// We only handle the cases that the ld/st meet these conditions
3037	auto IsTypeValidForTreeStructuredMerge = [&](Type Ty) -> bool* {
3038	auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: Ty);
3039	return FixedVecTy &&
3040	DL.getTypeSizeInBits(Ty: FixedVecTy->getElementType()) % `8` == `0` &&
3041	!FixedVecTy->getElementType()->isPointerTy();
3042	};
3043
3044	for (Slice &S : P) {
3045	auto *User = cast<Instruction>(Val: S.getUse()->getUser());
3046	// A "full-width" slice spans the entire alloca; it's either the single
3047	// init store (Pattern 2) or the single final load (both patterns).
3048	bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
3049	S.endOffset() == NewAllocaEndOffset);
3050	if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
3051	// Only handle simple (non-volatile, non-atomic) loads.
3052	if (!LI->isSimple() \|\|
3053	!IsTypeValidForTreeStructuredMerge(LI->getType()))
3054	return std::nullopt;
3055	if (IsFullWidth) {
3056	// We accept at most one full-width load (the "final" load, after
3057	// all the partial stores).
3058	if (FullLoad)
3059	return std::nullopt;
3060	FullLoad = LI;
3061	} else {
3062	// Partial load (RMW pattern only).
3063	LoadInfos.push_back(Elt: {.Load: LI, .BeginOffset: S.beginOffset(), .EndOffset: S.endOffset()});
3064	}
3065	} else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
3066	// Do not handle the case if
3067	// 1. The store does not meet the conditions in the helper function
3068	// 2. The store is not simple — we drop stores as part of the
3069	// rewrite, so volatile stores (which must be kept) and atomic
3070	// stores (which carry memory-ordering semantics) are unsound
3071	// to replace with SSA bookkeeping.
3072	// 3. The total store size is not a multiple of the allocated
3073	// element type size (required so the tree merge can produce a
3074	// vector whose element type matches the alloca).
3075	if (!SI->isSimple() \|\| !IsTypeValidForTreeStructuredMerge(
3076	SI->getValueOperand()->getType()))
3077	return std::nullopt;
3078	auto *StVecTy = cast<FixedVectorType>(Val: SI->getValueOperand()->getType());
3079	unsigned NumElts = StVecTy->getNumElements();
3080	unsigned EltSize = DL.getTypeSizeInBits(Ty: StVecTy->getElementType());
3081	if (NumElts * EltSize % AllocatedEltTySize != `0`)
3082	return std::nullopt;
3083	if (IsFullWidth) {
3084	// At most one full-width store is allowed — it's the init store
3085	// for the RMW pattern.
3086	if (InitStore)
3087	return std::nullopt;
3088	InitStore = SI;
3089	} else {
3090	StoreInfos.emplace_back(Args&: SI, Args: S.beginOffset(), Args: S.endOffset(),
3091	Args: SI->getValueOperand());
3092	}
3093	} else {
3094	// If we have instructions other than load and store, we cannot do
3095	// the tree structured merge.
3096	return std::nullopt;
3097	}
3098	}
3099
3100	// Need at least two partial stores to benefit from tree-merging; a
3101	// single store is already optimal as-is. This applies to both patterns
3102	// below, so check it before classifying.
3103	if (StoreInfos.size() < `2`)
3104	return std::nullopt;
3105
3106	// Classify the pattern by looking at what we collected:
3107	// Pattern 1 (stores-only): only partial stores + exactly one full load.
3108	// Pattern 2 (RMW): one full init store + partial loads + partial stores
3109	// (+ optional full final load). RMW also needs VecTy to be set
3110	// because we use getIndex() to convert byte offsets to element
3111	// indices, which requires a promoted vector alloca.
3112	bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
3113	bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
3114	if (!IsRMWPattern && !IsStoresOnlyPattern)
3115	return std::nullopt;
3116
3117	// All partial stores must live in the same basic block — the tree merge
3118	// is built in a single BB using block-order ordering (comesBefore).
3119	BasicBlock *StoreBB = StoreInfos [`0`].Store->getParent();
3120	for (auto &Info : StoreInfos)
3121	if (Info.Store->getParent() != StoreBB)
3122	return std::nullopt;
3123
3124	SmallVector<Value *, `4`> DeletedValues;
3125
3126	// Helper: pairwise tree-merge a list of vectors into a single vector.
3127	// At each iteration we merge each adjacent pair via mergeTwoVectors,
3128	// collect the merged values into Next, and (if Vals had odd length)
3129	// carry the trailing element through unchanged. Loop until one value
3130	// remains — the fully-merged vector.
3131	auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3132	IRBuilder<> &B) -> Value * {
3133	LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3134	while (Vals.size() > `1`) {
3135	SmallVector<Value *, `8`> Next;
3136	for (unsigned I = `0`, E = Vals.size(); I + `1` < E; I += `2`) {
3137	Value *M =
3138	mergeTwoVectors(V0: Vals [I], V1: Vals [I + `1`], DL, NewAIEltTy: AllocatedEltTy, Builder&: B);
3139	LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3140	Next.push_back(Elt: M);
3141	}
3142	if (Vals.size() % `2` == `1`)
3143	Next.push_back(Elt: Vals.back());
3144	Vals = std::move(Next);
3145	}
3146	return Vals [`0`];
3147	};
3148
3149	// Replace a full-width load with a load of the freshly-merged alloca.
3150	// The merge stored a value of type Merged->getType() into NewAI; we load
3151	// that same type back so every access to NewAI stays consistently typed
3152	// (otherwise the alloca is no longer promotable).
3153	auto ReplaceFullLoad = [&](LoadInst LoadToReplace, Value Merged) {
3154	IRBuilder<> LoadBuilder(LoadToReplace);
3155	Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3156	Ty: Merged->getType(), Ptr: &NewAI, Align: getSliceAlign(),
3157	isVolatile: LoadToReplace->isVolatile(),
3158	Name: LoadToReplace->getName() + ".sroa.new.load");
3159	if (NewLoad->getType() != LoadToReplace->getType())
3160	NewLoad = LoadBuilder.CreateBitCast(V: NewLoad, DestTy: LoadToReplace->getType());
3161	LoadToReplace->replaceAllUsesWith(V: NewLoad);
3162	DeletedValues.push_back(Elt: LoadToReplace);
3163	};
3164
3165	if (IsStoresOnlyPattern) {
3166	// Stores should not overlap and should cover the whole alloca.
3167	// Sort by begin offset to verify this with a single linear scan.
3168	llvm::sort(C&: StoreInfos, Comp: [](const StoreInfo &A, const StoreInfo &B) {
3169	return A.BeginOffset < B.BeginOffset;
3170	});
3171	// Check for gap or overlap: each begin offset must equal the previous
3172	// end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3173	// NewAllocaEndOffset) exactly.
3174	uint64_t Expected = NewAllocaBeginOffset;
3175	for (auto &Info : StoreInfos) {
3176	if (Info.BeginOffset != Expected)
3177	return std::nullopt;
3178	Expected = Info.EndOffset;
3179	}
3180	// Stores cover the entire alloca (no trailing gap either).
3181	if (Expected != NewAllocaEndOffset)
3182	return std::nullopt;
3183
3184	// The load should not be in the middle of the stores.
3185	// Note:
3186	// If the load is in a different basic block from the stores, we can
3187	// still do the tree-structured merge. We don't have store->load
3188	// forwarding here — the merged vector is stored back to NewAI and
3189	// the new load loads from NewAI. The forwarding will be handled
3190	// later when NewAI is promoted.
3191	BasicBlock *LoadBB = FullLoad->getParent();
3192	if (LoadBB == StoreBB) {
3193	for (auto &Info : StoreInfos)
3194	if (!Info.Store->comesBefore(Other: FullLoad))
3195	return std::nullopt;
3196	}
3197
3198	LLVM_DEBUG({
3199	dbgs() << "Tree structured merge rewrite (stores-only):\n";
3200	dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3201	for (auto [I, Info] : enumerate(StoreInfos)) {
3202	dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3203	<< Info.EndOffset << ") \tStore: " << *Info.Store
3204	<< "\tValue: " << *Info.StoredValue << "\n";
3205	}
3206	});
3207
3208	// StoreInfos is sorted by offset, not by block order. Anchoring to
3209	// StoreInfos.back().Store (last by offset) can place shuffles before
3210	// operands that appear later in the block (invalid SSA). Insert before
3211	// FullLoad when it shares the store block (after all stores, before
3212	// any later IR in that block). Otherwise insert before the store
3213	// block's terminator so the merge runs after every store and any
3214	// trailing instructions in that block.
3215	IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(Val: FullLoad)
3216	: StoreBB->getTerminator());
3217	SmallVector<Value *, `8`> Vals;
3218	for (const auto &Info : StoreInfos) {
3219	DeletedValues.push_back(Elt: Info.Store);
3220	Vals.push_back(Elt: Info.StoredValue);
3221	}
3222	// Merge all stored values and store the merged value into the alloca.
3223	Value *Merged = TreeMerge(Vals, Builder);
3224	Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3225
3226	// Replace the original load with a load of the newly-merged alloca.
3227	ReplaceFullLoad(FullLoad, Merged);
3228	return DeletedValues;
3229	}
3230
3231	// RMW pattern handling starts from here.
3232	// Like StoreBB above: keep the init store, all partial loads and all
3233	// partial stores in one basic block so we can reason about ordering
3234	// with comesBefore and build SSA without PHIs.
3235	if (InitStore->getParent() != StoreBB)
3236	return std::nullopt;
3237	if (any_of(Range&: LoadInfos, P: [&](const LoadInfo &I) {
3238	return I.Load->getParent() != StoreBB;
3239	}))
3240	return std::nullopt;
3241	// FullLoad (if any) is allowed to live in a different basic block. See
3242	// the note on the stores-only path: we don't do store->load forwarding
3243	// directly — the merged vector is stored to NewAI and the new load
3244	// loads from NewAI, so cross-BB ordering is resolved later when NewAI
3245	// is promoted.
3246
3247	// Collect the combined partial-load/partial-store accesses sorted
3248	// by block order. Used both for ordering checks and for the rewrite
3249	// walk below.
3250	struct Access {
3251	Instruction *Inst;
3252	uint64_t BeginOffset, EndOffset;
3253	bool IsStore;
3254	};
3255	SmallVector<Access, `16`> Accesses;
3256	Accesses.reserve(N: LoadInfos.size() + StoreInfos.size());
3257	for (const auto &L : LoadInfos)
3258	Accesses.push_back(Elt: {.Inst: L.Load, .BeginOffset: L.BeginOffset, .EndOffset: L.EndOffset, .IsStore: false});
3259	for (const auto &S : StoreInfos)
3260	Accesses.push_back(Elt: {.Inst: S.Store, .BeginOffset: S.BeginOffset, .EndOffset: S.EndOffset, .IsStore: true});
3261	llvm::sort(C&: Accesses, Comp: [](const Access &A, const Access &B) {
3262	return A.Inst->comesBefore(Other: B.Inst);
3263	});
3264
3265	// Ordering constraint 1: InitStore must come before every partial
3266	// access — they read/write the RMW state initialised by InitStore.
3267	// Accesses is sorted by block order, so the first element is the
3268	// earliest; checking it is enough.
3269	if (!InitStore->comesBefore(Other: Accesses.front().Inst))
3270	return std::nullopt;
3271	// Ordering constraint 2: when FullLoad shares the block with the
3272	// partial accesses, it must come after every one of them — otherwise
3273	// it could read a stale value. Accesses is sorted, so the last
3274	// element is the latest; checking it is enough. If FullLoad is in
3275	// another block, mem2reg forwards the merged store to it.
3276	if (FullLoad && FullLoad->getParent() == StoreBB &&
3277	!Accesses.back().Inst->comesBefore(Other: FullLoad))
3278	return std::nullopt;
3279
3280	// Coverage check: the distinct [begin, end) ranges touched by the
3281	// partial loads and stores must tile the alloca disjointly. That is
3282	// the only precondition the per-range SliceValues tracking below
3283	// needs — a disjoint tile guarantees the entries don't alias each
3284	// other. We don't check per-range load/store counts: a range with
3285	// only loads ends with SliceValues[r] = the init extract
3286	// (contributed to the final tree-merge), and a range with only
3287	// stores ends with SliceValues[r] = its last stored value. Both are
3288	// correct.
3289	using SliceRange = std::pair<uint64_t, uint64_t>;
3290	SmallVector<SliceRange, `8`> SortedRanges;
3291	SortedRanges.reserve(N: Accesses.size());
3292	for (auto &Acc : Accesses)
3293	SortedRanges.emplace_back(Args&: Acc.BeginOffset, Args&: Acc.EndOffset);
3294	llvm::sort(C&: SortedRanges);
3295	SortedRanges.erase(CS: llvm::unique(R&: SortedRanges), CE: SortedRanges.end());
3296	// Disjoint + contiguous tile of the whole alloca.
3297	uint64_t Expected = NewAllocaBeginOffset;
3298	for (auto &Range : SortedRanges) {
3299	if (Range.first != Expected)
3300	return std::nullopt;
3301	Expected = Range.second;
3302	}
3303	if (Expected != NewAllocaEndOffset)
3304	return std::nullopt;
3305
3306	LLVM_DEBUG({
3307	dbgs() << "Tree structured merge rewrite (RMW):\n";
3308	dbgs() << " Init store: " << *InitStore << "\n";
3309	if (FullLoad)
3310	dbgs() << " Final load: " << *FullLoad << "\n";
3311	dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3312	for (auto &Range : SortedRanges)
3313	dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3314	});
3315
3316	// Initialize SliceValues: one SSA value per slice range, tracking
3317	// the value the alloca currently holds at that range. Each entry
3318	// starts at the corresponding piece of the init store, obtained by
3319	// bitcasting the init value to the alloca's vector type (if needed)
3320	// and extracting the slice's sub-range.
3321	IRB.SetInsertPoint(InitStore->getNextNode());
3322	Value *InitVec = InitStore->getValueOperand();
3323	if (InitVec->getType() != NewAllocaTy)
3324	InitVec = IRB.CreateBitCast(V: InitVec, DestTy: NewAllocaTy, Name: "init.cast");
3325	DenseMap<SliceRange, Value *> SliceValues;
3326	for (auto &Range : SortedRanges) {
3327	unsigned BeginIdx = getIndex(Offset: Range.first);
3328	unsigned EndIdx = getIndex(Offset: Range.second);
3329	SliceValues [Range] = IRB.CreateShuffleVector(
3330	V: InitVec, Mask: createSequentialMask(Start: BeginIdx, NumInts: EndIdx - BeginIdx, NumUndefs: `0`),
3331	Name: "init.extract");
3332	}
3333	// The init store itself becomes dead — its value is consumed via the
3334	// extracts above.
3335	DeletedValues.push_back(Elt: InitStore);
3336
3337	// Walk accesses in block order:
3338	// - partial load at range r: replace with SliceValues[r] (bitcast
3339	// if the load's type differs from the current tracked value's
3340	// type, e.g. because a previous store wrote a vector with a
3341	// different element type);
3342	// - partial store at range r: update SliceValues[r] to the stored
3343	// value and drop the store.
3344	for (auto &Acc : Accesses) {
3345	SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3346	if (!Acc.IsStore) {
3347	Value *V = SliceValues [Range];
3348	if (V->getType() != Acc.Inst->getType()) {
3349	IRB.SetInsertPoint(cast<LoadInst>(Val: Acc.Inst));
3350	V = IRB.CreateBitCast(V, DestTy: Acc.Inst->getType());
3351	}
3352	Acc.Inst->replaceAllUsesWith(V);
3353	} else {
3354	SliceValues [Range] = cast<StoreInst>(Val: Acc.Inst)->getValueOperand();
3355	}
3356	DeletedValues.push_back(Elt: Acc.Inst);
3357	}
3358
3359	// Tree-merge the final per-range values (in range order) into the
3360	// alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3361	// shares the partial-access block) or otherwise to the block's
3362	// terminator — never to a partial access, since those are queued for
3363	// deletion. Both anchors are guaranteed to dominate every SliceValues
3364	// entry: each one is either an init extract (before any access) or a
3365	// stored value defined before its (now-deleted) store.
3366	IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3367	? cast<Instruction>(Val: FullLoad)
3368	: StoreBB->getTerminator());
3369	SmallVector<Value *, `8`> Vals;
3370	for (auto &Range : SortedRanges)
3371	Vals.push_back(Elt: SliceValues [Range]);
3372	Value *Merged = TreeMerge(Vals, Builder);
3373	Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3374
3375	// Replace the optional final full-width load with a load of the newly
3376	// merged alloca. Later promotion will forward the store above to it.
3377	if (FullLoad)
3378	ReplaceFullLoad(FullLoad, Merged);
3379
3380	return DeletedValues;
3381	}
3382
3383	private:
3384	// Make sure the other visit overloads are visible.
3385	using Base::visit;
3386
3387	// Every instruction which can end up as a user must have a rewrite rule.
3388	bool visitInstruction(Instruction &I) {
3389	LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3390	llvm_unreachable("No rewrite rule for this instruction!");
3391	}
3392
3393	Value getNewAllocaSlicePtr(IRBuilderTy &IRB, Type PointerTy) {
3394	// Note that the offset computation can use BeginOffset or NewBeginOffset
3395	// interchangeably for unsplit slices.
3396	assert(IsSplit \|\| BeginOffset == NewBeginOffset);
3397	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3398
3399	StringRef OldName = OldPtr->getName();
3400	// Skip through the last '.sroa.' component of the name.
3401	size_t LastSROAPrefix = OldName.rfind(Str: ".sroa.");
3402	if (LastSROAPrefix != StringRef::npos) {
3403	OldName = OldName.substr(Start: LastSROAPrefix + strlen(s: ".sroa."));
3404	// Look for an SROA slice index.
3405	size_t IndexEnd = OldName.find_first_not_of(Chars: "0123456789");
3406	if (IndexEnd != StringRef::npos && OldName [IndexEnd] == `'.'`) {
3407	// Strip the index and look for the offset.
3408	OldName = OldName.substr(Start: IndexEnd + `1`);
3409	size_t OffsetEnd = OldName.find_first_not_of(Chars: "0123456789");
3410	if (OffsetEnd != StringRef::npos && OldName [OffsetEnd] == `'.'`)
3411	// Strip the offset.
3412	OldName = OldName.substr(Start: OffsetEnd + `1`);
3413	}
3414	}
3415	// Strip any SROA suffixes as well.
3416	OldName = OldName.substr(Start: `0`, N: OldName.find(Str: ".sroa_"));
3417
3418	return getAdjustedPtr(IRB, DL, Ptr: &NewAI,
3419	Offset: APInt (DL.getIndexTypeSizeInBits(Ty: PointerTy), Offset),
3420	PointerTy, NamePrefix: Twine (OldName) + ".");
3421	}
3422
3423	/// Compute suitable alignment to access this slice of the new
3424	/// alloca.
3425	///
3426	/// You can optionally pass a type to this routine and if that type's ABI
3427	/// alignment is itself suitable, this will return zero.
3428	Align getSliceAlign() {
3429	return commonAlignment(A: NewAI.getAlign(),
3430	Offset: NewBeginOffset - NewAllocaBeginOffset);
3431	}
3432
3433	unsigned getIndex(uint64_t Offset) {
3434	assert(VecTy && "Can only call getIndex when rewriting a vector");
3435	uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3436	assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3437	uint32_t Index = RelOffset / ElementSize;
3438	assert(Index * ElementSize == RelOffset);
3439	return Index;
3440	}
3441
3442	void deleteIfTriviallyDead(Value *V) {
3443	Instruction *I = cast<Instruction>(Val: V);
3444	if (isInstructionTriviallyDead(I))
3445	Pass.DeadInsts.push_back(Elt: I);
3446	}
3447
3448	Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3449	unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3450	unsigned EndIndex = getIndex(Offset: NewEndOffset);
3451	assert(EndIndex > BeginIndex && "Empty vector!");
3452
3453	LoadInst *Load =
3454	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3455
3456	Load->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3457	LLVMContext::MD_access_group});
3458	return extractVector(IRB, V: Load, BeginIndex, EndIndex, Name: "vec");
3459	}
3460
3461	Value *rewriteIntegerLoad(LoadInst &LI) {
3462	assert(IntTy && "We cannot insert an integer to the alloca");
3463	assert(!LI.isVolatile());
3464	Value *V =
3465	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3466	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: IntTy);
3467	assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3468	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3469	if (Offset > `0` \|\| NewEndOffset < NewAllocaEndOffset) {
3470	IntegerType ExtractTy = Type::getIntNTy(C&: LI.getContext(), N: SliceSize `8`);
3471	V = extractInteger(DL, IRB, V, Ty: ExtractTy, Offset, Name: "extract");
3472	}
3473	// It is possible that the extracted type is not the load type. This
3474	// happens if there is a load past the end of the alloca, and as
3475	// a consequence the slice is narrower but still a candidate for integer
3476	// lowering. To handle this case, we just zero extend the extracted
3477	// integer.
3478	assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * `8` &&
3479	"Can only handle an extract for an overly wide load");
3480	if (cast<IntegerType>(Val: LI.getType())->getBitWidth() > SliceSize * `8`)
3481	V = IRB.CreateZExt(V, DestTy: LI.getType());
3482	return V;
3483	}
3484
3485	bool visitLoadInst(LoadInst &LI) {
3486	LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3487	Value *OldOp = LI.getOperand(i_nocapture: `0`);
3488	assert(OldOp == OldPtr);
3489
3490	AAMDNodes AATags = LI.getAAMetadata();
3491
3492	unsigned AS = LI.getPointerAddressSpace();
3493
3494	Type TargetTy = IsSplit ? Type::getIntNTy(C&: LI.getContext(), N: SliceSize `8`)
3495	: LI.getType();
3496	bool IsPtrAdjusted = false;
3497	Value *V;
3498	if (VecTy) {
3499	V = rewriteVectorizedLoadInst(LI);
3500	} else if (IntTy && LI.getType()->isIntegerTy()) {
3501	V = rewriteIntegerLoad(LI);
3502	} else if (NewBeginOffset == NewAllocaBeginOffset &&
3503	NewEndOffset == NewAllocaEndOffset &&
3504	(canConvertValue(DL, OldTy: NewAllocaTy, NewTy: TargetTy) \|\|
3505	(NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3506	DL.getTypeStoreSize(Ty: TargetTy).getFixedValue() > SliceSize &&
3507	!LI.isVolatile()))) {
3508	Value *NewPtr =
3509	getPtrToNewAI(AddrSpace: LI.getPointerAddressSpace(), IsVolatile: LI.isVolatile());
3510	LoadInst *NewLI = IRB.CreateAlignedLoad(
3511	Ty: NewAllocaTy, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3512	if (LI.isVolatile())
3513	NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3514	if (NewLI->isAtomic())
3515	NewLI->setAlignment(LI.getAlign());
3516
3517	// Copy any metadata that is valid for the new load. This may require
3518	// conversion to a different kind of metadata, e.g. !nonnull might change
3519	// to !range or vice versa.
3520	copyMetadataForLoad(Dest&: *NewLI, Source: LI);
3521
3522	// Do this after copyMetadataForLoad() to preserve the TBAA shift.
3523	if (AATags)
3524	NewLI->setAAMetadata(AATags.adjustForAccess(
3525	Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3526
3527	// Try to preserve nonnull metadata
3528	V = NewLI;
3529
3530	// If this is an integer load past the end of the slice (which means the
3531	// bytes outside the slice are undef or this load is dead) just forcibly
3532	// fix the integer size with correct handling of endianness.
3533	if (auto *AITy = dyn_cast<IntegerType>(Val: NewAllocaTy))
3534	if (auto *TITy = dyn_cast<IntegerType>(Val: TargetTy))
3535	if (AITy->getBitWidth() < TITy->getBitWidth()) {
3536	V = IRB.CreateZExt(V, DestTy: TITy, Name: "load.ext");
3537	if (DL.isBigEndian())
3538	V = IRB.CreateShl(LHS: V, RHS: TITy->getBitWidth() - AITy->getBitWidth(),
3539	Name: "endian_shift");
3540	}
3541	} else {
3542	Type *LTy = IRB.getPtrTy(AddrSpace: AS);
3543	LoadInst *NewLI =
3544	IRB.CreateAlignedLoad(Ty: TargetTy, Ptr: getNewAllocaSlicePtr(IRB, PointerTy: LTy),
3545	Align: getSliceAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3546
3547	if (AATags)
3548	NewLI->setAAMetadata(AATags.adjustForAccess(
3549	Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3550
3551	if (LI.isVolatile())
3552	NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3553	NewLI->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3554	LLVMContext::MD_access_group});
3555
3556	V = NewLI;
3557	IsPtrAdjusted = true;
3558	}
3559	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: TargetTy);
3560
3561	if (IsSplit) {
3562	assert(!LI.isVolatile());
3563	assert(LI.getType()->isIntegerTy() &&
3564	"Only integer type loads and stores are split");
3565	assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3566	"Split load isn't smaller than original load");
3567	assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3568	"Non-byte-multiple bit width");
3569	// Move the insertion point just past the load so that we can refer to it.
3570	BasicBlock::iterator LIIt = std::next(x: LI.getIterator());
3571	// Ensure the insertion point comes before any debug-info immediately
3572	// after the load, so that variable values referring to the load are
3573	// dominated by it.
3574	LIIt.setHeadBit(true);
3575	IRB.SetInsertPoint(LIIt);
3576	// Create a placeholder value with the same type as LI to use as the
3577	// basis for the new value. This allows us to replace the uses of LI with
3578	// the computed value, and then replace the placeholder with LI, leaving
3579	// LI only used for this computation.
3580	Value *Placeholder =
3581	new LoadInst (LI.getType(), PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AS)), "",
3582	false, Align (`1`));
3583	V = insertInteger(DL, IRB, Old: Placeholder, V, Offset: NewBeginOffset - BeginOffset,
3584	Name: "insert");
3585	LI.replaceAllUsesWith(V);
3586	Placeholder->replaceAllUsesWith(V: &LI);
3587	Placeholder->deleteValue();
3588	} else {
3589	LI.replaceAllUsesWith(V);
3590	}
3591
3592	Pass.DeadInsts.push_back(Elt: &LI);
3593	deleteIfTriviallyDead(V: OldOp);
3594	LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3595	return !LI.isVolatile() && !IsPtrAdjusted;
3596	}
3597
3598	bool rewriteVectorizedStoreInst(Value V, StoreInst &SI, Value OldOp,
3599	AAMDNodes AATags) {
3600	// Capture V for the purpose of debug-info accounting once it's converted
3601	// to a vector store.
3602	Value *OrigV = V;
3603	if (V->getType() != VecTy) {
3604	unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3605	unsigned EndIndex = getIndex(Offset: NewEndOffset);
3606	assert(EndIndex > BeginIndex && "Empty vector!");
3607	unsigned NumElements = EndIndex - BeginIndex;
3608	assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3609	"Too many elements!");
3610	Type *SliceTy = (NumElements == `1`)
3611	? ElementTy
3612	: FixedVectorType::get(ElementType: ElementTy, NumElts: NumElements);
3613	if (V->getType() != SliceTy)
3614	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: SliceTy);
3615
3616	// Mix in the existing elements.
3617	Value *Old =
3618	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3619	V = insertVector(IRB, Old, V, BeginIndex, Name: "vec");
3620	}
3621	StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3622	Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3623	LLVMContext::MD_access_group});
3624	if (AATags)
3625	Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3626	AccessTy: V->getType(), DL));
3627	Pass.DeadInsts.push_back(Elt: &SI);
3628
3629	// NOTE: Careful to use OrigV rather than V.
3630	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &SI,
3631	Inst: Store, Dest: Store->getPointerOperand(), Value: OrigV, DL);
3632	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3633	return true;
3634	}
3635
3636	bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3637	assert(IntTy && "We cannot extract an integer from the alloca");
3638	assert(!SI.isVolatile());
3639	if (DL.getTypeSizeInBits(Ty: V->getType()).getFixedValue() !=
3640	IntTy->getBitWidth()) {
3641	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3642	Name: "oldload");
3643	Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3644	assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3645	uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3646	V = insertInteger(DL, IRB, Old, V: SI.getValueOperand(), Offset, Name: "insert");
3647	}
3648	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3649	StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3650	Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3651	LLVMContext::MD_access_group});
3652	if (AATags)
3653	Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3654	AccessTy: V->getType(), DL));
3655
3656	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &SI,
3657	Inst: Store, Dest: Store->getPointerOperand(),
3658	Value: Store->getValueOperand(), DL);
3659
3660	Pass.DeadInsts.push_back(Elt: &SI);
3661	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3662	return true;
3663	}
3664
3665	bool visitStoreInst(StoreInst &SI) {
3666	LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3667	Value *OldOp = SI.getOperand(i_nocapture: `1`);
3668	assert(OldOp == OldPtr);
3669
3670	AAMDNodes AATags = SI.getAAMetadata();
3671	Value *V = SI.getValueOperand();
3672
3673	// Strip all inbounds GEPs and pointer casts to try to dig out any root
3674	// alloca that should be re-examined after promoting this alloca.
3675	if (V->getType()->isPointerTy())
3676	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V->stripInBoundsOffsets()))
3677	Pass.PostPromotionWorklist.insert(X: AI);
3678
3679	TypeSize StoreSize = DL.getTypeStoreSize(Ty: V->getType());
3680	if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3681	assert(!SI.isVolatile());
3682	assert(V->getType()->isIntegerTy() &&
3683	"Only integer type loads and stores are split");
3684	assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3685	"Non-byte-multiple bit width");
3686	IntegerType NarrowTy = Type::getIntNTy(C&: SI.getContext(), N: SliceSize `8`);
3687	V = extractInteger(DL, IRB, V, Ty: NarrowTy, Offset: NewBeginOffset - BeginOffset,
3688	Name: "extract");
3689	}
3690
3691	if (VecTy)
3692	return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3693	if (IntTy && V->getType()->isIntegerTy())
3694	return rewriteIntegerStore(V, SI, AATags);
3695
3696	StoreInst *NewSI;
3697	if (NewBeginOffset == NewAllocaBeginOffset &&
3698	NewEndOffset == NewAllocaEndOffset &&
3699	canConvertValue(DL, OldTy: V->getType(), NewTy: NewAllocaTy)) {
3700	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3701	Value *NewPtr =
3702	getPtrToNewAI(AddrSpace: SI.getPointerAddressSpace(), IsVolatile: SI.isVolatile());
3703
3704	NewSI =
3705	IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: SI.isVolatile());
3706	} else {
3707	unsigned AS = SI.getPointerAddressSpace();
3708	Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: IRB.getPtrTy(AddrSpace: AS));
3709	NewSI =
3710	IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: getSliceAlign(), isVolatile: SI.isVolatile());
3711	}
3712	NewSI->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3713	LLVMContext::MD_access_group});
3714	if (AATags)
3715	NewSI->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3716	AccessTy: V->getType(), DL));
3717	if (SI.isVolatile())
3718	NewSI->setAtomic(Ordering: SI.getOrdering(), SSID: SI.getSyncScopeID());
3719	if (NewSI->isAtomic())
3720	NewSI->setAlignment(SI.getAlign());
3721
3722	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &SI,
3723	Inst: NewSI, Dest: NewSI->getPointerOperand(),
3724	Value: NewSI->getValueOperand(), DL);
3725
3726	Pass.DeadInsts.push_back(Elt: &SI);
3727	deleteIfTriviallyDead(V: OldOp);
3728
3729	LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3730	return NewSI->getPointerOperand() == &NewAI &&
3731	NewSI->getValueOperand()->getType() == NewAllocaTy &&
3732	!SI.isVolatile();
3733	}
3734
3735	/// Compute an integer value from splatting an i8 across the given
3736	/// number of bytes.
3737	///
3738	/// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3739	/// call this routine.
3740	/// FIXME: Heed the advice above.
3741	///
3742	/// \param V The i8 value to splat.
3743	/// \param Size The number of bytes in the output (assuming i8 is one byte)
3744	Value getIntegerSplat(Value V, unsigned Size) {
3745	assert(Size > `0` && "Expected a positive number of bytes.");
3746	IntegerType *VTy = cast<IntegerType>(Val: V->getType());
3747	assert(VTy->getBitWidth() == `8` && "Expected an i8 value for the byte");
3748	if (Size == `1`)
3749	return V;
3750
3751	Type SplatIntTy = Type::getIntNTy(C&: VTy->getContext(), N: Size `8`);
3752	V = IRB.CreateMul(
3753	LHS: IRB.CreateZExt(V, DestTy: SplatIntTy, Name: "zext"),
3754	RHS: IRB.CreateUDiv(LHS: Constant::getAllOnesValue(Ty: SplatIntTy),
3755	RHS: IRB.CreateZExt(V: Constant::getAllOnesValue(Ty: V->getType()),
3756	DestTy: SplatIntTy)),
3757	Name: "isplat");
3758	return V;
3759	}
3760
3761	/// Compute a vector splat for a given element value.
3762	Value getVectorSplat(Value V, unsigned NumElements) {
3763	V = IRB.CreateVectorSplat(NumElts: NumElements, V, Name: "vsplat");
3764	LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3765	return V;
3766	}
3767
3768	bool visitMemSetInst(MemSetInst &II) {
3769	LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3770	assert(II.getRawDest() == OldPtr);
3771
3772	AAMDNodes AATags = II.getAAMetadata();
3773
3774	// If the memset has a variable size, it cannot be split, just adjust the
3775	// pointer to the new alloca.
3776	if (!isa<ConstantInt>(Val: II.getLength())) {
3777	assert(!IsSplit);
3778	assert(NewBeginOffset == BeginOffset);
3779	II.setDest(getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()));
3780	II.setDestAlignment(getSliceAlign());
3781	// In theory we should call migrateDebugInfo here. However, we do not
3782	// emit dbg.assign intrinsics for mem intrinsics storing through non-
3783	// constant geps, or storing a variable number of bytes.
3784	assert(at::getDVRAssignmentMarkers(&II).empty() &&
3785	"AT: Unexpected link to non-const GEP");
3786	deleteIfTriviallyDead(V: OldPtr);
3787	return false;
3788	}
3789
3790	// Record this instruction for deletion.
3791	Pass.DeadInsts.push_back(Elt: &II);
3792
3793	Type *ScalarTy = NewAllocaTy->getScalarType();
3794
3795	const bool CanContinue = [&]() {
3796	if (VecTy \|\| IntTy)
3797	return true;
3798	if (BeginOffset > NewAllocaBeginOffset \|\| EndOffset < NewAllocaEndOffset)
3799	return false;
3800	// Length must be in range for FixedVectorType.
3801	auto *C = cast<ConstantInt>(Val: II.getLength());
3802	const uint64_t Len = C->getLimitedValue();
3803	if (Len > std::numeric_limits<unsigned>::max())
3804	return false;
3805	auto *Int8Ty = IntegerType::getInt8Ty(C&: NewAI.getContext());
3806	auto *SrcTy = FixedVectorType::get(ElementType: Int8Ty, NumElts: Len);
3807	return canConvertValue(DL, OldTy: SrcTy, NewTy: NewAllocaTy) &&
3808	DL.isLegalInteger(Width: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue());
3809	}();
3810
3811	// If this doesn't map cleanly onto the alloca type, and that type isn't
3812	// a single value type, just emit a memset.
3813	if (!CanContinue) {
3814	Type *SizeTy = II.getLength()->getType();
3815	unsigned Sz = NewEndOffset - NewBeginOffset;
3816	Constant *Size = ConstantInt::get(Ty: SizeTy, V: Sz);
3817	MemIntrinsic *New = cast<MemIntrinsic>(Val: IRB.CreateMemSet(
3818	Ptr: getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()), Val: II.getValue(), Size,
3819	Align: MaybeAlign (getSliceAlign()), isVolatile: II.isVolatile()));
3820	if (AATags)
3821	New->setAAMetadata(
3822	AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset, AccessSize: Sz));
3823
3824	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &II,
3825	Inst: New, Dest: New->getRawDest(), Value: nullptr, DL);
3826
3827	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3828	return false;
3829	}
3830
3831	// If we can represent this as a simple value, we have to build the actual
3832	// value to store, which requires expanding the byte present in memset to
3833	// a sensible representation for the alloca type. This is essentially
3834	// splatting the byte to a sufficiently wide integer, splatting it across
3835	// any desired vector width, and bitcasting to the final type.
3836	Value *V;
3837
3838	if (VecTy) {
3839	// If this is a memset of a vectorized alloca, insert it.
3840	assert(ElementTy == ScalarTy);
3841
3842	unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3843	unsigned EndIndex = getIndex(Offset: NewEndOffset);
3844	assert(EndIndex > BeginIndex && "Empty vector!");
3845	unsigned NumElements = EndIndex - BeginIndex;
3846	assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3847	"Too many elements!");
3848
3849	Value *Splat = getIntegerSplat(
3850	V: II.getValue(), Size: DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / `8`);
3851	Splat = IRB.CreateBitPreservingCastChain(DL, V: Splat, NewTy: ElementTy);
3852	if (NumElements > `1`)
3853	Splat = getVectorSplat(V: Splat, NumElements);
3854
3855	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3856	Name: "oldload");
3857	V = insertVector(IRB, Old, V: Splat, BeginIndex, Name: "vec");
3858	} else if (IntTy) {
3859	// If this is a memset on an alloca where we can widen stores, insert the
3860	// set integer.
3861	assert(!II.isVolatile());
3862
3863	uint64_t Size = NewEndOffset - NewBeginOffset;
3864	V = getIntegerSplat(V: II.getValue(), Size);
3865
3866	if (IntTy && (NewBeginOffset != NewAllocaBeginOffset \|\|
3867	NewEndOffset != NewAllocaEndOffset)) {
3868	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI,
3869	Align: NewAI.getAlign(), Name: "oldload");
3870	Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3871	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3872	V = insertInteger(DL, IRB, Old, V, Offset, Name: "insert");
3873	} else {
3874	assert(V->getType() == IntTy &&
3875	"Wrong type for an alloca wide integer!");
3876	}
3877	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3878	} else {
3879	// Established these invariants above.
3880	assert(NewBeginOffset == NewAllocaBeginOffset);
3881	assert(NewEndOffset == NewAllocaEndOffset);
3882
3883	V = getIntegerSplat(V: II.getValue(),
3884	Size: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue() / `8`);
3885	if (VectorType *AllocaVecTy = dyn_cast<VectorType>(Val: NewAllocaTy))
3886	V = getVectorSplat(
3887	V, NumElements: cast<FixedVectorType>(Val: AllocaVecTy)->getNumElements());
3888
3889	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3890	}
3891
3892	Value *NewPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3893	StoreInst *New =
3894	IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: II.isVolatile());
3895	New->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3896	LLVMContext::MD_access_group});
3897	if (AATags)
3898	New->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3899	AccessTy: V->getType(), DL));
3900
3901	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &II,
3902	Inst: New, Dest: New->getPointerOperand(), Value: V, DL);
3903
3904	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3905	return !II.isVolatile();
3906	}
3907
3908	bool visitMemTransferInst(MemTransferInst &II) {
3909	// Rewriting of memory transfer instructions can be a bit tricky. We break
3910	// them into two categories: split intrinsics and unsplit intrinsics.
3911
3912	LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3913
3914	AAMDNodes AATags = II.getAAMetadata();
3915
3916	bool IsDest = &II.getRawDestUse() == OldUse;
3917	assert((IsDest && II.getRawDest() == OldPtr) \|\|
3918	(!IsDest && II.getRawSource() == OldPtr));
3919
3920	Align SliceAlign = getSliceAlign();
3921	// For unsplit intrinsics, we simply modify the source and destination
3922	// pointers in place. This isn't just an optimization, it is a matter of
3923	// correctness. With unsplit intrinsics we may be dealing with transfers
3924	// within a single alloca before SROA ran, or with transfers that have
3925	// a variable length. We may also be dealing with memmove instead of
3926	// memcpy, and so simply updating the pointers is the necessary for us to
3927	// update both source and dest of a single call.
3928	if (!IsSplittable) {
3929	Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3930	if (IsDest) {
3931	// Update the address component of linked dbg.assigns.
3932	for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(Inst: &II)) {
3933	if (llvm::is_contained(Range: DbgAssign->location_ops(), Element: II.getDest()) \|\|
3934	DbgAssign->getAddress() == II.getDest())
3935	DbgAssign->replaceVariableLocationOp(OldValue: II.getDest(), NewValue: AdjustedPtr);
3936	}
3937	II.setDest(AdjustedPtr);
3938	II.setDestAlignment(SliceAlign);
3939	} else {
3940	II.setSource(AdjustedPtr);
3941	II.setSourceAlignment(SliceAlign);
3942	}
3943
3944	LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3945	deleteIfTriviallyDead(V: OldPtr);
3946	return false;
3947	}
3948	// For split transfer intrinsics we have an incredibly useful assurance:
3949	// the source and destination do not reside within the same alloca, and at
3950	// least one of them does not escape. This means that we can replace
3951	// memmove with memcpy, and we don't need to worry about all manner of
3952	// downsides to splitting and transforming the operations.
3953
3954	// If this doesn't map cleanly onto the alloca type, and that type isn't
3955	// a single value type, just emit a memcpy.
3956	bool EmitMemCpy =
3957	!VecTy && !IntTy &&
3958	(BeginOffset > NewAllocaBeginOffset \|\| EndOffset < NewAllocaEndOffset \|\|
3959	SliceSize != DL.getTypeStoreSize(Ty: NewAllocaTy).getFixedValue() \|\|
3960	!DL.typeSizeEqualsStoreSize(Ty: NewAllocaTy) \|\|
3961	!NewAllocaTy->isSingleValueType());
3962
3963	// If we're just going to emit a memcpy, the alloca hasn't changed, and the
3964	// size hasn't been shrunk based on analysis of the viable range, this is
3965	// a no-op.
3966	if (EmitMemCpy && &OldAI == &NewAI) {
3967	// Ensure the start lines up.
3968	assert(NewBeginOffset == BeginOffset);
3969
3970	// Rewrite the size as needed.
3971	if (NewEndOffset != EndOffset)
3972	II.setLength(NewEndOffset - NewBeginOffset);
3973	return false;
3974	}
3975	// Record this instruction for deletion.
3976	Pass.DeadInsts.push_back(Elt: &II);
3977
3978	// Strip all inbounds GEPs and pointer casts to try to dig out any root
3979	// alloca that should be re-examined after rewriting this instruction.
3980	Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
3981	if (AllocaInst *AI =
3982	dyn_cast<AllocaInst>(Val: OtherPtr->stripInBoundsOffsets())) {
3983	assert(AI != &OldAI && AI != &NewAI &&
3984	"Splittable transfers cannot reach the same alloca on both ends.");
3985	Pass.Worklist.insert(X: AI);
3986	}
3987
3988	Type *OtherPtrTy = OtherPtr->getType();
3989	unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
3990
3991	// Compute the relative offset for the other pointer within the transfer.
3992	unsigned OffsetWidth = DL.getIndexSizeInBits(AS: OtherAS);
3993	APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
3994	Align OtherAlign =
3995	(IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
3996	OtherAlign =
3997	commonAlignment(A: OtherAlign, Offset: OtherOffset.zextOrTrunc(width: `64`).getZExtValue());
3998
3999	if (EmitMemCpy) {
4000	// Compute the other pointer, folding as much as possible to produce
4001	// a single, simple GEP in most cases.
4002	OtherPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
4003	NamePrefix: OtherPtr->getName() + ".");
4004
4005	Value *OurPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4006	Type *SizeTy = II.getLength()->getType();
4007	Constant *Size = ConstantInt::get(Ty: SizeTy, V: NewEndOffset - NewBeginOffset);
4008
4009	Value DestPtr, SrcPtr;
4010	MaybeAlign DestAlign, SrcAlign;
4011	// Note: IsDest is true iff we're copying into the new alloca slice
4012	if (IsDest) {
4013	DestPtr = OurPtr;
4014	DestAlign = SliceAlign;
4015	SrcPtr = OtherPtr;
4016	SrcAlign = OtherAlign;
4017	} else {
4018	DestPtr = OtherPtr;
4019	DestAlign = OtherAlign;
4020	SrcPtr = OurPtr;
4021	SrcAlign = SliceAlign;
4022	}
4023	CallInst *New = IRB.CreateMemCpy(Dst: DestPtr, DstAlign: DestAlign, Src: SrcPtr, SrcAlign,
4024	Size, isVolatile: II.isVolatile());
4025	if (AATags)
4026	New->setAAMetadata(AATags.shift(Offset: NewBeginOffset - BeginOffset));
4027
4028	APInt Offset(DL.getIndexTypeSizeInBits(Ty: DestPtr->getType()), `0`);
4029	if (IsDest) {
4030	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`,
4031	OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
4032	} else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4033	Val: DestPtr->stripAndAccumulateConstantOffsets(
4034	DL, Offset, /AllowNonInbounds/ true))) {
4035	migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * `8`,
4036	SliceSizeInBits: SliceSize * `8`, OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
4037	}
4038	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4039	return false;
4040	}
4041
4042	bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
4043	NewEndOffset == NewAllocaEndOffset;
4044	uint64_t Size = NewEndOffset - NewBeginOffset;
4045	unsigned BeginIndex = VecTy ? getIndex(Offset: NewBeginOffset) : `0`;
4046	unsigned EndIndex = VecTy ? getIndex(Offset: NewEndOffset) : `0`;
4047	unsigned NumElements = EndIndex - BeginIndex;
4048	IntegerType *SubIntTy =
4049	IntTy ? Type::getIntNTy(C&: IntTy->getContext(), N: Size * `8`) : nullptr;
4050
4051	// Reset the other pointer type to match the register type we're going to
4052	// use, but using the address space of the original other pointer.
4053	Type *OtherTy;
4054	if (VecTy && !IsWholeAlloca) {
4055	if (NumElements == `1`)
4056	OtherTy = VecTy->getElementType();
4057	else
4058	OtherTy = FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: NumElements);
4059	} else if (IntTy && !IsWholeAlloca) {
4060	OtherTy = SubIntTy;
4061	} else {
4062	OtherTy = NewAllocaTy;
4063	}
4064
4065	Value *AdjPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
4066	NamePrefix: OtherPtr->getName() + ".");
4067	MaybeAlign SrcAlign = OtherAlign;
4068	MaybeAlign DstAlign = SliceAlign;
4069	if (!IsDest)
4070	std::swap(a&: SrcAlign, b&: DstAlign);
4071
4072	Value *SrcPtr;
4073	Value *DstPtr;
4074
4075	if (IsDest) {
4076	DstPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
4077	SrcPtr = AdjPtr;
4078	} else {
4079	DstPtr = AdjPtr;
4080	SrcPtr = getPtrToNewAI(AddrSpace: II.getSourceAddressSpace(), IsVolatile: II.isVolatile());
4081	}
4082
4083	Value *Src;
4084	if (VecTy && !IsWholeAlloca && !IsDest) {
4085	Src =
4086	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
4087	Src = extractVector(IRB, V: Src, BeginIndex, EndIndex, Name: "vec");
4088	} else if (IntTy && !IsWholeAlloca && !IsDest) {
4089	Src =
4090	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
4091	Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: IntTy);
4092	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4093	Src = extractInteger(DL, IRB, V: Src, Ty: SubIntTy, Offset, Name: "extract");
4094	} else {
4095	LoadInst *Load = IRB.CreateAlignedLoad(Ty: OtherTy, Ptr: SrcPtr, Align: SrcAlign,
4096	isVolatile: II.isVolatile(), Name: "copyload");
4097	Load->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4098	LLVMContext::MD_access_group});
4099	if (AATags)
4100	Load->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4101	AccessTy: Load->getType(), DL));
4102	Src = Load;
4103	}
4104
4105	if (VecTy && !IsWholeAlloca && IsDest) {
4106	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4107	Name: "oldload");
4108	Src = insertVector(IRB, Old, V: Src, BeginIndex, Name: "vec");
4109	} else if (IntTy && !IsWholeAlloca && IsDest) {
4110	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4111	Name: "oldload");
4112	Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
4113	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4114	Src = insertInteger(DL, IRB, Old, V: Src, Offset, Name: "insert");
4115	Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: NewAllocaTy);
4116	}
4117
4118	StoreInst *Store = cast<StoreInst>(
4119	Val: IRB.CreateAlignedStore(Val: Src, Ptr: DstPtr, Align: DstAlign, isVolatile: II.isVolatile()));
4120	Store->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4121	LLVMContext::MD_access_group});
4122	if (AATags)
4123	Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4124	AccessTy: Src->getType(), DL));
4125
4126	APInt Offset(DL.getIndexTypeSizeInBits(Ty: DstPtr->getType()), `0`);
4127	if (IsDest) {
4128
4129	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &II,
4130	Inst: Store, Dest: DstPtr, Value: Src, DL);
4131	} else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4132	Val: DstPtr->stripAndAccumulateConstantOffsets(
4133	DL, Offset, /AllowNonInbounds/ true))) {
4134	migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * `8`, SliceSizeInBits: SliceSize * `8`,
4135	OldInst: &II, Inst: Store, Dest: DstPtr, Value: Src, DL);
4136	}
4137
4138	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4139	return !II.isVolatile();
4140	}
4141
4142	bool visitIntrinsicInst(IntrinsicInst &II) {
4143	assert((II.isLifetimeStartOrEnd() \|\| II.isDroppable()) &&
4144	"Unexpected intrinsic!");
4145	LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4146
4147	// Record this instruction for deletion.
4148	Pass.DeadInsts.push_back(Elt: &II);
4149
4150	if (II.isDroppable()) {
4151	assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4152	// TODO For now we forget assumed information, this can be improved.
4153	OldPtr->dropDroppableUsesIn(Usr&: II);
4154	return true;
4155	}
4156
4157	assert(II.getArgOperand(`0`) == OldPtr);
4158	Type *PointerTy = IRB.getPtrTy(AddrSpace: OldPtr->getType()->getPointerAddressSpace());
4159	Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4160	Value *New;
4161	if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4162	New = IRB.CreateLifetimeStart(Ptr);
4163	else
4164	New = IRB.CreateLifetimeEnd(Ptr);
4165
4166	(void)New;
4167	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4168
4169	return true;
4170	}
4171
4172	void fixLoadStoreAlign(Instruction &Root) {
4173	// This algorithm implements the same visitor loop as
4174	// hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4175	// or store found.
4176	SmallPtrSet<Instruction *, `4`> Visited;
4177	SmallVector<Instruction *, `4`> Uses;
4178	Visited.insert(Ptr: &Root);
4179	Uses.push_back(Elt: &Root);
4180	do {
4181	Instruction *I = Uses.pop_back_val();
4182
4183	if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
4184	LI->setAlignment(std::min(a: LI->getAlign(), b: getSliceAlign()));
4185	continue;
4186	}
4187	if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
4188	SI->setAlignment(std::min(a: SI->getAlign(), b: getSliceAlign()));
4189	continue;
4190	}
4191
4192	assert(isa<BitCastInst>(I) \|\| isa<AddrSpaceCastInst>(I) \|\|
4193	isa<PHINode>(I) \|\| isa<SelectInst>(I) \|\|
4194	isa<GetElementPtrInst>(I));
4195	for (User *U : I->users())
4196	if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
4197	Uses.push_back(Elt: cast<Instruction>(Val: U));
4198	} while (!Uses.empty());
4199	}
4200
4201	bool visitPHINode(PHINode &PN) {
4202	LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4203	assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4204	assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4205
4206	// We would like to compute a new pointer in only one place, but have it be
4207	// as local as possible to the PHI. To do that, we re-use the location of
4208	// the old pointer, which necessarily must be in the right position to
4209	// dominate the PHI.
4210	IRBuilderBase::InsertPointGuard Guard(IRB);
4211	if (isa<PHINode>(Val: OldPtr))
4212	IRB.SetInsertPoint(OldPtr->getParent()->getFirstInsertionPt());
4213	else
4214	IRB.SetInsertPoint(OldPtr);
4215	IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4216
4217	Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4218	// Replace the operands which were using the old pointer.
4219	std::replace(first: PN.op_begin(), last: PN.op_end(), old_value: cast<Value>(Val: OldPtr), new_value: NewPtr);
4220
4221	LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4222	deleteIfTriviallyDead(V: OldPtr);
4223
4224	// Fix the alignment of any loads or stores using this PHI node.
4225	fixLoadStoreAlign(Root&: PN);
4226
4227	// PHIs can't be promoted on their own, but often can be speculated. We
4228	// check the speculation outside of the rewriter so that we see the
4229	// fully-rewritten alloca.
4230	PHIUsers.insert(X: &PN);
4231	return true;
4232	}
4233
4234	bool visitSelectInst(SelectInst &SI) {
4235	LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4236	assert((SI.getTrueValue() == OldPtr \|\| SI.getFalseValue() == OldPtr) &&
4237	"Pointer isn't an operand!");
4238	assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4239	assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4240
4241	Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4242	// Replace the operands which were using the old pointer.
4243	if (SI.getOperand(i_nocapture: `1`) == OldPtr)
4244	SI.setOperand(i_nocapture: `1`, Val_nocapture: NewPtr);
4245	if (SI.getOperand(i_nocapture: `2`) == OldPtr)
4246	SI.setOperand(i_nocapture: `2`, Val_nocapture: NewPtr);
4247
4248	LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4249	deleteIfTriviallyDead(V: OldPtr);
4250
4251	// Fix the alignment of any loads or stores using this select.
4252	fixLoadStoreAlign(Root&: SI);
4253
4254	// Selects can't be promoted on their own, but often can be speculated. We
4255	// check the speculation outside of the rewriter so that we see the
4256	// fully-rewritten alloca.
4257	SelectUsers.insert(X: &SI);
4258	return true;
4259	}
4260	};
4261
4262	/// Visitor to rewrite aggregate loads and stores as scalar.
4263	///
4264	/// This pass aggressively rewrites all aggregate loads and stores on
4265	/// a particular pointer (or any pointer derived from it which we can identify)
4266	/// with scalar loads and stores.
4267	class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4268	// Befriend the base class so it can delegate to private visit methods.
4269	friend class InstVisitor<AggLoadStoreRewriter, bool>;
4270
4271	/// Queue of pointer uses to analyze and potentially rewrite.
4272	SmallVector<Use *, `8`> Queue;
4273
4274	/// Set to prevent us from cycling with phi nodes and loops.
4275	SmallPtrSet<User *, `8`> Visited;
4276
4277	/// The current pointer use being rewritten. This is used to dig up the used
4278	/// value (as opposed to the user).
4279	Use U = nullptr*;
4280
4281	/// Used to calculate offsets, and hence alignment, of subobjects.
4282	const DataLayout &DL;
4283
4284	IRBuilderTy &IRB;
4285
4286	public:
4287	AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4288	: DL(DL), IRB(IRB) {}
4289
4290	/// Rewrite loads and stores through a pointer and all pointers derived from
4291	/// it.
4292	bool rewrite(Instruction &I) {
4293	LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4294	enqueueUsers(I);
4295	bool Changed = false;
4296	while (!Queue.empty()) {
4297	U = Queue.pop_back_val();
4298	Changed \|= visit(I: cast<Instruction>(Val: U->getUser()));
4299	}
4300	return Changed;
4301	}
4302
4303	private:
4304	/// Enqueue all the users of the given instruction for further processing.
4305	/// This uses a set to de-duplicate users.
4306	void enqueueUsers(Instruction &I) {
4307	for (Use &U : I.uses())
4308	if (Visited.insert(Ptr: U.getUser()).second)
4309	Queue.push_back(Elt: &U);
4310	}
4311
4312	// Conservative default is to not rewrite anything.
4313	bool visitInstruction(Instruction &I) { return false; }
4314
4315	/// Generic recursive split emission class.
4316	template <typename Derived> class OpSplitter {
4317	protected:
4318	/// The builder used to form new instructions.
4319	IRBuilderTy &IRB;
4320
4321	/// The indices which to be used with insert- or extractvalue to select the
4322	/// appropriate value within the aggregate.
4323	SmallVector<unsigned, `4`> Indices;
4324
4325	/// The indices to a GEP instruction which will move Ptr to the correct slot
4326	/// within the aggregate.
4327	SmallVector<Value *, `4`> GEPIndices;
4328
4329	/// The base pointer of the original op, used as a base for GEPing the
4330	/// split operations.
4331	Value *Ptr;
4332
4333	/// The base pointee type being GEPed into.
4334	Type *BaseTy;
4335
4336	/// Known alignment of the base pointer.
4337	Align BaseAlign;
4338
4339	/// To calculate offset of each component so we can correctly deduce
4340	/// alignments.
4341	const DataLayout &DL;
4342
4343	/// Initialize the splitter with an insertion point, Ptr and start with a
4344	/// single zero GEP index.
4345	OpSplitter(Instruction InsertionPoint, Value Ptr, Type *BaseTy,
4346	Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4347	: IRB(IRB), GEPIndices (`1`, IRB.getInt32(C: `0`)), Ptr(Ptr), BaseTy(BaseTy),
4348	BaseAlign (BaseAlign), DL(DL) {
4349	IRB.SetInsertPoint(InsertionPoint);
4350	}
4351
4352	public:
4353	/// Generic recursive split emission routine.
4354	///
4355	/// This method recursively splits an aggregate op (load or store) into
4356	/// scalar or vector ops. It splits recursively until it hits a single value
4357	/// and emits that single value operation via the template argument.
4358	///
4359	/// The logic of this routine relies on GEPs and insertvalue and
4360	/// extractvalue all operating with the same fundamental index list, merely
4361	/// formatted differently (GEPs need actual values).
4362	///
4363	/// \param Ty The type being split recursively into smaller ops.
4364	/// \param Agg The aggregate value being built up or stored, depending on
4365	/// whether this is splitting a load or a store respectively.
4366	void emitSplitOps(Type Ty, Value &Agg, const Twine &Name) {
4367	if (Ty->isSingleValueType()) {
4368	unsigned Offset = DL.getIndexedOffsetInType(ElemTy: BaseTy, Indices: GEPIndices);
4369	return static_cast<Derived >(this*)->emitFunc(
4370	Ty, Agg, commonAlignment(A: BaseAlign, Offset), Name);
4371	}
4372
4373	if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
4374	unsigned OldSize = Indices.size();
4375	(void)OldSize;
4376	for (unsigned Idx = `0`, Size = ATy->getNumElements(); Idx != Size;
4377	++Idx) {
4378	assert(Indices.size() == OldSize && "Did not return to the old size");
4379	Indices.push_back(Elt: Idx);
4380	GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4381	emitSplitOps(Ty: ATy->getElementType(), Agg, Name: Name + "." + Twine (Idx));
4382	GEPIndices.pop_back();
4383	Indices.pop_back();
4384	}
4385	return;
4386	}
4387
4388	if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4389	unsigned OldSize = Indices.size();
4390	(void)OldSize;
4391	for (unsigned Idx = `0`, Size = STy->getNumElements(); Idx != Size;
4392	++Idx) {
4393	assert(Indices.size() == OldSize && "Did not return to the old size");
4394	Indices.push_back(Elt: Idx);
4395	GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4396	emitSplitOps(Ty: STy->getElementType(N: Idx), Agg, Name: Name + "." + Twine (Idx));
4397	GEPIndices.pop_back();
4398	Indices.pop_back();
4399	}
4400	return;
4401	}
4402
4403	llvm_unreachable("Only arrays and structs are aggregate loadable types");
4404	}
4405	};
4406
4407	struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4408	AAMDNodes AATags;
4409	// A vector to hold the split components that we want to emit
4410	// separate fake uses for.
4411	SmallVector<Value *, `4`> Components;
4412	// A vector to hold all the fake uses of the struct that we are splitting.
4413	// Usually there should only be one, but we are handling the general case.
4414	SmallVector<Instruction *, `1`> FakeUses;
4415
4416	LoadOpSplitter(Instruction InsertionPoint, Value Ptr, Type *BaseTy,
4417	AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4418	IRBuilderTy &IRB)
4419	: OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4420	IRB),
4421	AATags (AATags) {}
4422
4423	/// Emit a leaf load of a single value. This is called at the leaves of the
4424	/// recursive emission to actually load values.
4425	void emitFunc(Type Ty, Value &Agg, Align Alignment, const Twine &Name) {
4426	assert(Ty->isSingleValueType());
4427	// Load the single value and insert it using the indices.
4428	Value *GEP =
4429	IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4430	LoadInst *Load =
4431	IRB.CreateAlignedLoad(Ty, Ptr: GEP, Align: Alignment, Name: Name + ".load");
4432
4433	APInt Offset(
4434	DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), `0`);
4435	if (AATags &&
4436	GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset))
4437	Load->setAAMetadata(
4438	AATags.adjustForAccess(Offset: Offset.getZExtValue(), AccessTy: Load->getType(), DL));
4439	// Record the load so we can generate a fake use for this aggregate
4440	// component.
4441	Components.push_back(Elt: Load);
4442
4443	Agg = IRB.CreateInsertValue(Agg, Val: Load, Idxs: Indices, Name: Name + ".insert");
4444	LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4445	}
4446
4447	// Stash the fake uses that use the value generated by this instruction.
4448	void recordFakeUses(LoadInst &LI) {
4449	for (Use &U : LI.uses())
4450	if (auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser()))
4451	if (II->getIntrinsicID() == Intrinsic::fake_use)
4452	FakeUses.push_back(Elt: II);
4453	}
4454
4455	// Replace all fake uses of the aggregate with a series of fake uses, one
4456	// for each split component.
4457	void emitFakeUses() {
4458	for (Instruction *I : FakeUses) {
4459	IRB.SetInsertPoint(I);
4460	for (auto *V : Components)
4461	IRB.CreateIntrinsic(ID: Intrinsic::fake_use, Args: {V});
4462	I->eraseFromParent();
4463	}
4464	}
4465	};
4466
4467	bool visitLoadInst(LoadInst &LI) {
4468	assert(LI.getPointerOperand() == *U);
4469	if (!LI.isSimple() \|\| LI.getType()->isSingleValueType())
4470	return false;
4471
4472	// We have an aggregate being loaded, split it apart.
4473	LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4474	LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4475	getAdjustedAlignment(I: &LI, Offset: `0`), DL, IRB);
4476	Splitter.recordFakeUses(LI);
4477	Value *V = PoisonValue::get(T: LI.getType());
4478	Splitter.emitSplitOps(Ty: LI.getType(), Agg&: V, Name: LI.getName() + ".fca");
4479	Splitter.emitFakeUses();
4480	Visited.erase(Ptr: &LI);
4481	LI.replaceAllUsesWith(V);
4482	LI.eraseFromParent();
4483	return true;
4484	}
4485
4486	struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4487	StoreOpSplitter(Instruction InsertionPoint, Value Ptr, Type *BaseTy,
4488	AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4489	const DataLayout &DL, IRBuilderTy &IRB)
4490	: OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4491	DL, IRB),
4492	AATags (AATags), AggStore(AggStore) {}
4493	AAMDNodes AATags;
4494	StoreInst *AggStore;
4495	/// Emit a leaf store of a single value. This is called at the leaves of the
4496	/// recursive emission to actually produce stores.
4497	void emitFunc(Type Ty, Value &Agg, Align Alignment, const Twine &Name) {
4498	assert(Ty->isSingleValueType());
4499	// Extract the single value and store it using the indices.
4500	//
4501	// The gep and extractvalue values are factored out of the CreateStore
4502	// call to make the output independent of the argument evaluation order.
4503	Value *ExtractValue =
4504	IRB.CreateExtractValue(Agg, Idxs: Indices, Name: Name + ".extract");
4505	Value *InBoundsGEP =
4506	IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4507	StoreInst *Store =
4508	IRB.CreateAlignedStore(Val: ExtractValue, Ptr: InBoundsGEP, Align: Alignment);
4509
4510	APInt Offset(
4511	DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), `0`);
4512	GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset);
4513	if (AATags) {
4514	Store->setAAMetadata(AATags.adjustForAccess(
4515	Offset: Offset.getZExtValue(), AccessTy: ExtractValue->getType(), DL));
4516	}
4517
4518	// migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4519	// If we cannot (because there's an intervening non-const or unbounded
4520	// gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4521	// this instruction.
4522	Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets();
4523	if (auto *OldAI = dyn_cast<AllocaInst>(Val: Base)) {
4524	uint64_t SizeInBits =
4525	DL.getTypeSizeInBits(Ty: Store->getValueOperand()->getType());
4526	migrateDebugInfo(OldAlloca: OldAI, /IsSplit/ true, OldAllocaOffsetInBits: Offset.getZExtValue() * `8`,
4527	SliceSizeInBits: SizeInBits, OldInst: AggStore, Inst: Store,
4528	Dest: Store->getPointerOperand(), Value: Store->getValueOperand(),
4529	DL);
4530	} else {
4531	assert(at::getDVRAssignmentMarkers(Store).empty() &&
4532	"AT: unexpected debug.assign linked to store through "
4533	"unbounded GEP");
4534	}
4535	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4536	}
4537	};
4538
4539	bool visitStoreInst(StoreInst &SI) {
4540	if (!SI.isSimple() \|\| SI.getPointerOperand() != *U)
4541	return false;
4542	Value *V = SI.getValueOperand();
4543	if (V->getType()->isSingleValueType())
4544	return false;
4545
4546	// We have an aggregate being stored, split it apart.
4547	LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4548	StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4549	getAdjustedAlignment(I: &SI, Offset: `0`), DL, IRB);
4550	Splitter.emitSplitOps(Ty: V->getType(), Agg&: V, Name: V->getName() + ".fca");
4551	Visited.erase(Ptr: &SI);
4552	// The stores replacing SI each have markers describing fragments of the
4553	// assignment so delete the assignment markers linked to SI.
4554	at::deleteAssignmentMarkers(Inst: &SI);
4555	SI.eraseFromParent();
4556	return true;
4557	}
4558
4559	bool visitBitCastInst(BitCastInst &BC) {
4560	enqueueUsers(I&: BC);
4561	return false;
4562	}
4563
4564	bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4565	enqueueUsers(I&: ASC);
4566	return false;
4567	}
4568
4569	// Unfold gep (select cond, ptr1, ptr2), idx
4570	// => select cond, gep(ptr1, idx), gep(ptr2, idx)
4571	// and gep ptr, (select cond, idx1, idx2)
4572	// => select cond, gep(ptr, idx1), gep(ptr, idx2)
4573	// We also allow for i1 zext indices, which are equivalent to selects.
4574	bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4575	// Check whether the GEP has exactly one select operand and all indices
4576	// will become constant after the transform.
4577	Instruction *Sel = dyn_cast<SelectInst>(Val: GEPI.getPointerOperand());
4578	for (Value *Op : GEPI.indices()) {
4579	if (auto *SI = dyn_cast<SelectInst>(Val: Op)) {
4580	if (Sel)
4581	return false;
4582
4583	Sel = SI;
4584	if (!isa<ConstantInt>(Val: SI->getTrueValue()) \|\|
4585	!isa<ConstantInt>(Val: SI->getFalseValue()))
4586	return false;
4587	continue;
4588	}
4589	if (auto *ZI = dyn_cast<ZExtInst>(Val: Op)) {
4590	if (Sel)
4591	return false;
4592	Sel = ZI;
4593	if (!ZI->getSrcTy()->isIntegerTy(BitWidth: `1`))
4594	return false;
4595	continue;
4596	}
4597
4598	if (!isa<ConstantInt>(Val: Op))
4599	return false;
4600	}
4601
4602	if (!Sel)
4603	return false;
4604
4605	LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4606	dbgs() << " original: " << *Sel << "\n";
4607	dbgs() << " " << GEPI << "\n";);
4608
4609	auto GetNewOps = [&](Value *SelOp) {
4610	SmallVector<Value *> NewOps;
4611	for (Value *Op : GEPI.operands())
4612	if (Op == Sel)
4613	NewOps.push_back(Elt: SelOp);
4614	else
4615	NewOps.push_back(Elt: Op);
4616	return NewOps;
4617	};
4618
4619	Value Cond, True, *False;
4620	Instruction MDFrom = nullptr*;
4621	if (auto *SI = dyn_cast<SelectInst>(Val: Sel)) {
4622	Cond = SI->getCondition();
4623	True = SI->getTrueValue();
4624	False = SI->getFalseValue();
4625	MDFrom = SI;
4626	} else {
4627	Cond = Sel->getOperand(i: `0`);
4628	True = ConstantInt::get(Ty: Sel->getType(), V: `1`);
4629	False = ConstantInt::get(Ty: Sel->getType(), V: `0`);
4630	}
4631	SmallVector<Value *> TrueOps = GetNewOps(True);
4632	SmallVector<Value *> FalseOps = GetNewOps(False);
4633
4634	IRB.SetInsertPoint(&GEPI);
4635	GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4636
4637	Type *Ty = GEPI.getSourceElementType();
4638	Value *NTrue = IRB.CreateGEP(Ty, Ptr: TrueOps [`0`], IdxList: ArrayRef(TrueOps).drop_front(),
4639	Name: True->getName() + ".sroa.gep", NW);
4640
4641	Value *NFalse =
4642	IRB.CreateGEP(Ty, Ptr: FalseOps [`0`], IdxList: ArrayRef(FalseOps).drop_front(),
4643	Name: False->getName() + ".sroa.gep", NW);
4644
4645	Value *NSel = MDFrom
4646	? IRB.CreateSelect(C: Cond, True: NTrue, False: NFalse,
4647	Name: Sel->getName() + ".sroa.sel", MDFrom)
4648	: IRB.CreateSelectWithUnknownProfile(
4649	C: Cond, True: NTrue, False: NFalse, DEBUG_TYPE,
4650	Name: Sel->getName() + ".sroa.sel");
4651	Visited.erase(Ptr: &GEPI);
4652	GEPI.replaceAllUsesWith(V: NSel);
4653	GEPI.eraseFromParent();
4654	Instruction *NSelI = cast<Instruction>(Val: NSel);
4655	Visited.insert(Ptr: NSelI);
4656	enqueueUsers(I&: *NSelI);
4657
4658	LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4659	dbgs() << " " << *NFalse << "\n";
4660	dbgs() << " " << *NSel << "\n";);
4661
4662	return true;
4663	}
4664
4665	// Unfold gep (phi ptr1, ptr2), idx
4666	// => phi ((gep ptr1, idx), (gep ptr2, idx))
4667	// and gep ptr, (phi idx1, idx2)
4668	// => phi ((gep ptr, idx1), (gep ptr, idx2))
4669	bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4670	// To prevent infinitely expanding recursive phis, bail if the GEP pointer
4671	// operand (looking through the phi if it is the phi we want to unfold) is
4672	// an instruction besides a static alloca.
4673	PHINode *Phi = dyn_cast<PHINode>(Val: GEPI.getPointerOperand());
4674	auto IsInvalidPointerOperand = [](Value *V) {
4675	if (!isa<Instruction>(Val: V))
4676	return false;
4677	if (auto *AI = dyn_cast<AllocaInst>(Val: V))
4678	return !AI->isStaticAlloca();
4679	return true;
4680	};
4681	if (Phi) {
4682	if (any_of(Range: Phi->operands(), P: IsInvalidPointerOperand))
4683	return false;
4684	} else {
4685	if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4686	return false;
4687	}
4688	// Check whether the GEP has exactly one phi operand (including the pointer
4689	// operand) and all indices will become constant after the transform.
4690	for (Value *Op : GEPI.indices()) {
4691	if (auto *SI = dyn_cast<PHINode>(Val: Op)) {
4692	if (Phi)
4693	return false;
4694
4695	Phi = SI;
4696	if (!all_of(Range: Phi->incoming_values(),
4697	P: [](Value V) { return* isa<ConstantInt>(Val: V); }))
4698	return false;
4699	continue;
4700	}
4701
4702	if (!isa<ConstantInt>(Val: Op))
4703	return false;
4704	}
4705
4706	if (!Phi)
4707	return false;
4708
4709	LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4710	dbgs() << " original: " << *Phi << "\n";
4711	dbgs() << " " << GEPI << "\n";);
4712
4713	auto GetNewOps = [&](Value *PhiOp) {
4714	SmallVector<Value *> NewOps;
4715	for (Value *Op : GEPI.operands())
4716	if (Op == Phi)
4717	NewOps.push_back(Elt: PhiOp);
4718	else
4719	NewOps.push_back(Elt: Op);
4720	return NewOps;
4721	};
4722
4723	IRB.SetInsertPoint(Phi);
4724	PHINode *NewPhi = IRB.CreatePHI(Ty: GEPI.getType(), NumReservedValues: Phi->getNumIncomingValues(),
4725	Name: Phi->getName() + ".sroa.phi");
4726
4727	Type *SourceTy = GEPI.getSourceElementType();
4728	// We only handle arguments, constants, and static allocas here, so we can
4729	// insert GEPs at the end of the entry block.
4730	IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4731	for (unsigned I = `0`, E = Phi->getNumIncomingValues(); I != E; ++I) {
4732	Value *Op = Phi->getIncomingValue(i: I);
4733	BasicBlock *BB = Phi->getIncomingBlock(i: I);
4734	Value *NewGEP;
4735	if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= `0`) {
4736	NewGEP = NewPhi->getIncomingValue(i: NI);
4737	} else {
4738	SmallVector<Value *> NewOps = GetNewOps(Op);
4739	NewGEP =
4740	IRB.CreateGEP(Ty: SourceTy, Ptr: NewOps [`0`], IdxList: ArrayRef(NewOps).drop_front(),
4741	Name: Phi->getName() + ".sroa.gep", NW: GEPI.getNoWrapFlags());
4742	}
4743	NewPhi->addIncoming(V: NewGEP, BB);
4744	}
4745
4746	Visited.erase(Ptr: &GEPI);
4747	GEPI.replaceAllUsesWith(V: NewPhi);
4748	GEPI.eraseFromParent();
4749	Visited.insert(Ptr: NewPhi);
4750	enqueueUsers(I&: *NewPhi);
4751
4752	LLVM_DEBUG(dbgs() << " to: ";
4753	for (Value *In
4754	: NewPhi->incoming_values()) dbgs()
4755	<< "\n " << *In;
4756	dbgs() << "\n " << *NewPhi << `'\n'`);
4757
4758	return true;
4759	}
4760
4761	bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4762	if (unfoldGEPSelect(GEPI))
4763	return true;
4764
4765	if (unfoldGEPPhi(GEPI))
4766	return true;
4767
4768	enqueueUsers(I&: GEPI);
4769	return false;
4770	}
4771
4772	bool visitPHINode(PHINode &PN) {
4773	enqueueUsers(I&: PN);
4774	return false;
4775	}
4776
4777	bool visitSelectInst(SelectInst &SI) {
4778	enqueueUsers(I&: SI);
4779	return false;
4780	}
4781	};
4782
4783	} // end anonymous namespace
4784
4785	/// Strip aggregate type wrapping.
4786	///
4787	/// This removes no-op aggregate types wrapping an underlying type. It will
4788	/// strip as many layers of types as it can without changing either the type
4789	/// size or the allocated size.
4790	static Type stripAggregateTypeWrapping(const* DataLayout &DL, Type *Ty) {
4791	if (Ty->isSingleValueType())
4792	return Ty;
4793
4794	uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4795	uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4796
4797	Type *InnerTy;
4798	if (ArrayType *ArrTy = dyn_cast<ArrayType>(Val: Ty)) {
4799	InnerTy = ArrTy->getElementType();
4800	} else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4801	const StructLayout *SL = DL.getStructLayout(Ty: STy);
4802	unsigned Index = SL->getElementContainingOffset(FixedOffset: `0`);
4803	InnerTy = STy->getElementType(N: Index);
4804	} else {
4805	return Ty;
4806	}
4807
4808	if (AllocSize > DL.getTypeAllocSize(Ty: InnerTy).getFixedValue() \|\|
4809	TypeSize > DL.getTypeSizeInBits(Ty: InnerTy).getFixedValue())
4810	return Ty;
4811
4812	return stripAggregateTypeWrapping(DL, Ty: InnerTy);
4813	}
4814
4815	/// Try to find a partition of the aggregate type passed in for a given
4816	/// offset and size.
4817	///
4818	/// This recurses through the aggregate type and tries to compute a subtype
4819	/// based on the offset and size. When the offset and size span a sub-section
4820	/// of an array, it will even compute a new array type for that sub-section,
4821	/// and the same for structs.
4822	///
4823	/// Note that this routine is very strict and tries to find a partition of the
4824	/// type which produces the exact* right offset and size. It is not forgiving*
4825	/// when the size or offset cause either end of type-based partition to be off.
4826	/// Also, this is a best-effort routine. It is reasonable to give up and not
4827	/// return a type if necessary.
4828	static Type getTypePartition(const* DataLayout &DL, Type *Ty, uint64_t Offset,
4829	uint64_t Size) {
4830	if (Offset == `0` && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4831	return stripAggregateTypeWrapping(DL, Ty);
4832	if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() \|\|
4833	(DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4834	return nullptr;
4835
4836	if (isa<ArrayType>(Val: Ty) \|\| isa<VectorType>(Val: Ty)) {
4837	Type *ElementTy;
4838	uint64_t TyNumElements;
4839	if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
4840	ElementTy = AT->getElementType();
4841	TyNumElements = AT->getNumElements();
4842	} else {
4843	// FIXME: This isn't right for vectors with non-byte-sized or
4844	// non-power-of-two sized elements.
4845	auto *VT = cast<FixedVectorType>(Val: Ty);
4846	ElementTy = VT->getElementType();
4847	TyNumElements = VT->getNumElements();
4848	}
4849	uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4850	uint64_t NumSkippedElements = Offset / ElementSize;
4851	if (NumSkippedElements >= TyNumElements)
4852	return nullptr;
4853	Offset -= NumSkippedElements * ElementSize;
4854
4855	// First check if we need to recurse.
4856	if (Offset > `0` \|\| Size < ElementSize) {
4857	// Bail if the partition ends in a different array element.
4858	if ((Offset + Size) > ElementSize)
4859	return nullptr;
4860	// Recurse through the element type trying to peel off offset bytes.
4861	return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4862	}
4863	assert(Offset == `0`);
4864
4865	if (Size == ElementSize)
4866	return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4867	assert(Size > ElementSize);
4868	uint64_t NumElements = Size / ElementSize;
4869	if (NumElements * ElementSize != Size)
4870	return nullptr;
4871	return ArrayType::get(ElementType: ElementTy, NumElements);
4872	}
4873
4874	StructType *STy = dyn_cast<StructType>(Val: Ty);
4875	if (!STy)
4876	return nullptr;
4877
4878	const StructLayout *SL = DL.getStructLayout(Ty: STy);
4879
4880	if (SL->getSizeInBits().isScalable())
4881	return nullptr;
4882
4883	if (Offset >= SL->getSizeInBytes())
4884	return nullptr;
4885	uint64_t EndOffset = Offset + Size;
4886	if (EndOffset > SL->getSizeInBytes())
4887	return nullptr;
4888
4889	unsigned Index = SL->getElementContainingOffset(FixedOffset: Offset);
4890	Offset -= SL->getElementOffset(Idx: Index);
4891
4892	Type *ElementTy = STy->getElementType(N: Index);
4893	uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4894	if (Offset >= ElementSize)
4895	return nullptr; // The offset points into alignment padding.
4896
4897	// See if any partition must be contained by the element.
4898	if (Offset > `0` \|\| Size < ElementSize) {
4899	if ((Offset + Size) > ElementSize)
4900	return nullptr;
4901	return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4902	}
4903	assert(Offset == `0`);
4904
4905	if (Size == ElementSize)
4906	return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4907
4908	StructType::element_iterator EI = STy->element_begin() + Index,
4909	EE = STy->element_end();
4910	if (EndOffset < SL->getSizeInBytes()) {
4911	unsigned EndIndex = SL->getElementContainingOffset(FixedOffset: EndOffset);
4912	if (Index == EndIndex)
4913	return nullptr; // Within a single element and its padding.
4914
4915	// Don't try to form "natural" types if the elements don't line up with the
4916	// expected size.
4917	// FIXME: We could potentially recurse down through the last element in the
4918	// sub-struct to find a natural end point.
4919	if (SL->getElementOffset(Idx: EndIndex) != EndOffset)
4920	return nullptr;
4921
4922	assert(Index < EndIndex);
4923	EE = STy->element_begin() + EndIndex;
4924	}
4925
4926	// Try to build up a sub-structure.
4927	StructType *SubTy =
4928	StructType::get(Context&: STy->getContext(), Elements: ArrayRef(EI, EE), isPacked: STy->isPacked());
4929	const StructLayout *SubSL = DL.getStructLayout(Ty: SubTy);
4930	if (Size != SubSL->getSizeInBytes())
4931	return nullptr; // The sub-struct doesn't have quite the size needed.
4932
4933	return SubTy;
4934	}
4935
4936	/// Try to find a slice in the map that partially overlaps with S, i.e. some but
4937	/// not all of its range is contained within the range of S, and which forms a
4938	/// load/store pair with S (i.e. the two slices are used to perform a
4939	/// memmove-like copy). SliceMap is expected to contain the slices for a single
4940	/// Partition.
4941	static Slice *findOverlappingCopySlice(Slice &S,
4942	InstructionSliceMap &SliceMap) {
4943	// Single byte slices can't partially overlap anything
4944	if (S.endOffset() - S.beginOffset() == `1`)
4945	return nullptr;
4946	// The source/destination of this slice needs to be a memory instruction
4947	// whose slice overlaps this one.
4948	Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
4949	Instruction J = nullptr*;
4950	if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
4951	if (!LI->hasOneUser())
4952	return nullptr;
4953	J = cast<Instruction>(Val: *LI->user_begin());
4954	} else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4955	J = dyn_cast<Instruction>(Val: SI->getValueOperand());
4956	if (!J \|\| !J->hasOneUser())
4957	return nullptr;
4958	} else {
4959	return nullptr;
4960	}
4961	// Check if there's a slice that corresponds to J that overlaps this slice
4962	for (Slice *JS : SliceMap [J]) {
4963	if (S.beginOffset() > JS->beginOffset() &&
4964	S.beginOffset() < JS->endOffset())
4965	return JS;
4966	if (JS->beginOffset() > S.beginOffset() &&
4967	JS->beginOffset() < S.endOffset())
4968	return JS;
4969	}
4970	return nullptr;
4971	}
4972
4973	/// Pre-split loads and stores to simplify rewriting.
4974	///
4975	/// We want to break up the splittable load+store pairs as much as
4976	/// possible. This is important to do as a preprocessing step, as once we
4977	/// start rewriting the accesses to partitions of the alloca we lose the
4978	/// necessary information to correctly split apart paired loads and stores
4979	/// which both point into this alloca. The case to consider is something like
4980	/// the following:
4981	///
4982	/// %a = alloca [12 x i8]
4983	/// %gep1 = getelementptr i8, ptr %a, i32 0
4984	/// %gep2 = getelementptr i8, ptr %a, i32 4
4985	/// %gep3 = getelementptr i8, ptr %a, i32 8
4986	/// store float 0.0, ptr %gep1
4987	/// store float 1.0, ptr %gep2
4988	/// %v = load i64, ptr %gep1
4989	/// store i64 %v, ptr %gep2
4990	/// %f1 = load float, ptr %gep2
4991	/// %f2 = load float, ptr %gep3
4992	///
4993	/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
4994	/// promote everything so we recover the 2 SSA values that should have been
4995	/// there all along.
4996	///
4997	/// \returns true if any changes are made.
4998	bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
4999	LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
5000
5001	// Track the loads and stores which are candidates for pre-splitting here, in
5002	// the order they first appear during the partition scan. These give stable
5003	// iteration order and a basis for tracking which loads and stores we
5004	// actually split.
5005	SmallVector<LoadInst *, `4`> Loads;
5006	SmallVector<StoreInst *, `4`> Stores;
5007
5008	// We need to accumulate the splits required of each load or store where we
5009	// can find them via a direct lookup. This is important to cross-check loads
5010	// and stores against each other. We also track the slice so that we can kill
5011	// all the slices that end up split.
5012	struct SplitOffsets {
5013	Slice *S;
5014	std::vector<uint64_t> Splits;
5015	};
5016	SmallDenseMap<Instruction *, SplitOffsets, `8`> SplitOffsetsMap;
5017
5018	// Track loads out of this alloca which cannot, for any reason, be pre-split.
5019	// This is important as we also cannot pre-split stores of those loads!
5020	// FIXME: This is all pretty gross. It means that we can be more aggressive
5021	// in pre-splitting when the load feeding the store happens to come from
5022	// a separate alloca. Put another way, the effectiveness of SROA would be
5023	// decreased by a frontend which just concatenated all of its local allocas
5024	// into one big flat alloca. But defeating such patterns is exactly the job
5025	// SROA is tasked with! Sadly, to not have this discrepancy we would have
5026	// change store pre-splitting to actually force pre-splitting of the load
5027	// that feeds it and all stores. That makes pre-splitting much harder, but
5028	// maybe it would make it more principled?
5029	SmallPtrSet<LoadInst *, `8`> UnsplittableLoads;
5030
5031	LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
5032	for (auto &P : AS.partitions()) {
5033	InstructionSliceMap SliceMap;
5034	for (Slice &S : P) {
5035	SliceMap [cast<Instruction>(Val: S.getUse()->getUser())].insert(Ptr: &S);
5036	}
5037	for (Slice &S : P) {
5038	Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
5039	bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
5040	Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
5041	if (!S.isSplittable() \|\| (!ExtendsPastPartitionEnd && !CopyOverlap)) {
5042	// If this is a load we have to track that it can't participate in any
5043	// pre-splitting. If this is a store of a load we have to track that
5044	// that load also can't participate in any pre-splitting.
5045	if (auto *LI = dyn_cast<LoadInst>(Val: I))
5046	UnsplittableLoads.insert(Ptr: LI);
5047	else if (auto *SI = dyn_cast<StoreInst>(Val: I))
5048	if (auto *LI = dyn_cast<LoadInst>(Val: SI->getValueOperand()))
5049	UnsplittableLoads.insert(Ptr: LI);
5050	continue;
5051	}
5052	assert(P.endOffset() > S.beginOffset() &&
5053	"Empty or backwards partition!");
5054
5055	// Determine if this is a pre-splittable slice.
5056	if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5057	assert(!LI->isVolatile() && "Cannot split volatile loads!");
5058
5059	// The load must be used exclusively to store into other pointers for
5060	// us to be able to arbitrarily pre-split it. The stores must also be
5061	// simple to avoid changing semantics.
5062	auto IsLoadSimplyStored = [](LoadInst *LI) {
5063	for (User *LU : LI->users()) {
5064	auto *SI = dyn_cast<StoreInst>(Val: LU);
5065	if (!SI \|\| !SI->isSimple())
5066	return false;
5067	}
5068	return true;
5069	};
5070	if (!IsLoadSimplyStored (LI)) {
5071	UnsplittableLoads.insert(Ptr: LI);
5072	continue;
5073	}
5074
5075	Loads.push_back(Elt: LI);
5076	} else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5077	if (S.getUse() != &SI->getOperandUse(i: SI->getPointerOperandIndex()))
5078	// Skip stores of* pointers. FIXME: This shouldn't even be possible!*
5079	continue;
5080	auto *StoredLoad = dyn_cast<LoadInst>(Val: SI->getValueOperand());
5081	if (!StoredLoad \|\| !StoredLoad->isSimple())
5082	continue;
5083	assert(!SI->isVolatile() && "Cannot split volatile stores!");
5084
5085	Stores.push_back(Elt: SI);
5086	} else {
5087	// Other uses cannot be pre-split.
5088	continue;
5089	}
5090
5091	// Record the initial split.
5092	LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
5093	auto &Offsets = SplitOffsetsMap [I];
5094	assert(Offsets.Splits.empty() &&
5095	"Should not have splits the first time we see an instruction!");
5096	Offsets.S = &S;
5097	if (CopyOverlap) {
5098	// S is being moved to CopyOverlap, which overlaps with S, so we split S
5099	// into three parts:
5100	// Initially outside the overlap and copied into the overlap.*
5101	// Either initially inside the overlap and remains inside it after*
5102	// the copy, or initially outside and remains outside, depending on
5103	// if the overlap or non-overlap area is larger, and which will be
5104	// empty if they are equal.
5105	// Initially inside the overlap and copied outside the overlap.*
5106	// This should result in the first and last parts being promoted to
5107	// scalars. If the middle part is an overlapping copy then we repeat
5108	// this process until it isn't.
5109	uint64_t OverlapStart =
5110	std::max(a: S.beginOffset(), b: CopyOverlap->beginOffset());
5111	uint64_t OverlapEnd = std::min(a: S.endOffset(), b: CopyOverlap->endOffset());
5112	uint64_t OverlapSize = OverlapEnd - OverlapStart;
5113	uint64_t SliceSize = S.endOffset() - S.beginOffset();
5114	uint64_t NonOverlapSize = SliceSize - OverlapSize;
5115	if (OverlapSize < NonOverlapSize) {
5116	// When the overlap area is smaller the middle part is initially and
5117	// remains outside the overlap, so splitting once is enough. The size
5118	// of the first and last part both equal the overlap size, so the
5119	// split points are that amount from the start and end of the slice.
5120	Offsets.Splits.push_back(x: OverlapSize);
5121	Offsets.Splits.push_back(x: SliceSize - OverlapSize);
5122	} else if (OverlapSize > NonOverlapSize) {
5123	// When the overlap area is larger the middle part is initially and
5124	// remains inside the overlap, so we repeatedly split it.
5125	for (uint64_t Split = NonOverlapSize; Split <= SliceSize / `2`;
5126	Split += NonOverlapSize) {
5127	Offsets.Splits.push_back(x: Split);
5128	if (Split != SliceSize - Split) {
5129	Offsets.Splits.push_back(x: SliceSize - Split);
5130	}
5131	}
5132	// Sort the splits as they need to be in ascending order.
5133	llvm::sort(C&: Offsets.Splits);
5134	} else {
5135	// Here the overlap and non-overlap size are the same, so the middle
5136	// part is empty and the slice is split exactly in the center.
5137	Offsets.Splits.push_back(x: OverlapSize);
5138	}
5139	} else {
5140	Offsets.Splits.push_back(x: P.endOffset() - S.beginOffset());
5141	}
5142	}
5143	}
5144
5145	// We may have split loads where some of their stores are split stores. For
5146	// such loads and stores, we can only pre-split them if their splits exactly
5147	// match relative to their starting offset. We have to verify this prior to
5148	// any rewriting.
5149	llvm::erase_if(C&: Stores, P: [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
5150	// Lookup the load we are storing in our map of split
5151	// offsets.
5152	auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5153	// If it was completely unsplittable, then we're done,
5154	// and this store can't be pre-split.
5155	if (UnsplittableLoads.count(Ptr: LI))
5156	return true;
5157
5158	auto LoadOffsetsI = SplitOffsetsMap.find(Val: LI);
5159	if (LoadOffsetsI == SplitOffsetsMap.end())
5160	return false; // Unrelated loads are definitely safe.
5161	auto &LoadOffsets = LoadOffsetsI ->second;
5162
5163	// Now lookup the store's offsets.
5164	auto &StoreOffsets = SplitOffsetsMap [SI];
5165
5166	// If the relative offsets of each split in the load and
5167	// store match exactly, then we can split them and we
5168	// don't need to remove them here.
5169	if (LoadOffsets.Splits == StoreOffsets.Splits)
5170	return false;
5171
5172	LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
5173	<< " " << *LI << "\n"
5174	<< " " << *SI << "\n");
5175
5176	// We've found a store and load that we need to split
5177	// with mismatched relative splits. Just give up on them
5178	// and remove both instructions from our list of
5179	// candidates.
5180	UnsplittableLoads.insert(Ptr: LI);
5181	return true;
5182	});
5183	// Now we have to go back* through all the stores, because a later store may*
5184	// have caused an earlier store's load to become unsplittable and if it is
5185	// unsplittable for the later store, then we can't rely on it being split in
5186	// the earlier store either.
5187	llvm::erase_if(C&: Stores, P: [&UnsplittableLoads](StoreInst *SI) {
5188	auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5189	return UnsplittableLoads.count(Ptr: LI);
5190	});
5191	// Once we've established all the loads that can't be split for some reason,
5192	// filter any that made it into our list out.
5193	llvm::erase_if(C&: Loads, P: [&UnsplittableLoads](LoadInst *LI) {
5194	return UnsplittableLoads.count(Ptr: LI);
5195	});
5196
5197	// If no loads or stores are left, there is no pre-splitting to be done for
5198	// this alloca.
5199	if (Loads.empty() && Stores.empty())
5200	return false;
5201
5202	// From here on, we can't fail and will be building new accesses, so rig up
5203	// an IR builder.
5204	IRBuilderTy IRB(&AI);
5205
5206	// Collect the new slices which we will merge into the alloca slices.
5207	SmallVector<Slice, `4`> NewSlices;
5208
5209	// Track any allocas we end up splitting loads and stores for so we iterate
5210	// on them.
5211	SmallPtrSet<AllocaInst *, `4`> ResplitPromotableAllocas;
5212
5213	// At this point, we have collected all of the loads and stores we can
5214	// pre-split, and the specific splits needed for them. We actually do the
5215	// splitting in a specific order in order to handle when one of the loads in
5216	// the value operand to one of the stores.
5217	//
5218	// First, we rewrite all of the split loads, and just accumulate each split
5219	// load in a parallel structure. We also build the slices for them and append
5220	// them to the alloca slices.
5221	SmallDenseMap<LoadInst , std::vector<LoadInst >, `1`> SplitLoadsMap;
5222	std::vector<LoadInst *> SplitLoads;
5223	const DataLayout &DL = AI.getDataLayout();
5224	for (LoadInst *LI : Loads) {
5225	SplitLoads.clear();
5226
5227	auto &Offsets = SplitOffsetsMap [LI];
5228	unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5229	assert(LI->getType()->getIntegerBitWidth() % `8` == `0` &&
5230	"Load must have type size equal to store size");
5231	assert(LI->getType()->getIntegerBitWidth() / `8` >= SliceSize &&
5232	"Load must be >= slice size");
5233
5234	uint64_t BaseOffset = Offsets.S->beginOffset();
5235	assert(BaseOffset + SliceSize > BaseOffset &&
5236	"Cannot represent alloca access size using 64-bit integers!");
5237
5238	Instruction *BasePtr = cast<Instruction>(Val: LI->getPointerOperand());
5239	IRB.SetInsertPoint(LI);
5240
5241	LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5242
5243	uint64_t PartOffset = `0`, PartSize = Offsets.Splits.front();
5244	int Idx = `0`, Size = Offsets.Splits.size();
5245	for (;;) {
5246	auto PartTy = Type::getIntNTy(C&: LI->getContext(), N: PartSize `8`);
5247	auto AS = LI->getPointerAddressSpace();
5248	auto *PartPtrTy = LI->getPointerOperandType();
5249	LoadInst *PLoad = IRB.CreateAlignedLoad(
5250	Ty: PartTy,
5251	Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5252	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5253	PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5254	Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5255	/IsVolatile/ isVolatile: false, Name: LI->getName());
5256	PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5257	LLVMContext::MD_access_group});
5258
5259	// Append this load onto the list of split loads so we can find it later
5260	// to rewrite the stores.
5261	SplitLoads.push_back(x: PLoad);
5262
5263	// Now build a new slice for the alloca.
5264	NewSlices.push_back(
5265	Elt: Slice (BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5266	&PLoad->getOperandUse(i: PLoad->getPointerOperandIndex()),
5267	/IsSplittable/ false));
5268	LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5269	<< ", " << NewSlices.back().endOffset()
5270	<< "): " << *PLoad << "\n");
5271
5272	// See if we've handled all the splits.
5273	if (Idx >= Size)
5274	break;
5275
5276	// Setup the next partition.
5277	PartOffset = Offsets.Splits [Idx];
5278	++Idx;
5279	PartSize = (Idx < Size ? Offsets.Splits [Idx] : SliceSize) - PartOffset;
5280	}
5281
5282	// Now that we have the split loads, do the slow walk over all uses of the
5283	// load and rewrite them as split stores, or save the split loads to use
5284	// below if the store is going to be split there anyways.
5285	bool DeferredStores = false;
5286	for (User *LU : LI->users()) {
5287	StoreInst *SI = cast<StoreInst>(Val: LU);
5288	if (!Stores.empty() && SplitOffsetsMap.count(Val: SI)) {
5289	DeferredStores = true;
5290	LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5291	<< "\n");
5292	continue;
5293	}
5294
5295	Value *StoreBasePtr = SI->getPointerOperand();
5296	IRB.SetInsertPoint(SI);
5297	AAMDNodes AATags = SI->getAAMetadata();
5298
5299	LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5300
5301	for (int Idx = `0`, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5302	LoadInst *PLoad = SplitLoads [Idx];
5303	uint64_t PartOffset = Idx == `0` ? `0` : Offsets.Splits [Idx - `1`];
5304	auto *PartPtrTy = SI->getPointerOperandType();
5305
5306	auto AS = SI->getPointerAddressSpace();
5307	StoreInst *PStore = IRB.CreateAlignedStore(
5308	Val: PLoad,
5309	Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5310	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5311	PointerTy: PartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5312	Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5313	/IsVolatile/ isVolatile: false);
5314	PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5315	LLVMContext::MD_access_group,
5316	LLVMContext::MD_DIAssignID});
5317
5318	if (AATags)
5319	PStore->setAAMetadata(
5320	AATags.adjustForAccess(Offset: PartOffset, AccessTy: PLoad->getType(), DL));
5321	LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5322	}
5323
5324	// We want to immediately iterate on any allocas impacted by splitting
5325	// this store, and we have to track any promotable alloca (indicated by
5326	// a direct store) as needing to be resplit because it is no longer
5327	// promotable.
5328	if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: StoreBasePtr)) {
5329	ResplitPromotableAllocas.insert(Ptr: OtherAI);
5330	Worklist.insert(X: OtherAI);
5331	} else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5332	Val: StoreBasePtr->stripInBoundsOffsets())) {
5333	Worklist.insert(X: OtherAI);
5334	}
5335
5336	// Mark the original store as dead.
5337	DeadInsts.push_back(Elt: SI);
5338	}
5339
5340	// Save the split loads if there are deferred stores among the users.
5341	if (DeferredStores)
5342	SplitLoadsMap.insert(KV: std::make_pair(x&: LI, y: std::move(SplitLoads)));
5343
5344	// Mark the original load as dead and kill the original slice.
5345	DeadInsts.push_back(Elt: LI);
5346	Offsets.S->kill();
5347	}
5348
5349	// Second, we rewrite all of the split stores. At this point, we know that
5350	// all loads from this alloca have been split already. For stores of such
5351	// loads, we can simply look up the pre-existing split loads. For stores of
5352	// other loads, we split those loads first and then write split stores of
5353	// them.
5354	for (StoreInst *SI : Stores) {
5355	auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5356	IntegerType *Ty = cast<IntegerType>(Val: LI->getType());
5357	assert(Ty->getBitWidth() % `8` == `0`);
5358	uint64_t StoreSize = Ty->getBitWidth() / `8`;
5359	assert(StoreSize > `0` && "Cannot have a zero-sized integer store!");
5360
5361	auto &Offsets = SplitOffsetsMap [SI];
5362	assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5363	"Slice size should always match load size exactly!");
5364	uint64_t BaseOffset = Offsets.S->beginOffset();
5365	assert(BaseOffset + StoreSize > BaseOffset &&
5366	"Cannot represent alloca access size using 64-bit integers!");
5367
5368	Value *LoadBasePtr = LI->getPointerOperand();
5369	Instruction *StoreBasePtr = cast<Instruction>(Val: SI->getPointerOperand());
5370
5371	LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5372
5373	// Check whether we have an already split load.
5374	auto SplitLoadsMapI = SplitLoadsMap.find(Val: LI);
5375	std::vector<LoadInst > SplitLoads = nullptr;
5376	if (SplitLoadsMapI != SplitLoadsMap.end()) {
5377	SplitLoads = &SplitLoadsMapI ->second;
5378	assert(SplitLoads->size() == Offsets.Splits.size() + `1` &&
5379	"Too few split loads for the number of splits in the store!");
5380	} else {
5381	LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5382	}
5383
5384	uint64_t PartOffset = `0`, PartSize = Offsets.Splits.front();
5385	int Idx = `0`, Size = Offsets.Splits.size();
5386	for (;;) {
5387	auto PartTy = Type::getIntNTy(C&: Ty->getContext(), N: PartSize `8`);
5388	auto *LoadPartPtrTy = LI->getPointerOperandType();
5389	auto *StorePartPtrTy = SI->getPointerOperandType();
5390
5391	// Either lookup a split load or create one.
5392	LoadInst *PLoad;
5393	if (SplitLoads) {
5394	PLoad = (*SplitLoads)[Idx];
5395	} else {
5396	IRB.SetInsertPoint(LI);
5397	auto AS = LI->getPointerAddressSpace();
5398	PLoad = IRB.CreateAlignedLoad(
5399	Ty: PartTy,
5400	Ptr: getAdjustedPtr(IRB, DL, Ptr: LoadBasePtr,
5401	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5402	PointerTy: LoadPartPtrTy, NamePrefix: LoadBasePtr->getName() + "."),
5403	Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5404	/IsVolatile/ isVolatile: false, Name: LI->getName());
5405	PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5406	LLVMContext::MD_access_group});
5407	}
5408
5409	// And store this partition.
5410	IRB.SetInsertPoint(SI);
5411	auto AS = SI->getPointerAddressSpace();
5412	StoreInst *PStore = IRB.CreateAlignedStore(
5413	Val: PLoad,
5414	Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5415	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5416	PointerTy: StorePartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5417	Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5418	/IsVolatile/ isVolatile: false);
5419	PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5420	LLVMContext::MD_access_group});
5421
5422	// Now build a new slice for the alloca.
5423	NewSlices.push_back(
5424	Elt: Slice (BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5425	&PStore->getOperandUse(i: PStore->getPointerOperandIndex()),
5426	/IsSplittable/ false));
5427	LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5428	<< ", " << NewSlices.back().endOffset()
5429	<< "): " << *PStore << "\n");
5430	if (!SplitLoads) {
5431	LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5432	}
5433
5434	// See if we've finished all the splits.
5435	if (Idx >= Size)
5436	break;
5437
5438	// Setup the next partition.
5439	PartOffset = Offsets.Splits [Idx];
5440	++Idx;
5441	PartSize = (Idx < Size ? Offsets.Splits [Idx] : StoreSize) - PartOffset;
5442	}
5443
5444	// We want to immediately iterate on any allocas impacted by splitting
5445	// this load, which is only relevant if it isn't a load of this alloca and
5446	// thus we didn't already split the loads above. We also have to keep track
5447	// of any promotable allocas we split loads on as they can no longer be
5448	// promoted.
5449	if (!SplitLoads) {
5450	if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: LoadBasePtr)) {
5451	assert(OtherAI != &AI && "We can't re-split our own alloca!");
5452	ResplitPromotableAllocas.insert(Ptr: OtherAI);
5453	Worklist.insert(X: OtherAI);
5454	} else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5455	Val: LoadBasePtr->stripInBoundsOffsets())) {
5456	assert(OtherAI != &AI && "We can't re-split our own alloca!");
5457	Worklist.insert(X: OtherAI);
5458	}
5459	}
5460
5461	// Mark the original store as dead now that we've split it up and kill its
5462	// slice. Note that we leave the original load in place unless this store
5463	// was its only use. It may in turn be split up if it is an alloca load
5464	// for some other alloca, but it may be a normal load. This may introduce
5465	// redundant loads, but where those can be merged the rest of the optimizer
5466	// should handle the merging, and this uncovers SSA splits which is more
5467	// important. In practice, the original loads will almost always be fully
5468	// split and removed eventually, and the splits will be merged by any
5469	// trivial CSE, including instcombine.
5470	if (LI->hasOneUse()) {
5471	assert(*LI->user_begin() == SI && "Single use isn't this store!");
5472	DeadInsts.push_back(Elt: LI);
5473	}
5474	DeadInsts.push_back(Elt: SI);
5475	Offsets.S->kill();
5476	}
5477
5478	// Remove the killed slices that have ben pre-split.
5479	llvm::erase_if(C&: AS, P: [](const Slice &S) { return S.isDead(); });
5480
5481	// Insert our new slices. This will sort and merge them into the sorted
5482	// sequence.
5483	AS.insert(NewSlices);
5484
5485	LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5486	#ifndef NDEBUG
5487	for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5488	LLVM_DEBUG(AS.print(dbgs(), I, " "));
5489	#endif
5490
5491	// Finally, don't try to promote any allocas that new require re-splitting.
5492	// They have already been added to the worklist above.
5493	PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5494
5495	return true;
5496	}
5497
5498	/// Try to canonicalize a homogeneous struct partition to a vector type.
5499	///
5500	/// We can do this if all the elements of the struct are the same and the
5501	/// corresponding vector has the same byte-level layout. This can sometimes
5502	/// eliminate allocas because structs cannot get promoted to LLVM values, but
5503	/// vectors can.
5504	///
5505	/// We only apply this transformation when all users of the partition are memory
5506	/// intrinsics. Otherwise, if there is a load or store of some other type to the
5507	/// partition, SROA would select that type.
5508	///
5509	/// Applying this transformation too early may hinder memcpyopt, which may
5510	/// generate better code when eliminating allocas. For example, see
5511	/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5512	/// this before memcpyopt can initialize previously uninitialized memory when
5513	/// the alloca gets promoted to an SSA value. For another example, see
5514	/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5515	/// this before memcpyopt can result in promoting an alloca so that we load a
5516	/// temporary value instead of copying the temporary value into memory, whereas
5517	/// memcpyopt eliminates the temporary altogether.
5518	///
5519	/// As such, we only apply this transformation after memcpyopt has run. We gate
5520	/// this transformation by the "AggregateToVector" pass option.
5521	static FixedVectorType tryCanonicalizeStructToVector(StructType STy,
5522	Partition &P,
5523	const DataLayout &DL) {
5524	unsigned NumElts = STy->getNumElements();
5525
5526	Type *EltTy = STy->getElementType(N: `0`);
5527	if (!llvm::all_equal(Range: STy->elements()))
5528	return nullptr;
5529
5530	bool IsIntegralPointerTy =
5531	EltTy->isPointerTy() && !DL.isNonIntegralPointerType(Ty: EltTy);
5532	if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5533	!IsIntegralPointerTy)
5534	return nullptr;
5535
5536	// Ensure the struct is tightly packed so that the bit-layout is the same as
5537	// the corresponding vector. For example, this prevents a miscompile for
5538	// { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5539	// tightly packed elements and trailing padding.
5540	if (DL.getTypeSizeInBits(Ty: EltTy) != DL.getTypeAllocSizeInBits(Ty: EltTy))
5541	return nullptr;
5542
5543	auto *VTy = FixedVectorType::get(ElementType: EltTy, NumElts);
5544	TypeSize StructSize = DL.getStructLayout(Ty: STy)->getSizeInBytes();
5545	TypeSize VectorSize = DL.getTypeStoreSize(Ty: VTy);
5546	// After ruling out per-element padding, make sure a vector load/store
5547	// covers the same number of bytes as the struct layout.
5548	if (StructSize != VectorSize)
5549	return nullptr;
5550
5551	auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5552	if (S.isDead())
5553	return true;
5554	auto *U = S.getUse();
5555	if (!U)
5556	return true;
5557
5558	User *Usr = U->getUser();
5559	if (isa<LifetimeIntrinsic>(Val: Usr) \|\| isa<DbgInfoIntrinsic>(Val: Usr))
5560	return true;
5561
5562	return isa<MemIntrinsic>(Val: Usr);
5563	};
5564
5565	for (const Slice &S : P)
5566	if (!IsIgnorableOrMemIntrinsicSlice (S))
5567	return nullptr;
5568
5569	for (const Slice *S : P.splitSliceTails())
5570	if (!IsIgnorableOrMemIntrinsicSlice (*S))
5571	return nullptr;
5572
5573	return VTy;
5574	}
5575
5576	/// Select a partition type for an alloca partition.
5577	///
5578	/// Try to compute a friendly type for this partition of the alloca. This
5579	/// won't always succeed, in which case we fall back to a legal integer type
5580	/// or an i8 array of an appropriate size.
5581	///
5582	/// \returns A tuple with the following elements:
5583	/// - PartitionType: The computed type for this partition.
5584	/// - IsIntegerWideningViable: True if integer widening promotion is used.
5585	/// - VectorType: The vector type if vector promotion is used, otherwise
5586	/// nullptr.
5587	static std::tuple<Type , bool, VectorType >
5588	selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
5589	LLVMContext &C, bool AggregateToVector) {
5590	auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5591	VectorType SelectedVecTy, bool* SelectedIntWidening) {
5592	LLVM_DEBUG({
5593	dbgs() << "selectPartitionType path=" << Path
5594	<< " func=" << AI.getFunction()->getName() << " alloca=";
5595	if (AI.hasName())
5596	dbgs() << AI.getName();
5597	else
5598	dbgs() << "<unnamed>";
5599	dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5600	<< ") size=" << P.size();
5601	if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5602	dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5603	if (SelectedTy)
5604	dbgs() << " chosen=" << *SelectedTy;
5605	if (SelectedVecTy)
5606	dbgs() << " vec=" << *SelectedVecTy;
5607	dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5608	});
5609	};
5610	// First check if the partition is viable for vector promotion.
5611	//
5612	// We prefer vector promotion over integer widening promotion when:
5613	// - The vector element type is a floating-point type.
5614	// - All the loads/stores to the alloca are vector loads/stores to the
5615	// entire alloca or load/store a single element of the vector.
5616	//
5617	// Otherwise when there is an integer vector with mixed type loads/stores we
5618	// prefer integer widening promotion because it's more likely the user is
5619	// doing bitwise arithmetic and we generate better code.
5620	VectorType *VecTy =
5621	isVectorPromotionViable(P, DL, VScale: AI.getFunction()->getVScaleValue());
5622	// If the vector element type is a floating-point type, we prefer vector
5623	// promotion. If the vector has one element, let the below code select
5624	// whether we promote with the vector or scalar.
5625	if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5626	VecTy->getElementCount().getFixedValue() > `1`) {
5627	LogSelection ("direct-fp-vecty", VecTy, VecTy, false);
5628	return {VecTy, false, VecTy};
5629	}
5630
5631	// Check if there is a common type that all slices of the partition use that
5632	// spans the partition.
5633	auto [CommonUseTy, LargestIntTy] =
5634	findCommonType(B: P.begin(), E: P.end(), EndOffset: P.endOffset());
5635	if (CommonUseTy) {
5636	TypeSize CommonUseSize = DL.getTypeAllocSize(Ty: CommonUseTy);
5637	if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5638	// We prefer vector promotion here because if vector promotion is viable
5639	// and there is a common type used, then it implies the second listed
5640	// condition for preferring vector promotion is true.
5641	if (VecTy) {
5642	LogSelection ("common-type-vecty", VecTy, VecTy, false);
5643	return {VecTy, false, VecTy};
5644	}
5645	bool IntWiden = isIntegerWideningViable(P, AllocaTy: CommonUseTy, DL);
5646	LogSelection ("common-type", CommonUseTy, nullptr, IntWiden);
5647	return {CommonUseTy, IntWiden, nullptr};
5648	}
5649	}
5650
5651	// Can we find an appropriate subtype in the original allocated
5652	// type?
5653	if (Type *TypePartitionTy = getTypePartition(DL, Ty: AI.getAllocatedType(),
5654	Offset: P.beginOffset(), Size: P.size())) {
5655	// If the partition is an integer array that can be spanned by a legal
5656	// integer type, prefer to represent it as a legal integer type because
5657	// it's more likely to be promotable.
5658	if (TypePartitionTy->isArrayTy() &&
5659	TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5660	DL.isLegalInteger(Width: P.size() * `8`))
5661	TypePartitionTy = Type::getIntNTy(C, N: P.size() * `8`);
5662	// There was no common type used, so we prefer integer widening promotion.
5663	if (isIntegerWideningViable(P, AllocaTy: TypePartitionTy, DL)) {
5664	LogSelection ("type-partition-int-widen", TypePartitionTy, nullptr, true);
5665	return {TypePartitionTy, true, nullptr};
5666	}
5667	if (VecTy) {
5668	LogSelection ("type-partition-vecty", VecTy, VecTy, false);
5669	return {VecTy, false, VecTy};
5670	}
5671	// If we couldn't promote with TypePartitionTy, try with the largest
5672	// integer type used.
5673	if (LargestIntTy &&
5674	DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size() &&
5675	isIntegerWideningViable(P, AllocaTy: LargestIntTy, DL)) {
5676	LogSelection ("largest-int-int-widen", LargestIntTy, nullptr, true);
5677	return {LargestIntTy, true, nullptr};
5678	}
5679
5680	// Try homogeneous struct to vector canonicalization when requested. Running
5681	// this too early can hide memcpy chains from MemCpyOpt.
5682	if (AggregateToVector) {
5683	if (auto *STy = dyn_cast<StructType>(Val: TypePartitionTy)) {
5684	if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5685	LogSelection ("struct-fallback-vecty", VTy, nullptr, false);
5686	return {VTy, false, nullptr};
5687	}
5688	}
5689	}
5690
5691	// Fallback to TypePartitionTy and we probably won't promote.
5692	LogSelection ("type-partition-fallback", TypePartitionTy, nullptr, false);
5693	return {TypePartitionTy, false, nullptr};
5694	}
5695
5696	// Select the largest integer type used if it spans the partition.
5697	if (LargestIntTy &&
5698	DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size()) {
5699	LogSelection ("largest-int-fallback", LargestIntTy, nullptr, false);
5700	return {LargestIntTy, false, nullptr};
5701	}
5702
5703	// Select a legal integer type if it spans the partition.
5704	if (DL.isLegalInteger(Width: P.size() * `8`)) {
5705	Type IntTy = Type::getIntNTy(C, N: P.size() `8`);
5706	LogSelection ("legal-int-fallback", IntTy, nullptr, false);
5707	return {IntTy, false, nullptr};
5708	}
5709
5710	// Fallback to an i8 array.
5711	Type *ArrayTy = ArrayType::get(ElementType: Type::getInt8Ty(C), NumElements: P.size());
5712	LogSelection ("byte-array-fallback", ArrayTy, nullptr, false);
5713	return {ArrayTy, false, nullptr};
5714	}
5715
5716	/// Rewrite an alloca partition's users.
5717	///
5718	/// This routine drives both of the rewriting goals of the SROA pass. It tries
5719	/// to rewrite uses of an alloca partition to be conducive for SSA value
5720	/// promotion. If the partition needs a new, more refined alloca, this will
5721	/// build that new alloca, preserving as much type information as possible, and
5722	/// rewrite the uses of the old alloca to point at the new one and have the
5723	/// appropriate new offsets. It also evaluates how successful the rewrite was
5724	/// at enabling promotion and if it was successful queues the alloca to be
5725	/// promoted.
5726	std::pair<AllocaInst *, uint64_t>
5727	SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5728	const DataLayout &DL = AI.getDataLayout();
5729	// Select the type for the new alloca that spans the partition.
5730	auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5731	selectPartitionType(P, DL, AI, C&: *C, AggregateToVector);
5732
5733	// Check for the case where we're going to rewrite to a new alloca of the
5734	// exact same type as the original, and with the same access offsets. In that
5735	// case, re-use the existing alloca, but still run through the rewriter to
5736	// perform phi and select speculation.
5737	// P.beginOffset() can be non-zero even with the same type in a case with
5738	// out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5739	AllocaInst *NewAI;
5740	if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == `0`) {
5741	NewAI = &AI;
5742	// FIXME: We should be able to bail at this point with "nothing changed".
5743	// FIXME: We might want to defer PHI speculation until after here.
5744	// FIXME: return nullptr;
5745	} else {
5746	// Make sure the alignment is compatible with P.beginOffset().
5747	const Align Alignment = commonAlignment(A: AI.getAlign(), Offset: P.beginOffset());
5748	NewAI =
5749	new AllocaInst (PartitionTy, AI.getAddressSpace(), nullptr, Alignment,
5750	AI.getName() + ".sroa." + Twine (P.begin() - AS.begin()),
5751	AI.getIterator());
5752	tryEnforceAlignment(V: NewAI, PrefAlign: DL.getPrefTypeAlign(Ty: PartitionTy), DL);
5753	// Copy the old AI debug location over to the new one.
5754	NewAI->setDebugLoc(AI.getDebugLoc());
5755	++NumNewAllocas;
5756	}
5757
5758	LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5759	<< "," << P.endOffset() << ") to: " << *NewAI << "\n");
5760
5761	// Track the high watermark on the worklist as it is only relevant for
5762	// promoted allocas. We will reset it to this point if the alloca is not in
5763	// fact scheduled for promotion.
5764	unsigned PPWOldSize = PostPromotionWorklist.size();
5765	unsigned NumUses = `0`;
5766	SmallSetVector<PHINode *, `8`> PHIUsers;
5767	SmallSetVector<SelectInst *, `8`> SelectUsers;
5768
5769	AllocaSliceRewriter Rewriter(
5770	DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5771	IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5772	bool Promotable = true;
5773	// Check whether we can have tree-structured merge.
5774	if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5775	NumUses += DeletedValues ->size() + `1`;
5776	for (Value V : DeletedValues)
5777	DeadInsts.push_back(Elt: V);
5778	} else {
5779	for (Slice *S : P.splitSliceTails()) {
5780	Promotable &= Rewriter.visit(I: S);
5781	++NumUses;
5782	}
5783	for (Slice &S : P) {
5784	Promotable &= Rewriter.visit(I: &S);
5785	++NumUses;
5786	}
5787	}
5788
5789	NumAllocaPartitionUses += NumUses;
5790	MaxUsesPerAllocaPartition.updateMax(V: NumUses);
5791
5792	// Now that we've processed all the slices in the new partition, check if any
5793	// PHIs or Selects would block promotion.
5794	SmallVector<PHINode *, `8`> NewSpeculatablePHIs;
5795	SmallVector<PHINode *, `2`> NewPHIsWithStoreToRewrite;
5796	for (PHINode *PHI : PHIUsers) {
5797	if (isSafePHIToSpeculate(PN&: *PHI)) {
5798	NewSpeculatablePHIs.push_back(Elt: PHI);
5799	continue;
5800	}
5801	if (getPHIStoreToRewrite(PN&: *PHI, PreserveCFG, DT&: DTU->getDomTree())) {
5802	NewPHIsWithStoreToRewrite.push_back(Elt: PHI);
5803	continue;
5804	}
5805
5806	Promotable = false;
5807	SelectUsers.clear();
5808	break;
5809	}
5810
5811	SmallVector<std::pair<SelectInst *, RewriteableMemOps>, `2`>
5812	NewSelectsToRewrite;
5813	NewSelectsToRewrite.reserve(N: SelectUsers.size());
5814	for (SelectInst *Sel : SelectUsers) {
5815	std::optional<RewriteableMemOps> Ops =
5816	isSafeSelectToSpeculate(SI&: *Sel, PreserveCFG);
5817	if (!Ops) {
5818	Promotable = false;
5819	break;
5820	}
5821	NewSelectsToRewrite.emplace_back(Args: std::make_pair(x&: Sel, y&: *Ops));
5822	}
5823
5824	if (Promotable) {
5825	for (Use *U : AS.getDeadUsesIfPromotable()) {
5826	auto *OldInst = dyn_cast<Instruction>(Val: U->get());
5827	Value::dropDroppableUse(U&: *U);
5828	if (OldInst)
5829	if (isInstructionTriviallyDead(I: OldInst))
5830	DeadInsts.push_back(Elt: OldInst);
5831	}
5832	if (NewSpeculatablePHIs.empty() && NewPHIsWithStoreToRewrite.empty() &&
5833	SelectUsers.empty()) {
5834	// Promote the alloca.
5835	PromotableAllocas.insert(X: NewAI);
5836	} else {
5837	// If we have either PHIs or Selects to rewrite, add them to those
5838	// worklists and re-queue the new alloca so that we promote it on the next
5839	// iteration.
5840	SpeculatablePHIs.insert_range(R&: NewSpeculatablePHIs);
5841	PHIsWithStoreToRewrite.insert_range(R&: NewPHIsWithStoreToRewrite);
5842	SelectsToRewrite.reserve(NumEntries: SelectsToRewrite.size() +
5843	NewSelectsToRewrite.size());
5844	for (auto &&KV : llvm::make_range(
5845	x: std::make_move_iterator(i: NewSelectsToRewrite.begin()),
5846	y: std::make_move_iterator(i: NewSelectsToRewrite.end())))
5847	SelectsToRewrite.insert(KV: std::move(KV));
5848	Worklist.insert(X: NewAI);
5849	}
5850	} else {
5851	// Drop any post-promotion work items if promotion didn't happen.
5852	while (PostPromotionWorklist.size() > PPWOldSize)
5853	PostPromotionWorklist.pop_back();
5854
5855	// We couldn't promote and we didn't create a new partition, nothing
5856	// happened.
5857	if (NewAI == &AI)
5858	return {nullptr, `0`};
5859
5860	// If we can't promote the alloca, iterate on it to check for new
5861	// refinements exposed by splitting the current alloca. Don't iterate on an
5862	// alloca which didn't actually change and didn't get promoted.
5863	Worklist.insert(X: NewAI);
5864	}
5865
5866	return {NewAI, DL.getTypeSizeInBits(Ty: PartitionTy).getFixedValue()};
5867	}
5868
5869	// There isn't a shared interface to get the "address" parts out of a
5870	// dbg.declare and dbg.assign, so provide some wrappers.
5871	bool isKillAddress(const DbgVariableRecord *DVR) {
5872	if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5873	return DVR->isKillAddress();
5874	return DVR->isKillLocation();
5875	}
5876
5877	const DIExpression getAddressExpression(const* DbgVariableRecord *DVR) {
5878	if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5879	return DVR->getAddressExpression();
5880	return DVR->getExpression();
5881	}
5882
5883	/// Create or replace an existing fragment in a DIExpression with \p Frag.
5884	/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5885	/// operation, add \p BitExtractOffset to the offset part.
5886	///
5887	/// Returns the new expression, or nullptr if this fails (see details below).
5888	///
5889	/// This function is similar to DIExpression::createFragmentExpression except
5890	/// for 3 important distinctions:
5891	/// 1. The new fragment isn't relative to an existing fragment.
5892	/// 2. It assumes the computed location is a memory location. This means we
5893	/// don't need to perform checks that creating the fragment preserves the
5894	/// expression semantics.
5895	/// 3. Existing extract_bits are modified independently of fragment changes
5896	/// using \p BitExtractOffset. A change to the fragment offset or size
5897	/// may affect a bit extract. But a bit extract offset can change
5898	/// independently of the fragment dimensions.
5899	///
5900	/// Returns the new expression, or nullptr if one couldn't be created.
5901	/// Ideally this is only used to signal that a bit-extract has become
5902	/// zero-sized (and thus the new debug record has no size and can be
5903	/// dropped), however, it fails for other reasons too - see the FIXME below.
5904	///
5905	/// FIXME: To keep the change that introduces this function NFC it bails
5906	/// in some situations unecessarily, e.g. when fragment and bit extract
5907	/// sizes differ.
5908	static DIExpression createOrReplaceFragment(const* DIExpression *Expr,
5909	DIExpression::FragmentInfo Frag,
5910	int64_t BitExtractOffset) {
5911	SmallVector<uint64_t, `8`> Ops;
5912	bool HasFragment = false;
5913	bool HasBitExtract = false;
5914
5915	for (auto &Op : Expr->expr_ops()) {
5916	if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5917	HasFragment = true;
5918	continue;
5919	}
5920	if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val: Op)) {
5921	HasBitExtract = true;
5922	int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5923	int64_t ExtractSizeInBits = Extract.getSizeInBits();
5924
5925	// DIExpression::createFragmentExpression doesn't know how to handle
5926	// a fragment that is smaller than the extract. Copy the behaviour
5927	// (bail) to avoid non-NFC changes.
5928	// FIXME: Don't do this.
5929	if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5930	return nullptr;
5931
5932	assert(BitExtractOffset <= `0`);
5933	int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5934
5935	// DIExpression::createFragmentExpression doesn't know what to do
5936	// if the new extract starts "outside" the existing one. Copy the
5937	// behaviour (bail) to avoid non-NFC changes.
5938	// FIXME: Don't do this.
5939	if (AdjustedOffset < `0`)
5940	return nullptr;
5941
5942	Ops.push_back(Elt: Op.getOp());
5943	Ops.push_back(Elt: std::max<int64_t>(a: `0`, b: AdjustedOffset));
5944	Ops.push_back(Elt: ExtractSizeInBits);
5945	continue;
5946	}
5947	Op.appendToVector(V&: Ops);
5948	}
5949
5950	// Unsupported by createFragmentExpression, so don't support it here yet to
5951	// preserve NFC-ness.
5952	if (HasFragment && HasBitExtract)
5953	return nullptr;
5954
5955	if (!HasBitExtract) {
5956	Ops.push_back(Elt: dwarf::DW_OP_LLVM_fragment);
5957	Ops.push_back(Elt: Frag.OffsetInBits);
5958	Ops.push_back(Elt: Frag.SizeInBits);
5959	}
5960	return DIExpression::get(Context&: Expr->getContext(), Elements: Ops);
5961	}
5962
5963	/// Insert a new DbgRecord.
5964	/// \p Orig Original to copy record type, debug loc and variable from, and
5965	/// additionally value and value expression for dbg_assign records.
5966	/// \p NewAddr Location's new base address.
5967	/// \p NewAddrExpr New expression to apply to address.
5968	/// \p BeforeInst Insert position.
5969	/// \p NewFragment New fragment (absolute, non-relative).
5970	/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
5971	static void
5972	insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord Orig, AllocaInst NewAddr,
5973	DIExpression NewAddrExpr, Instruction BeforeInst,
5974	std::optional<DIExpression::FragmentInfo> NewFragment,
5975	int64_t BitExtractAdjustment) {
5976	(void)DIB;
5977
5978	// A dbg_assign puts fragment info in the value expression only. The address
5979	// expression has already been built: NewAddrExpr. A dbg_declare puts the
5980	// new fragment info into NewAddrExpr (as it only has one expression).
5981	DIExpression *NewFragmentExpr =
5982	Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
5983	if (NewFragment)
5984	NewFragmentExpr = createOrReplaceFragment(Expr: NewFragmentExpr, Frag: *NewFragment,
5985	BitExtractOffset: BitExtractAdjustment);
5986	if (!NewFragmentExpr)
5987	return;
5988
5989	if (Orig->isDbgDeclare()) {
5990	DbgVariableRecord *DVR = DbgVariableRecord::createDVRDeclare(
5991	Address: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
5992	BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
5993	Here: BeforeInst->getIterator());
5994	return;
5995	}
5996
5997	if (Orig->isDbgValue()) {
5998	DbgVariableRecord *DVR = DbgVariableRecord::createDbgVariableRecord(
5999	Location: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
6000	// Drop debug information if the expression doesn't start with a
6001	// DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
6002	// describes the address of alloca rather than the value inside the alloca.
6003	if (!NewFragmentExpr->startsWithDeref())
6004	DVR->setKillAddress();
6005	BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
6006	Here: BeforeInst->getIterator());
6007	return;
6008	}
6009
6010	// Apply a DIAssignID to the store if it doesn't already have it.
6011	if (!NewAddr->hasMetadata(KindID: LLVMContext::MD_DIAssignID)) {
6012	NewAddr->setMetadata(KindID: LLVMContext::MD_DIAssignID,
6013	Node: DIAssignID::getDistinct(Context&: NewAddr->getContext()));
6014	}
6015
6016	DbgVariableRecord *NewAssign = DbgVariableRecord::createLinkedDVRAssign(
6017	LinkedInstr: NewAddr, Val: Orig->getValue(), Variable: Orig->getVariable(), Expression: NewFragmentExpr, Address: NewAddr,
6018	AddressExpression: NewAddrExpr, DI: Orig->getDebugLoc());
6019	LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
6020	(void)NewAssign;
6021	}
6022
6023	/// Walks the slices of an alloca and form partitions based on them,
6024	/// rewriting each of their uses.
6025	bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
6026	if (AS.begin() == AS.end())
6027	return false;
6028
6029	unsigned NumPartitions = `0`;
6030	bool Changed = false;
6031	const DataLayout &DL = AI.getDataLayout();
6032
6033	// First try to pre-split loads and stores.
6034	Changed \|= presplitLoadsAndStores(AI, AS);
6035
6036	// Now that we have identified any pre-splitting opportunities,
6037	// mark loads and stores unsplittable except for the following case.
6038	// We leave a slice splittable if all other slices are disjoint or fully
6039	// included in the slice, such as whole-alloca loads and stores.
6040	// If we fail to split these during pre-splitting, we want to force them
6041	// to be rewritten into a partition.
6042	bool IsSorted = true;
6043
6044	uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
6045	// We can split at the begin and end offsets of each slice, but only if those
6046	// offsets don't lie inside another slice. Because slices are ordered by
6047	// increasing begin offset, and then decreasing end offset, we can consider
6048	// the slices as being split up into sets with the same begin offset where we
6049	// can ignore every slice except the first (the begin offset will already be
6050	// handled as the begin offset of the set, and the end offset we know is not a
6051	// splittable offset as it's inside the first slice of the set).
6052	SparseBitVector<> SplittableOffset;
6053	uint64_t CurBegin = `0`, CurEnd = `0`;
6054	for (Slice &S : AS) {
6055	// Check if we have a new set of slices
6056	if (S.beginOffset() > CurBegin \|\| S.endOffset() > CurEnd) {
6057	// If the start isn't inside the previous set it's splittable
6058	if (S.beginOffset() >= CurEnd) {
6059	SplittableOffset.set(S.beginOffset());
6060	}
6061	// If the previous end is inside this slice then remove it
6062	if (CurEnd > S.beginOffset() && CurEnd < S.endOffset()) {
6063	SplittableOffset.reset(Idx: CurEnd);
6064	}
6065	CurBegin = S.beginOffset();
6066	// If the end offset isn't inside the previous set it's splittable. We
6067	// also don't update the end offset in that case, as the next set may also
6068	// be inside the previous set.
6069	if (S.endOffset() > CurEnd) {
6070	CurEnd = S.endOffset();
6071	SplittableOffset.set(CurEnd);
6072	}
6073	}
6074	}
6075
6076	for (Slice &S : AS) {
6077	if (!S.isSplittable())
6078	continue;
6079
6080	if ((S.beginOffset() > AllocaSize \|\|
6081	SplittableOffset.test(Idx: S.beginOffset())) &&
6082	(S.endOffset() > AllocaSize \|\| SplittableOffset.test(Idx: S.endOffset())))
6083	continue;
6084
6085	if (isa<LoadInst>(Val: S.getUse()->getUser()) \|\|
6086	isa<StoreInst>(Val: S.getUse()->getUser())) {
6087	S.makeUnsplittable();
6088	IsSorted = false;
6089	}
6090	}
6091
6092	if (!IsSorted)
6093	llvm::stable_sort(Range&: AS);
6094
6095	/// Describes the allocas introduced by rewritePartition in order to migrate
6096	/// the debug info.
6097	struct Fragment {
6098	AllocaInst *Alloca;
6099	uint64_t Offset;
6100	uint64_t Size;
6101	Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
6102	: Alloca(AI), Offset(O), Size(S) {}
6103	};
6104	SmallVector<Fragment, `4`> Fragments;
6105
6106	// Rewrite each partition.
6107	for (auto &P : AS.partitions()) {
6108	auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
6109	if (NewAI) {
6110	Changed = true;
6111	if (NewAI != &AI) {
6112	uint64_t SizeOfByte = `8`;
6113	// Don't include any padding.
6114	uint64_t Size = std::min(a: ActiveBits, b: P.size() * SizeOfByte);
6115	Fragments.push_back(
6116	Elt: Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
6117	}
6118	}
6119	++NumPartitions;
6120	}
6121
6122	NumAllocaPartitions += NumPartitions;
6123	MaxPartitionsPerAlloca.updateMax(V: NumPartitions);
6124
6125	// Migrate debug information from the old alloca to the new alloca(s)
6126	// and the individual partitions.
6127	auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
6128	// Can't overlap with undef memory.
6129	if (isKillAddress(DVR: DbgVariable))
6130	return;
6131
6132	const Value *DbgPtr = DbgVariable->getAddress();
6133	DIExpression::FragmentInfo VarFrag =
6134	DbgVariable->getFragmentOrEntireVariable();
6135	// Get the address expression constant offset if one exists and the ops
6136	// that come after it.
6137	int64_t CurrentExprOffsetInBytes = `0`;
6138	SmallVector<uint64_t> PostOffsetOps;
6139	if (!getAddressExpression(DVR: DbgVariable)
6140	->extractLeadingOffset(OffsetInBytes&: CurrentExprOffsetInBytes, RemainingOps&: PostOffsetOps))
6141	return; // Couldn't interpret this DIExpression - drop the var.
6142
6143	// Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
6144	int64_t ExtractOffsetInBits = `0`;
6145	for (auto Op : getAddressExpression(DVR: DbgVariable)->expr_ops()) {
6146	if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val&: Op)) {
6147	ExtractOffsetInBits = Extract.getOffsetInBits();
6148	break;
6149	}
6150	}
6151
6152	DIBuilder DIB(AI.getModule(), /AllowUnresolved/* false);
6153	for (auto Fragment : Fragments) {
6154	int64_t OffsetFromLocationInBits;
6155	std::optional<DIExpression::FragmentInfo> NewDbgFragment;
6156	// Find the variable fragment that the new alloca slice covers.
6157	// Drop debug info for this variable fragment if we can't compute an
6158	// intersect between it and the alloca slice.
6159	if (!DIExpression::calculateFragmentIntersect(
6160	DL, SliceStart: &AI, SliceOffsetInBits: Fragment.Offset, SliceSizeInBits: Fragment.Size, DbgPtr,
6161	DbgPtrOffsetInBits: CurrentExprOffsetInBytes * `8`, DbgExtractOffsetInBits: ExtractOffsetInBits, VarFrag,
6162	Result&: NewDbgFragment, OffsetFromLocationInBits))
6163	continue; // Do not migrate this fragment to this slice.
6164
6165	// Zero sized fragment indicates there's no intersect between the variable
6166	// fragment and the alloca slice. Skip this slice for this variable
6167	// fragment.
6168	if (NewDbgFragment && !NewDbgFragment ->SizeInBits)
6169	continue; // Do not migrate this fragment to this slice.
6170
6171	// No fragment indicates DbgVariable's variable or fragment exactly
6172	// overlaps the slice; copy its fragment (or nullopt if there isn't one).
6173	if (!NewDbgFragment)
6174	NewDbgFragment = DbgVariable->getFragment();
6175
6176	// Reduce the new expression offset by the bit-extract offset since
6177	// we'll be keeping that.
6178	int64_t OffestFromNewAllocaInBits =
6179	OffsetFromLocationInBits - ExtractOffsetInBits;
6180	// We need to adjust an existing bit extract if the offset expression
6181	// can't eat the slack (i.e., if the new offset would be negative).
6182	int64_t BitExtractOffset =
6183	std::min<int64_t>(a: `0`, b: OffestFromNewAllocaInBits);
6184	// The magnitude of a negative value indicates the number of bits into
6185	// the existing variable fragment that the memory region begins. The new
6186	// variable fragment already excludes those bits - the new DbgPtr offset
6187	// only needs to be applied if it's positive.
6188	OffestFromNewAllocaInBits =
6189	std::max(a: int64_t(`0`), b: OffestFromNewAllocaInBits);
6190
6191	// Rebuild the expression:
6192	// {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
6193	// Add NewDbgFragment later, because dbg.assigns don't want it in the
6194	// address expression but the value expression instead.
6195	DIExpression *NewExpr = DIExpression::get(Context&: AI.getContext(), Elements: PostOffsetOps);
6196	if (OffestFromNewAllocaInBits > `0`) {
6197	int64_t OffsetInBytes = (OffestFromNewAllocaInBits + `7`) / `8`;
6198	NewExpr = DIExpression::prepend(Expr: NewExpr, /flags=/Flags: `0`, Offset: OffsetInBytes);
6199	}
6200
6201	// Remove any existing intrinsics on the new alloca describing
6202	// the variable fragment.
6203	auto RemoveOne = [DbgVariable](auto *OldDII) {
6204	auto SameVariableFragment = [](const auto LHS, const* auto *RHS) {
6205	return LHS->getVariable() == RHS->getVariable() &&
6206	LHS->getDebugLoc()->getInlinedAt() ==
6207	RHS->getDebugLoc()->getInlinedAt();
6208	};
6209	if (SameVariableFragment(OldDII, DbgVariable))
6210	OldDII->eraseFromParent();
6211	};
6212	for_each(Range: findDVRDeclares(V: Fragment.Alloca), F: RemoveOne);
6213	for_each(Range: findDVRValues(V: Fragment.Alloca), F: RemoveOne);
6214	insertNewDbgInst(DIB, Orig: DbgVariable, NewAddr: Fragment.Alloca, NewAddrExpr: NewExpr, BeforeInst: &AI,
6215	NewFragment: NewDbgFragment, BitExtractAdjustment: BitExtractOffset);
6216	}
6217	};
6218
6219	// Migrate debug information from the old alloca to the new alloca(s)
6220	// and the individual partitions.
6221	for_each(Range: findDVRDeclares(V: &AI), F: MigrateOne);
6222	for_each(Range: findDVRValues(V: &AI), F: MigrateOne);
6223	for_each(Range: at::getDVRAssignmentMarkers(Inst: &AI), F: MigrateOne);
6224
6225	return Changed;
6226	}
6227
6228	/// Clobber a use with poison, deleting the used value if it becomes dead.
6229	void SROA::clobberUse(Use &U) {
6230	Value *OldV = U;
6231	// Replace the use with an poison value.
6232	U = PoisonValue::get(T: OldV->getType());
6233
6234	// Check for this making an instruction dead. We have to garbage collect
6235	// all the dead instructions to ensure the uses of any alloca end up being
6236	// minimal.
6237	if (Instruction *OldI = dyn_cast<Instruction>(Val: OldV))
6238	if (isInstructionTriviallyDead(I: OldI)) {
6239	DeadInsts.push_back(Elt: OldI);
6240	}
6241	}
6242
6243	/// A basic LoadAndStorePromoter that does not remove store nodes.
6244	class BasicLoadAndStorePromoter : public LoadAndStorePromoter {
6245	public:
6246	BasicLoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S,
6247	Type *ZeroType)
6248	: LoadAndStorePromoter (Insts, S), ZeroType(ZeroType) {}
6249	bool shouldDelete(Instruction I) const* override {
6250	return !isa<StoreInst>(Val: I) && !isa<AllocaInst>(Val: I);
6251	}
6252
6253	Value getValueToUseForAlloca(Instruction I) const override {
6254	return UndefValue::get(T: ZeroType);
6255	}
6256
6257	private:
6258	Type *ZeroType;
6259	};
6260
6261	bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6262	// Look through each "partition", looking for slices with the same start/end
6263	// that do not overlap with any before them. The slices are sorted by
6264	// increasing beginOffset. We don't use AS.partitions(), as it will use a more
6265	// sophisticated algorithm that takes splittable slices into account.
6266	LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6267	bool AllSameAndValid = true;
6268	Type PartitionType = nullptr*;
6269	SmallVector<Instruction *> Insts;
6270	uint64_t BeginOffset = `0`;
6271	uint64_t EndOffset = `0`;
6272
6273	auto Flush = [&]() {
6274	if (AllSameAndValid && !Insts.empty()) {
6275	LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6276	<< EndOffset << ")\n");
6277	SmallVector<PHINode *, `4`> NewPHIs;
6278	SSAUpdater SSA(&NewPHIs);
6279	Insts.push_back(Elt: &AI);
6280	BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6281	Promoter.run(Insts);
6282	}
6283	AllSameAndValid = true;
6284	PartitionType = nullptr;
6285	Insts.clear();
6286	};
6287
6288	for (Slice &S : AS) {
6289	auto *User = cast<Instruction>(Val: S.getUse()->getUser());
6290	if (isAssumeLikeIntrinsic(I: User)) {
6291	LLVM_DEBUG({
6292	dbgs() << "Ignoring slice: ";
6293	AS.print(dbgs(), &S);
6294	});
6295	continue;
6296	}
6297	if (S.beginOffset() >= EndOffset) {
6298	Flush ();
6299	BeginOffset = S.beginOffset();
6300	EndOffset = S.endOffset();
6301	} else if (S.beginOffset() != BeginOffset \|\| S.endOffset() != EndOffset) {
6302	if (AllSameAndValid) {
6303	LLVM_DEBUG({
6304	dbgs() << "Slice does not match range [" << BeginOffset << ", "
6305	<< EndOffset << ")";
6306	AS.print(dbgs(), &S);
6307	});
6308	AllSameAndValid = false;
6309	}
6310	EndOffset = std::max(a: EndOffset, b: S.endOffset());
6311	continue;
6312	}
6313
6314	if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
6315	Type *UserTy = LI->getType();
6316	// LoadAndStorePromoter requires all the types to be the same.
6317	if (!LI->isSimple() \|\| (PartitionType && UserTy != PartitionType))
6318	AllSameAndValid = false;
6319	PartitionType = UserTy;
6320	Insts.push_back(Elt: User);
6321	} else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
6322	Type *UserTy = SI->getValueOperand()->getType();
6323	if (!SI->isSimple() \|\| (PartitionType && UserTy != PartitionType))
6324	AllSameAndValid = false;
6325	PartitionType = UserTy;
6326	Insts.push_back(Elt: User);
6327	} else {
6328	AllSameAndValid = false;
6329	}
6330	}
6331
6332	Flush ();
6333	return true;
6334	}
6335
6336	/// Analyze an alloca for SROA.
6337	///
6338	/// This analyzes the alloca to ensure we can reason about it, builds
6339	/// the slices of the alloca, and then hands it off to be split and
6340	/// rewritten as needed.
6341	std::pair<bool /Changed/, bool /CFGChanged/>
6342	SROA::runOnAlloca(AllocaInst &AI) {
6343	bool Changed = false;
6344	bool CFGChanged = false;
6345
6346	LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6347	++NumAllocasAnalyzed;
6348
6349	// Special case dead allocas, as they're trivial.
6350	if (AI.use_empty()) {
6351	AI.eraseFromParent();
6352	Changed = true;
6353	return {Changed, CFGChanged};
6354	}
6355	const DataLayout &DL = AI.getDataLayout();
6356
6357	// Skip alloca forms that this analysis can't handle.
6358	std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6359	if (AI.isArrayAllocation() \|\| !Size \|\| Size ->isScalable() \|\| Size ->isZero())
6360	return {Changed, CFGChanged};
6361
6362	// First, split any FCA loads and stores touching this alloca to promote
6363	// better splitting and promotion opportunities.
6364	IRBuilderTy IRB(&AI);
6365	AggLoadStoreRewriter AggRewriter(DL, IRB);
6366	Changed \|= AggRewriter.rewrite(I&: AI);
6367
6368	// Build the slices using a recursive instruction-visiting builder.
6369	AllocaSlices AS(DL, AI);
6370	LLVM_DEBUG(AS.print(dbgs()));
6371	if (AS.isEscaped())
6372	return {Changed, CFGChanged};
6373
6374	if (AS.isEscapedReadOnly()) {
6375	Changed \|= propagateStoredValuesToLoads(AI, AS);
6376	return {Changed, CFGChanged};
6377	}
6378
6379	// Delete all the dead users of this alloca before splitting and rewriting it.
6380	for (Instruction *DeadUser : AS.getDeadUsers()) {
6381	// Free up everything used by this instruction.
6382	for (Use &DeadOp : DeadUser->operands())
6383	clobberUse(U&: DeadOp);
6384
6385	// Now replace the uses of this instruction.
6386	DeadUser->replaceAllUsesWith(V: PoisonValue::get(T: DeadUser->getType()));
6387
6388	// And mark it for deletion.
6389	DeadInsts.push_back(Elt: DeadUser);
6390	Changed = true;
6391	}
6392	for (Use *DeadOp : AS.getDeadOperands()) {
6393	clobberUse(U&: *DeadOp);
6394	Changed = true;
6395	}
6396
6397	// No slices to split. Leave the dead alloca for a later pass to clean up.
6398	if (AS.begin() == AS.end())
6399	return {Changed, CFGChanged};
6400
6401	Changed \|= splitAlloca(AI, AS);
6402
6403	LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6404	while (!SpeculatablePHIs.empty())
6405	speculatePHINodeLoads(IRB, PN&: *SpeculatablePHIs.pop_back_val());
6406
6407	LLVM_DEBUG(dbgs() << " Rewriting stores through PHIs\n");
6408	auto RemainingPHIsWithStoreToRewrite = PHIsWithStoreToRewrite.takeVector();
6409	while (!RemainingPHIsWithStoreToRewrite.empty()) {
6410	PHINode *PN = RemainingPHIsWithStoreToRewrite.pop_back_val();
6411	auto *SI = cast<StoreInst>(Val: PN->user_back());
6412	CFGChanged \|= rewritePHINodeStore(PN&: PN, SI&: SI, DTU&: *DTU, Worklist);
6413	}
6414
6415	LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6416	auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6417	while (!RemainingSelectsToRewrite.empty()) {
6418	const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6419	CFGChanged \|=
6420	rewriteSelectInstMemOps(SI&: K, Ops: V, IRB, DTU: PreserveCFG ? nullptr* : DTU);
6421	}
6422
6423	return {Changed, CFGChanged};
6424	}
6425
6426	/// Delete the dead instructions accumulated in this run.
6427	///
6428	/// Recursively deletes the dead instructions we've accumulated. This is done
6429	/// at the very end to maximize locality of the recursive delete and to
6430	/// minimize the problems of invalidated instruction pointers as such pointers
6431	/// are used heavily in the intermediate stages of the algorithm.
6432	///
6433	/// We also record the alloca instructions deleted here so that they aren't
6434	/// subsequently handed to mem2reg to promote.
6435	bool SROA::deleteDeadInstructions(
6436	SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6437	bool Changed = false;
6438	while (!DeadInsts.empty()) {
6439	Instruction *I = dyn_cast_or_null<Instruction>(Val: DeadInsts.pop_back_val());
6440	if (!I)
6441	continue;
6442	LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6443
6444	// If the instruction is an alloca, find the possible dbg.declare connected
6445	// to it, and remove it too. We must do this before calling RAUW or we will
6446	// not be able to find it.
6447	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: I)) {
6448	DeletedAllocas.insert(Ptr: AI);
6449	for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6450	OldDII->eraseFromParent();
6451	}
6452
6453	at::deleteAssignmentMarkers(Inst: I);
6454	I->replaceAllUsesWith(V: UndefValue::get(T: I->getType()));
6455
6456	for (Use &Operand : I->operands())
6457	if (Instruction *U = dyn_cast<Instruction>(Val&: Operand)) {
6458	// Zero out the operand and see if it becomes trivially dead.
6459	Operand = nullptr;
6460	if (isInstructionTriviallyDead(I: U))
6461	DeadInsts.push_back(Elt: U);
6462	}
6463
6464	++NumDeleted;
6465	I->eraseFromParent();
6466	Changed = true;
6467	}
6468	return Changed;
6469	}
6470	/// Promote the allocas, using the best available technique.
6471	///
6472	/// This attempts to promote whatever allocas have been identified as viable in
6473	/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6474	/// This function returns whether any promotion occurred.
6475	bool SROA::promoteAllocas() {
6476	if (PromotableAllocas.empty())
6477	return false;
6478
6479	if (ScalarOptions::Global.sroa_skip_mem2reg) {
6480	LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6481	} else {
6482	LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6483	NumPromoted += PromotableAllocas.size();
6484	PromoteMemToReg(Allocas: PromotableAllocas.getArrayRef(), DT&: DTU->getDomTree(), AC);
6485	}
6486
6487	PromotableAllocas.clear();
6488	return true;
6489	}
6490
6491	std::pair<bool /Changed/, bool /CFGChanged/> SROA::runSROA(Function &F) {
6492	LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6493
6494	const DataLayout &DL = F.getDataLayout();
6495	BasicBlock &EntryBB = F.getEntryBlock();
6496	for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(x: EntryBB.end());
6497	I != E; ++I) {
6498	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val&: I)) {
6499	std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6500	if (Size && Size ->isScalable() && isAllocaPromotable(AI))
6501	PromotableAllocas.insert(X: AI);
6502	else
6503	Worklist.insert(X: AI);
6504	}
6505	}
6506
6507	bool Changed = false;
6508	bool CFGChanged = false;
6509	// A set of deleted alloca instruction pointers which should be removed from
6510	// the list of promotable allocas.
6511	SmallPtrSet<AllocaInst *, `4`> DeletedAllocas;
6512
6513	do {
6514	while (!Worklist.empty()) {
6515	auto [IterationChanged, IterationCFGChanged] =
6516	runOnAlloca(AI&: *Worklist.pop_back_val());
6517	Changed \|= IterationChanged;
6518	CFGChanged \|= IterationCFGChanged;
6519
6520	Changed \|= deleteDeadInstructions(DeletedAllocas);
6521
6522	// Remove the deleted allocas from various lists so that we don't try to
6523	// continue processing them.
6524	if (!DeletedAllocas.empty()) {
6525	Worklist.set_subtract(DeletedAllocas);
6526	PostPromotionWorklist.set_subtract(DeletedAllocas);
6527	PromotableAllocas.set_subtract(DeletedAllocas);
6528	DeletedAllocas.clear();
6529	}
6530	}
6531
6532	Changed \|= promoteAllocas();
6533
6534	Worklist = PostPromotionWorklist;
6535	PostPromotionWorklist.clear();
6536	} while (!Worklist.empty());
6537
6538	assert((!CFGChanged \|\| Changed) && "Can not only modify the CFG.");
6539	assert((!CFGChanged \|\| !PreserveCFG) &&
6540	"Should not have modified the CFG when told to preserve it.");
6541
6542	if (Changed && isAssignmentTrackingEnabled(M: *F.getParent())) {
6543	for (auto &BB : F) {
6544	RemoveRedundantDbgInstrs(BB: &BB);
6545	}
6546	}
6547
6548	return {Changed, CFGChanged};
6549	}
6550
6551	PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
6552	DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
6553	AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
6554	DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6555	auto [Changed, CFGChanged] =
6556	SROA (&F.getContext(), &DTU, &AC, Options).runSROA(F);
6557	if (!Changed)
6558	return PreservedAnalyses::all();
6559	PreservedAnalyses PA;
6560	if (!CFGChanged)
6561	PA.preserveSet<CFGAnalyses>();
6562	PA.preserve<DominatorTreeAnalysis>();
6563	return PA;
6564	}
6565
6566	void SROAPass::printPipeline(
6567	raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6568	static_cast<PassInfoMixin<SROAPass> >(this*)->printPipeline(
6569	OS, MapClassName2PassName);
6570	OS << `'<'`
6571	<< (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6572	: "modify-cfg");
6573	if (Options.AggregateToVector)
6574	OS << ";aggregate-to-vector";
6575	OS << `'>'`;
6576	}
6577
6578	SROAPass::SROAPass(SROAOptions Options) : Options (Options) {}
6579
6580	namespace {
6581
6582	/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6583	class SROALegacyPass : public FunctionPass {
6584	SROAOptions Options;
6585
6586	public:
6587	static char ID;
6588
6589	SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
6590	: FunctionPass (ID), Options (Options) {
6591	initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
6592	}
6593
6594	bool runOnFunction(Function &F) override {
6595	if (skipFunction(F))
6596	return false;
6597
6598	DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6599	AssumptionCache &AC =
6600	getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6601	DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6602	auto [Changed, _] = SROA (&F.getContext(), &DTU, &AC, Options).runSROA(F);
6603	return Changed;
6604	}
6605
6606	void getAnalysisUsage(AnalysisUsage &AU) const override {
6607	AU.addRequired<AssumptionCacheTracker>();
6608	AU.addRequired<DominatorTreeWrapperPass>();
6609	AU.addPreserved<GlobalsAAWrapperPass>();
6610	AU.addPreserved<DominatorTreeWrapperPass>();
6611	}
6612
6613	StringRef getPassName() const override { return "SROA"; }
6614	};
6615
6616	} // end anonymous namespace
6617
6618	char SROALegacyPass::ID = `0`;
6619
6620	FunctionPass llvm::createSROAPass(bool* PreserveCFG, bool AggregateToVector) {
6621	return new SROALegacyPass (SROAOptions (PreserveCFG ? SROAOptions::PreserveCFG
6622	: SROAOptions::ModifyCFG,
6623	AggregateToVector));
6624	}
6625
6626	INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6627	"Scalar Replacement Of Aggregates", false, false)
6628	INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6629	INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
6630	INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
6631	false, false)
6632

Browse the source code of llvm_projects/llvm/lib/Transforms/Scalar/SROA.cpp