SROA.cpp source code [llvm_projects/llvm/lib/Transforms/Scalar/SROA.cpp]

1	//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	/// \file
9	/// This transformation implements the well known scalar replacement of
10	/// aggregates transformation. It tries to identify promotable elements of an
11	/// aggregate alloca, and promote them to registers. It will also try to
12	/// convert uses of an element (or set of elements) of an alloca into a vector
13	/// or bitfield-style integer scalar if appropriate.
14	///
15	/// It works to do this with minimal slicing of the alloca so that regions
16	/// which are merely transferred in and out of external memory remain unchanged
17	/// and are not decomposed to scalar code.
18	///
19	/// Because this also performs alloca promotion, it can be thought of as also
20	/// serving the purpose of SSA formation. The algorithm iterates on the
21	/// function until all opportunities for promotion have been realized.
22	///
23	//===----------------------------------------------------------------------===//
24
25	#include "llvm/Transforms/Scalar/SROA.h"
26	#include "ScalarOptions.h"
27	#include "llvm/ADT/APInt.h"
28	#include "llvm/ADT/ArrayRef.h"
29	#include "llvm/ADT/DenseMap.h"
30	#include "llvm/ADT/MapVector.h"
31	#include "llvm/ADT/PointerIntPair.h"
32	#include "llvm/ADT/STLExtras.h"
33	#include "llvm/ADT/SetVector.h"
34	#include "llvm/ADT/SmallPtrSet.h"
35	#include "llvm/ADT/SmallVector.h"
36	#include "llvm/ADT/SparseBitVector.h"
37	#include "llvm/ADT/Statistic.h"
38	#include "llvm/ADT/StringRef.h"
39	#include "llvm/ADT/Twine.h"
40	#include "llvm/ADT/iterator.h"
41	#include "llvm/ADT/iterator_range.h"
42	#include "llvm/Analysis/AssumptionCache.h"
43	#include "llvm/Analysis/DomTreeUpdater.h"
44	#include "llvm/Analysis/GlobalsModRef.h"
45	#include "llvm/Analysis/Loads.h"
46	#include "llvm/Analysis/PtrUseVisitor.h"
47	#include "llvm/Analysis/ValueTracking.h"
48	#include "llvm/Analysis/VectorUtils.h"
49	#include "llvm/IR/BasicBlock.h"
50	#include "llvm/IR/Constant.h"
51	#include "llvm/IR/ConstantFolder.h"
52	#include "llvm/IR/Constants.h"
53	#include "llvm/IR/DIBuilder.h"
54	#include "llvm/IR/DataLayout.h"
55	#include "llvm/IR/DebugInfo.h"
56	#include "llvm/IR/DebugInfoMetadata.h"
57	#include "llvm/IR/DerivedTypes.h"
58	#include "llvm/IR/Dominators.h"
59	#include "llvm/IR/Function.h"
60	#include "llvm/IR/GlobalAlias.h"
61	#include "llvm/IR/IRBuilder.h"
62	#include "llvm/IR/InstVisitor.h"
63	#include "llvm/IR/Instruction.h"
64	#include "llvm/IR/Instructions.h"
65	#include "llvm/IR/IntrinsicInst.h"
66	#include "llvm/IR/LLVMContext.h"
67	#include "llvm/IR/Metadata.h"
68	#include "llvm/IR/Module.h"
69	#include "llvm/IR/Operator.h"
70	#include "llvm/IR/PassManager.h"
71	#include "llvm/IR/Type.h"
72	#include "llvm/IR/Use.h"
73	#include "llvm/IR/User.h"
74	#include "llvm/IR/Value.h"
75	#include "llvm/IR/ValueHandle.h"
76	#include "llvm/InitializePasses.h"
77	#include "llvm/Pass.h"
78	#include "llvm/Support/Casting.h"
79	#include "llvm/Support/Compiler.h"
80	#include "llvm/Support/Debug.h"
81	#include "llvm/Support/ErrorHandling.h"
82	#include "llvm/Support/raw_ostream.h"
83	#include "llvm/Transforms/Scalar.h"
84	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
85	#include "llvm/Transforms/Utils/Local.h"
86	#include "llvm/Transforms/Utils/PromoteMemToReg.h"
87	#include "llvm/Transforms/Utils/SSAUpdater.h"
88	#include <algorithm>
89	#include <cassert>
90	#include <cstddef>
91	#include <cstdint>
92	#include <cstring>
93	#include <iterator>
94	#include <string>
95	#include <tuple>
96	#include <utility>
97	#include <variant>
98	#include <vector>
99
100	using namespace llvm;
101
102	#define DEBUG_TYPE "sroa"
103
104	STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105	STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106	STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107	STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108	STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109	STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110	STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111	STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112	STATISTIC(NumLoadsPredicated,
113	"Number of loads rewritten into predicated loads to allow promotion");
114	STATISTIC(
115	NumStoresPredicated,
116	"Number of stores rewritten into predicated stores to allow promotion");
117	STATISTIC(NumDeleted, "Number of instructions deleted");
118	STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120	namespace {
121
122	class AllocaSliceRewriter;
123	class AllocaSlices;
124	class Partition;
125	class Slice;
126
127	class SelectHandSpeculativity {
128	unsigned char Storage = `0`; // None are speculatable by default.
129	using TrueVal = Bitfield::Element<bool, `0`, `1`>; // Low 0'th bit.
130	using FalseVal = Bitfield::Element<bool, `1`, `1`>; // Low 1'th bit.
131	public:
132	SelectHandSpeculativity() = default;
133	SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
134	bool isSpeculatable(bool isTrueVal) const;
135	bool areAllSpeculatable() const;
136	bool areAnySpeculatable() const;
137	bool areNoneSpeculatable() const;
138	// For interop as int half of PointerIntPair.
139	explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
140	explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
141	};
142	static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
143
144	using PossiblySpeculatableLoad =
145	PointerIntPair<LoadInst *, `2`, SelectHandSpeculativity>;
146	using UnspeculatableStore = StoreInst *;
147	using RewriteableMemOp =
148	std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
149	using RewriteableMemOps = SmallVector<RewriteableMemOp, `2`>;
150	using InstructionSliceMap =
151	SmallDenseMap<Instruction , SmallPtrSet<Slice , `8`>, `8`>;
152
153	/// An optimization pass providing Scalar Replacement of Aggregates.
154	///
155	/// This pass takes allocations which can be completely analyzed (that is, they
156	/// don't escape) and tries to turn them into scalar SSA values. There are
157	/// a few steps to this process.
158	///
159	/// 1) It takes allocations of aggregates and analyzes the ways in which they
160	/// are used to try to split them into smaller allocations, ideally of
161	/// a single scalar data type. It will split up memcpy and memset accesses
162	/// as necessary and try to isolate individual scalar accesses.
163	/// 2) It will transform accesses into forms which are suitable for SSA value
164	/// promotion. This can be replacing a memset with a scalar store of an
165	/// integer value, or it can involve speculating operations on a PHI or
166	/// select to be a PHI or select of the results.
167	/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
168	/// onto insert and extract operations on a vector value, and convert them to
169	/// this form. By doing so, it will enable promotion of vector aggregates to
170	/// SSA vector values.
171	class SROA {
172	LLVMContext *const C;
173	DomTreeUpdater *const DTU;
174	AssumptionCache *const AC;
175	const bool PreserveCFG;
176	const bool AggregateToVector;
177
178	/// Worklist of alloca instructions to simplify.
179	///
180	/// Each alloca in the function is added to this. Each new alloca formed gets
181	/// added to it as well to recursively simplify unless that alloca can be
182	/// directly promoted. Finally, each time we rewrite a use of an alloca other
183	/// the one being actively rewritten, we add it back onto the list if not
184	/// already present to ensure it is re-visited.
185	SmallSetVector<AllocaInst *, `16`> Worklist;
186
187	/// A collection of instructions to delete.
188	/// We try to batch deletions to simplify code and make things a bit more
189	/// efficient. We also make sure there is no dangling pointers.
190	SmallVector<WeakVH, `8`> DeadInsts;
191
192	/// Post-promotion worklist.
193	///
194	/// Sometimes we discover an alloca which has a high probability of becoming
195	/// viable for SROA after a round of promotion takes place. In those cases,
196	/// the alloca is enqueued here for re-processing.
197	///
198	/// Note that we have to be very careful to clear allocas out of this list in
199	/// the event they are deleted.
200	SmallSetVector<AllocaInst *, `16`> PostPromotionWorklist;
201
202	/// A collection of alloca instructions we can directly promote.
203	SetVector<AllocaInst , SmallVector<AllocaInst >,
204	SmallPtrSet<AllocaInst *, `16`>, `16`>
205	PromotableAllocas;
206
207	/// A worklist of PHIs to speculate prior to promoting allocas.
208	///
209	/// All of these PHIs have been checked for the safety of speculation and by
210	/// being speculated will allow promoting allocas currently in the promotable
211	/// queue.
212	SmallSetVector<PHINode *, `8`> SpeculatablePHIs;
213
214	/// A worklist of PHIs whose stores should be predicated onto incoming edges.
215	///
216	/// All of these PHIs have been checked so that rewriting their stores will
217	/// allow the corresponding allocas to be promoted. Edges which require
218	/// splitting are only present when CFG modification is allowed.
219	SmallSetVector<PHINode *, `8`> PHIsWithStoreToRewrite;
220
221	/// A worklist of select instructions to rewrite prior to promoting
222	/// allocas.
223	SmallMapVector<SelectInst *, RewriteableMemOps, `8`> SelectsToRewrite;
224
225	/// Check whether the loads and stores of the select can be speculatively
226	/// executed or predicated (if \p PreserveCFG is false).
227	///
228	/// Speculation converts a load like
229	///
230	/// %P = select i1 %cond, ptr %Alloca, ptr %Other
231	/// %V = load <type>, ptr %P
232	///
233	/// to:
234	///
235	/// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
236	/// %V2 = load <type>, ptr %Other
237	/// %V = select i1 %cond, <type> %V1, <type> %V2
238	///
239	/// For stores and some kinds of loads that can't be speculated, we need to
240	/// predicate. Predication changes the CFG, so it requires \p PreserveCFG to
241	/// be false. Predication converts a store like:
242	///
243	/// block:
244	/// %P = select i1 %cond, ptr %Alloca, ptr %Other
245	/// store <type> %Value, ptr %P
246	///
247	/// to:
248	///
249	/// block:
250	/// br i1 %cond, label %block.then, label %block.else
251	///
252	/// block.then:
253	/// store <type> %Value, ptr %Alloca
254	/// br label %block.cont
255	///
256	/// block.else:
257	/// store <type> %Value, ptr %Other
258	/// br label %block.cont
259	///
260	/// Returns `std::nullopt` when
261	/// - the select has users other than loads or stores, or
262	/// - at least one of the loads and stores is volatile, or
263	/// - the loads and stores can't be speculated and can't be predicated because
264	/// we're not allowed to modify the CFG. (\p PreserveCFG is true)
265	///
266	/// Otherwise the stores and loads in the returned vector can either be
267	/// speculated or predicated.
268	static std::optional<RewriteableMemOps>
269	isSafeSelectToSpeculateOrPredicate(SelectInst &SI, bool PreserveCFG);
270
271	public:
272	SROA(LLVMContext C, DomTreeUpdater DTU, AssumptionCache *AC,
273	SROAOptions Options)
274	: C(C), DTU(DTU), AC(AC),
275	PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
276	AggregateToVector(Options.AggregateToVector) {}
277
278	/// Main run method used by both the SROAPass and by the legacy pass.
279	std::pair<bool /Changed/, bool /CFGChanged/> runSROA(Function &F);
280
281	private:
282	friend class AllocaSliceRewriter;
283
284	bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
285	std::pair<AllocaInst *, uint64_t>
286	rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
287	bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
288	bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
289	std::pair<bool /Changed/, bool /CFGChanged/> runOnAlloca(AllocaInst &AI);
290	void clobberUse(Use &U);
291	bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
292	bool promoteAllocas();
293	};
294
295	} // end anonymous namespace
296
297	/// Calculate the fragment of a variable to use when slicing a store
298	/// based on the slice dimensions, existing fragment, and base storage
299	/// fragment.
300	/// Results:
301	/// UseFrag - Use Target as the new fragment.
302	/// UseNoFrag - The new slice already covers the whole variable.
303	/// Skip - The new alloca slice doesn't include this variable.
304	/// FIXME: Can we use calculateFragmentIntersect instead?
305	namespace {
306	enum FragCalcResult { UseFrag, UseNoFrag, Skip };
307	}
308	static FragCalcResult
309	calculateFragment(DILocalVariable *Variable,
310	uint64_t NewStorageSliceOffsetInBits,
311	uint64_t NewStorageSliceSizeInBits,
312	std::optional<DIExpression::FragmentInfo> StorageFragment,
313	std::optional<DIExpression::FragmentInfo> CurrentFragment,
314	DIExpression::FragmentInfo &Target) {
315	// If the base storage describes part of the variable apply the offset and
316	// the size constraint.
317	if (StorageFragment) {
318	Target.SizeInBits =
319	std::min(a: NewStorageSliceSizeInBits, b: StorageFragment ->SizeInBits);
320	Target.OffsetInBits =
321	NewStorageSliceOffsetInBits + StorageFragment ->OffsetInBits;
322	} else {
323	Target.SizeInBits = NewStorageSliceSizeInBits;
324	Target.OffsetInBits = NewStorageSliceOffsetInBits;
325	}
326
327	// If this slice extracts the entirety of an independent variable from a
328	// larger alloca, do not produce a fragment expression, as the variable is
329	// not fragmented.
330	if (!CurrentFragment) {
331	if (auto Size = Variable->getSizeInBits()) {
332	// Treat the current fragment as covering the whole variable.
333	CurrentFragment = DIExpression::FragmentInfo(*Size, `0`);
334	if (Target == CurrentFragment)
335	return UseNoFrag;
336	}
337	}
338
339	// No additional work to do if there isn't a fragment already, or there is
340	// but it already exactly describes the new assignment.
341	if (!CurrentFragment \|\| *CurrentFragment == Target)
342	return UseFrag;
343
344	// Reject the target fragment if it doesn't fit wholly within the current
345	// fragment. TODO: We could instead chop up the target to fit in the case of
346	// a partial overlap.
347	if (Target.startInBits() < CurrentFragment ->startInBits() \|\|
348	Target.endInBits() > CurrentFragment ->endInBits())
349	return Skip;
350
351	// Target fits within the current fragment, return it.
352	return UseFrag;
353	}
354
355	static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
356	return DebugVariable (DVR->getVariable(), std::nullopt,
357	DVR->getDebugLoc().getInlinedAt());
358	}
359
360	/// Find linked dbg.assign and generate a new one with the correct
361	/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
362	/// value component is copied from the old dbg.assign to the new.
363	/// \param OldAlloca Alloca for the variable before splitting.
364	/// \param IsSplit True if the store (not necessarily alloca)
365	/// is being split.
366	/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
367	/// \param SliceSizeInBits New number of bits being written to.
368	/// \param OldInst Instruction that is being split.
369	/// \param Inst New instruction performing this part of the
370	/// split store.
371	/// \param Dest Store destination.
372	/// \param Value Stored value.
373	/// \param DL Datalayout.
374	static void migrateDebugInfo(AllocaInst OldAlloca, bool* IsSplit,
375	uint64_t OldAllocaOffsetInBits,
376	uint64_t SliceSizeInBits, Instruction *OldInst,
377	Instruction Inst, Value Dest, Value *Value,
378	const DataLayout &DL) {
379	// If we want allocas to be migrated using this helper then we need to ensure
380	// that the BaseFragments map code still works. A simple solution would be
381	// to choose to always clone alloca dbg_assigns (rather than sometimes
382	// "stealing" them).
383	assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
384
385	auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(Inst: OldInst);
386	// Nothing to do if OldInst has no linked dbg.assign intrinsics.
387	if (DVRAssignMarkerRange.empty())
388	return;
389
390	LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
391	LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
392	LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
393	LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
394	<< "\n");
395	LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
396	LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
397	LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
398	LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
399	if (Value)
400	LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
401
402	/// Map of aggregate variables to their fragment associated with OldAlloca.
403	DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>>
404	BaseFragments;
405	for (auto *DVR : at::getDVRAssignmentMarkers(Inst: OldAlloca))
406	BaseFragments [getAggregateVariable(DVR)] =
407	DVR->getExpression()->getFragmentInfo();
408
409	// The new inst needs a DIAssignID unique metadata tag (if OldInst has
410	// one). It shouldn't already have one: assert this assumption.
411	assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
412	DIAssignID NewID = nullptr*;
413	auto &Ctx = Inst->getContext();
414	DIBuilder DIB(OldInst->getModule(), /AllowUnresolved/* false);
415	assert(OldAlloca->isStaticAlloca());
416
417	auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
418	LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
419	<< "\n");
420	auto *Expr = DbgAssign->getExpression();
421	bool SetKillLocation = false;
422
423	if (IsSplit) {
424	std::optional<DIExpression::FragmentInfo> BaseFragment;
425	{
426	auto R = BaseFragments.find(Val: getAggregateVariable(DVR: DbgAssign));
427	if (R == BaseFragments.end())
428	return;
429	BaseFragment = R ->second;
430	}
431	std::optional<DIExpression::FragmentInfo> CurrentFragment =
432	Expr->getFragmentInfo();
433	DIExpression::FragmentInfo NewFragment;
434	FragCalcResult Result = calculateFragment(
435	Variable: DbgAssign->getVariable(), NewStorageSliceOffsetInBits: OldAllocaOffsetInBits, NewStorageSliceSizeInBits: SliceSizeInBits,
436	StorageFragment: BaseFragment, CurrentFragment, Target&: NewFragment);
437
438	if (Result == Skip)
439	return;
440	if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
441	if (CurrentFragment) {
442	// Rewrite NewFragment to be relative to the existing one (this is
443	// what createFragmentExpression wants). CalculateFragment has
444	// already resolved the size for us. FIXME: Should it return the
445	// relative fragment too?
446	NewFragment.OffsetInBits -= CurrentFragment ->OffsetInBits;
447	}
448	// Add the new fragment info to the existing expression if possible.
449	if (auto E = DIExpression::createFragmentExpression(
450	Expr, OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits)) {
451	Expr = *E;
452	} else {
453	// Otherwise, add the new fragment info to an empty expression and
454	// discard the value component of this dbg.assign as the value cannot
455	// be computed with the new fragment.
456	Expr = *DIExpression::createFragmentExpression(
457	Expr: DIExpression::get(Context&: Expr->getContext(), Elements: {}),
458	OffsetInBits: NewFragment.OffsetInBits, SizeInBits: NewFragment.SizeInBits);
459	SetKillLocation = true;
460	}
461	}
462	}
463
464	// If we haven't created a DIAssignID ID do that now and attach it to Inst.
465	if (!NewID) {
466	NewID = DIAssignID::getDistinct(Context&: Ctx);
467	Inst->setMetadata(KindID: LLVMContext::MD_DIAssignID, Node: NewID);
468	}
469
470	DbgVariableRecord *NewAssign;
471	if (IsSplit) {
472	::Value *NewValue = Value ? Value : DbgAssign->getValue();
473	NewAssign = cast<DbgVariableRecord>(Val: DIB.insertDbgAssign(
474	LinkedInstr: Inst, Val: NewValue, SrcVar: DbgAssign->getVariable(), ValExpr: Expr, Addr: Dest,
475	AddrExpr: DIExpression::get(Context&: Expr->getContext(), Elements: {}), DL: DbgAssign->getDebugLoc()));
476	} else {
477	// The store is not split, simply steal the existing dbg_assign.
478	NewAssign = DbgAssign;
479	NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
480	NewAssign->setAddress(Dest);
481	if (Value)
482	NewAssign->replaceVariableLocationOp(OpIdx: `0u`, NewValue: Value);
483	assert(Expr == NewAssign->getExpression());
484	}
485
486	// If we've updated the value but the original dbg.assign has an arglist
487	// then kill it now - we can't use the requested new value.
488	// We can't replace the DIArgList with the new value as it'd leave
489	// the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
490	// an arglist). And we can't keep the DIArgList in case the linked store
491	// is being split - in which case the DIArgList + expression may no longer
492	// be computing the correct value.
493	// This should be a very rare situation as it requires the value being
494	// stored to differ from the dbg.assign (i.e., the value has been
495	// represented differently in the debug intrinsic for some reason).
496	SetKillLocation \|=
497	Value && (DbgAssign->hasArgList() \|\|
498	!DbgAssign->getExpression()->isSingleLocationExpression());
499	if (SetKillLocation)
500	NewAssign->setKillLocation();
501
502	// We could use more precision here at the cost of some additional (code)
503	// complexity - if the original dbg.assign was adjacent to its store, we
504	// could position this new dbg.assign adjacent to its store rather than the
505	// old dbg.assgn. That would result in interleaved dbg.assigns rather than
506	// what we get now:
507	// split store !1
508	// split store !2
509	// dbg.assign !1
510	// dbg.assign !2
511	// This (current behaviour) results results in debug assignments being
512	// noted as slightly offset (in code) from the store. In practice this
513	// should have little effect on the debugging experience due to the fact
514	// that all the split stores should get the same line number.
515	if (NewAssign != DbgAssign) {
516	NewAssign->moveBefore(MoveBefore: DbgAssign->getIterator());
517	NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
518	}
519	LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
520	};
521
522	for_each(Range&: DVRAssignMarkerRange, F: MigrateDbgAssign);
523	}
524
525	namespace {
526
527	/// A custom IRBuilder inserter which prefixes all names, but only in
528	/// Assert builds.
529	class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
530	std::string Prefix;
531
532	Twine getNameWithPrefix(const Twine &Name) const {
533	return Name.isTriviallyEmpty() ? Name : Prefix + Name;
534	}
535
536	public:
537	void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
538
539	void InsertHelper(Instruction I, const* Twine &Name,
540	BasicBlock::iterator InsertPt) const {
541	IRBuilderDefaultInserter::InsertHelper(I, Name: getNameWithPrefix(Name),
542	InsertPt);
543	}
544	};
545
546	/// Provide a type for IRBuilder that drops names in release builds.
547	using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>;
548
549	/// A used slice of an alloca.
550	///
551	/// This structure represents a slice of an alloca used by some instruction. It
552	/// stores both the begin and end offsets of this use, a pointer to the use
553	/// itself, and a flag indicating whether we can classify the use as splittable
554	/// or not when forming partitions of the alloca.
555	class Slice {
556	/// The beginning offset of the range.
557	uint64_t BeginOffset = `0`;
558
559	/// The ending offset, not included in the range.
560	uint64_t EndOffset = `0`;
561
562	/// Storage for both the use of this slice and whether it can be
563	/// split.
564	PointerIntPair<Use , `1`, bool*> UseAndIsSplittable;
565
566	public:
567	Slice() = default;
568
569	Slice(uint64_t BeginOffset, uint64_t EndOffset, Use U, bool* IsSplittable)
570	: BeginOffset(BeginOffset), EndOffset(EndOffset),
571	UseAndIsSplittable (U, IsSplittable) {}
572
573	uint64_t beginOffset() const { return BeginOffset; }
574	uint64_t endOffset() const { return EndOffset; }
575
576	bool isSplittable() const { return UseAndIsSplittable.getInt(); }
577	void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
578
579	Use getUse() const* { return UseAndIsSplittable.getPointer(); }
580
581	bool isDead() const { return getUse() == nullptr; }
582	void kill() { UseAndIsSplittable.setPointer(nullptr); }
583
584	/// Support for ordering ranges.
585	///
586	/// This provides an ordering over ranges such that start offsets are
587	/// always increasing, and within equal start offsets, the end offsets are
588	/// decreasing. Thus the spanning range comes first in a cluster with the
589	/// same start position.
590	bool operator<(const Slice &RHS) const {
591	if (beginOffset() < RHS.beginOffset())
592	return true;
593	if (beginOffset() > RHS.beginOffset())
594	return false;
595	if (isSplittable() != RHS.isSplittable())
596	return !isSplittable();
597	if (endOffset() > RHS.endOffset())
598	return true;
599	return false;
600	}
601
602	/// Support comparison with a single offset to allow binary searches.
603	[[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
604	return LHS.beginOffset() < RHSOffset;
605	}
606	[[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
607	return LHSOffset < RHS.beginOffset();
608	}
609
610	bool operator==(const Slice &RHS) const {
611	return isSplittable() == RHS.isSplittable() &&
612	beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
613	}
614	bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
615	};
616
617	/// Representation of the alloca slices.
618	///
619	/// This class represents the slices of an alloca which are formed by its
620	/// various uses. If a pointer escapes, we can't fully build a representation
621	/// for the slices used and we reflect that in this structure. The uses are
622	/// stored, sorted by increasing beginning offset and with unsplittable slices
623	/// starting at a particular offset before splittable slices.
624	class AllocaSlices {
625	public:
626	/// Construct the slices of a particular alloca.
627	AllocaSlices(const DataLayout &DL, AllocaInst &AI);
628
629	/// Test whether a pointer to the allocation escapes our analysis.
630	///
631	/// If this is true, the slices are never fully built and should be
632	/// ignored.
633	bool isEscaped() const { return PointerEscapingInstr; }
634	bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
635
636	/// Support for iterating over the slices.
637	/// @{
638	using iterator = SmallVectorImpl<Slice>::iterator;
639	using range = iterator_range<iterator>;
640
641	iterator begin() { return Slices.begin(); }
642	iterator end() { return Slices.end(); }
643
644	using const_iterator = SmallVectorImpl<Slice>::const_iterator;
645	using const_range = iterator_range<const_iterator>;
646
647	const_iterator begin() const { return Slices.begin(); }
648	const_iterator end() const { return Slices.end(); }
649	/// @}
650
651	/// Erase a range of slices.
652	void erase(iterator Start, iterator Stop) { Slices.erase(CS: Start, CE: Stop); }
653
654	/// Insert new slices for this alloca.
655	///
656	/// This moves the slices into the alloca's slices collection, and re-sorts
657	/// everything so that the usual ordering properties of the alloca's slices
658	/// hold.
659	void insert(ArrayRef<Slice> NewSlices) {
660	int OldSize = Slices.size();
661	Slices.append(in_start: NewSlices.begin(), in_end: NewSlices.end());
662	auto SliceI = Slices.begin() + OldSize;
663	std::stable_sort(first: SliceI, last: Slices.end());
664	std::inplace_merge(first: Slices.begin(), middle: SliceI, last: Slices.end());
665	}
666
667	// Forward declare the iterator and range accessor for walking the
668	// partitions.
669	class partition_iterator;
670	iterator_range<partition_iterator> partitions();
671
672	/// Access the dead users for this alloca.
673	ArrayRef<Instruction > getDeadUsers() const* { return DeadUsers; }
674
675	/// Access Uses that should be dropped if the alloca is promotable.
676	ArrayRef<Use > getDeadUsesIfPromotable() const* {
677	return DeadUseIfPromotable;
678	}
679
680	/// Access the dead operands referring to this alloca.
681	///
682	/// These are operands which have cannot actually be used to refer to the
683	/// alloca as they are outside its range and the user doesn't correct for
684	/// that. These mostly consist of PHI node inputs and the like which we just
685	/// need to replace with undef.
686	ArrayRef<Use > getDeadOperands() const* { return DeadOperands; }
687
688	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
689	void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
690	void printSlice(raw_ostream &OS, const_iterator I,
691	StringRef Indent = " ") const;
692	void printUse(raw_ostream &OS, const_iterator I,
693	StringRef Indent = " ") const;
694	void print(raw_ostream &OS) const;
695	void dump(const_iterator I) const;
696	void dump() const;
697	#endif
698
699	private:
700	template <typename DerivedT, typename RetT = void> class BuilderBase;
701	class SliceBuilder;
702
703	friend class AllocaSlices::SliceBuilder;
704
705	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
706	/// Handle to alloca instruction to simplify method interfaces.
707	AllocaInst &AI;
708	#endif
709
710	/// The instruction responsible for this alloca not having a known set
711	/// of slices.
712	///
713	/// When an instruction (potentially) escapes the pointer to the alloca, we
714	/// store a pointer to that here and abort trying to form slices of the
715	/// alloca. This will be null if the alloca slices are analyzed successfully.
716	Instruction *PointerEscapingInstr;
717	Instruction *PointerEscapingInstrReadOnly;
718
719	/// The slices of the alloca.
720	///
721	/// We store a vector of the slices formed by uses of the alloca here. This
722	/// vector is sorted by increasing begin offset, and then the unsplittable
723	/// slices before the splittable ones. See the Slice inner class for more
724	/// details.
725	SmallVector<Slice, `8`> Slices;
726
727	/// Instructions which will become dead if we rewrite the alloca.
728	///
729	/// Note that these are not separated by slice. This is because we expect an
730	/// alloca to be completely rewritten or not rewritten at all. If rewritten,
731	/// all these instructions can simply be removed and replaced with poison as
732	/// they come from outside of the allocated space.
733	SmallVector<Instruction *, `8`> DeadUsers;
734
735	/// Uses which will become dead if can promote the alloca.
736	SmallVector<Use *, `8`> DeadUseIfPromotable;
737
738	/// Operands which will become dead if we rewrite the alloca.
739	///
740	/// These are operands that in their particular use can be replaced with
741	/// poison when we rewrite the alloca. These show up in out-of-bounds inputs
742	/// to PHI nodes and the like. They aren't entirely dead (there might be
743	/// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
744	/// want to swap this particular input for poison to simplify the use lists of
745	/// the alloca.
746	SmallVector<Use *, `8`> DeadOperands;
747	};
748
749	/// A partition of the slices.
750	///
751	/// An ephemeral representation for a range of slices which can be viewed as
752	/// a partition of the alloca. This range represents a span of the alloca's
753	/// memory which cannot be split, and provides access to all of the slices
754	/// overlapping some part of the partition.
755	///
756	/// Objects of this type are produced by traversing the alloca's slices, but
757	/// are only ephemeral and not persistent.
758	class Partition {
759	private:
760	friend class AllocaSlices;
761	friend class AllocaSlices::partition_iterator;
762
763	using iterator = AllocaSlices::iterator;
764
765	/// The beginning and ending offsets of the alloca for this
766	/// partition.
767	uint64_t BeginOffset = `0`, EndOffset = `0`;
768
769	/// The start and end iterators of this partition.
770	iterator SI, SJ;
771
772	/// A collection of split slice tails overlapping the partition.
773	SmallVector<Slice *, `4`> SplitTails;
774
775	/// Raw constructor builds an empty partition starting and ending at
776	/// the given iterator.
777	Partition(iterator SI) : SI(SI), SJ(SI) {}
778
779	public:
780	/// The start offset of this partition.
781	///
782	/// All of the contained slices start at or after this offset.
783	uint64_t beginOffset() const { return BeginOffset; }
784
785	/// The end offset of this partition.
786	///
787	/// All of the contained slices end at or before this offset.
788	uint64_t endOffset() const { return EndOffset; }
789
790	/// The size of the partition.
791	///
792	/// Note that this can never be zero.
793	uint64_t size() const {
794	assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
795	return EndOffset - BeginOffset;
796	}
797
798	/// Test whether this partition contains no slices, and merely spans
799	/// a region occupied by split slices.
800	bool empty() const { return SI == SJ; }
801
802	/// \name Iterate slices that start within the partition.
803	/// These may be splittable or unsplittable. They have a begin offset >= the
804	/// partition begin offset.
805	/// @{
806	// FIXME: We should probably define a "concat_iterator" helper and use that
807	// to stitch together pointee_iterators over the split tails and the
808	// contiguous iterators of the partition. That would give a much nicer
809	// interface here. We could then additionally expose filtered iterators for
810	// split, unsplit, and unsplittable splices based on the usage patterns.
811	iterator begin() const { return SI; }
812	iterator end() const { return SJ; }
813	/// @}
814
815	/// Get the sequence of split slice tails.
816	///
817	/// These tails are of slices which start before this partition but are
818	/// split and overlap into the partition. We accumulate these while forming
819	/// partitions.
820	ArrayRef<Slice > splitSliceTails() const* { return SplitTails; }
821	};
822
823	} // end anonymous namespace
824
825	/// An iterator over partitions of the alloca's slices.
826	///
827	/// This iterator implements the core algorithm for partitioning the alloca's
828	/// slices. It is a forward iterator as we don't support backtracking for
829	/// efficiency reasons, and re-use a single storage area to maintain the
830	/// current set of split slices.
831	///
832	/// It is templated on the slice iterator type to use so that it can operate
833	/// with either const or non-const slice iterators.
834	class AllocaSlices::partition_iterator
835	: public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
836	Partition> {
837	friend class AllocaSlices;
838
839	/// Most of the state for walking the partitions is held in a class
840	/// with a nice interface for examining them.
841	Partition P;
842
843	/// We need to keep the end of the slices to know when to stop.
844	AllocaSlices::iterator SE;
845
846	/// We also need to keep track of the maximum split end offset seen.
847	/// FIXME: Do we really?
848	uint64_t MaxSplitSliceEndOffset = `0`;
849
850	/// Sets the partition to be empty at given iterator, and sets the
851	/// end iterator.
852	partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
853	: P (SI), SE(SE) {
854	// If not already at the end, advance our state to form the initial
855	// partition.
856	if (SI != SE)
857	advance();
858	}
859
860	/// Advance the iterator to the next partition.
861	///
862	/// Requires that the iterator not be at the end of the slices.
863	void advance() {
864	assert((P.SI != SE \|\| !P.SplitTails.empty()) &&
865	"Cannot advance past the end of the slices!");
866
867	// Clear out any split uses which have ended.
868	if (!P.SplitTails.empty()) {
869	if (P.EndOffset >= MaxSplitSliceEndOffset) {
870	// If we've finished all splits, this is easy.
871	P.SplitTails.clear();
872	MaxSplitSliceEndOffset = `0`;
873	} else {
874	// Remove the uses which have ended in the prior partition. This
875	// cannot change the max split slice end because we just checked that
876	// the prior partition ended prior to that max.
877	llvm::erase_if(C&: P.SplitTails,
878	P: [&](Slice S) { return* S->endOffset() <= P.EndOffset; });
879	assert(llvm::any_of(P.SplitTails,
880	[&](Slice *S) {
881	return S->endOffset() == MaxSplitSliceEndOffset;
882	}) &&
883	"Could not find the current max split slice offset!");
884	assert(llvm::all_of(P.SplitTails,
885	[&](Slice *S) {
886	return S->endOffset() <= MaxSplitSliceEndOffset;
887	}) &&
888	"Max split slice end offset is not actually the max!");
889	}
890	}
891
892	// If P.SI is already at the end, then we've cleared the split tail and
893	// now have an end iterator.
894	if (P.SI == SE) {
895	assert(P.SplitTails.empty() && "Failed to clear the split slices!");
896	return;
897	}
898
899	// If we had a non-empty partition previously, set up the state for
900	// subsequent partitions.
901	if (P.SI != P.SJ) {
902	// Accumulate all the splittable slices which started in the old
903	// partition into the split list.
904	for (Slice &S : P)
905	if (S.isSplittable() && S.endOffset() > P.EndOffset) {
906	P.SplitTails.push_back(Elt: &S);
907	MaxSplitSliceEndOffset =
908	std::max(a: S.endOffset(), b: MaxSplitSliceEndOffset);
909	}
910
911	// Start from the end of the previous partition.
912	P.SI = P.SJ;
913
914	// If P.SI is now at the end, we at most have a tail of split slices.
915	if (P.SI == SE) {
916	P.BeginOffset = P.EndOffset;
917	P.EndOffset = MaxSplitSliceEndOffset;
918	return;
919	}
920
921	// If the we have split slices and the next slice is after a gap and is
922	// not splittable immediately form an empty partition for the split
923	// slices up until the next slice begins.
924	if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
925	!P.SI->isSplittable()) {
926	P.BeginOffset = P.EndOffset;
927	P.EndOffset = P.SI->beginOffset();
928	return;
929	}
930	}
931
932	// OK, we need to consume new slices. Set the end offset based on the
933	// current slice, and step SJ past it. The beginning offset of the
934	// partition is the beginning offset of the next slice unless we have
935	// pre-existing split slices that are continuing, in which case we begin
936	// at the prior end offset.
937	P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
938	P.EndOffset = P.SI->endOffset();
939	++P.SJ;
940
941	// There are two strategies to form a partition based on whether the
942	// partition starts with an unsplittable slice or a splittable slice.
943	if (!P.SI->isSplittable()) {
944	// When we're forming an unsplittable region, it must always start at
945	// the first slice and will extend through its end.
946	assert(P.BeginOffset == P.SI->beginOffset());
947
948	// Form a partition including all of the overlapping slices with this
949	// unsplittable slice.
950	while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
951	if (!P.SJ->isSplittable())
952	P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
953	++P.SJ;
954	}
955
956	// We have a partition across a set of overlapping unsplittable
957	// partitions.
958	return;
959	}
960
961	// If we're starting with a splittable slice, then we need to form
962	// a synthetic partition spanning it and any other overlapping splittable
963	// splices.
964	assert(P.SI->isSplittable() && "Forming a splittable partition!");
965
966	// Collect all of the overlapping splittable slices.
967	while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
968	P.SJ->isSplittable()) {
969	P.EndOffset = std::max(a: P.EndOffset, b: P.SJ->endOffset());
970	++P.SJ;
971	}
972
973	// Back upiP.EndOffset if we ended the span early when encountering an
974	// unsplittable slice. This synthesizes the early end offset of
975	// a partition spanning only splittable slices.
976	if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
977	assert(!P.SJ->isSplittable());
978	P.EndOffset = P.SJ->beginOffset();
979	}
980	}
981
982	public:
983	bool operator==(const partition_iterator &RHS) const {
984	assert(SE == RHS.SE &&
985	"End iterators don't match between compared partition iterators!");
986
987	// The observed positions of partitions is marked by the P.SI iterator and
988	// the emptiness of the split slices. The latter is only relevant when
989	// P.SI == SE, as the end iterator will additionally have an empty split
990	// slices list, but the prior may have the same P.SI and a tail of split
991	// slices.
992	if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
993	assert(P.SJ == RHS.P.SJ &&
994	"Same set of slices formed two different sized partitions!");
995	assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
996	"Same slice position with differently sized non-empty split "
997	"slice tails!");
998	return true;
999	}
1000	return false;
1001	}
1002
1003	partition_iterator &operator++() {
1004	advance();
1005	return *this;
1006	}
1007
1008	Partition &operator() { return* P; }
1009	};
1010
1011	/// A forward range over the partitions of the alloca's slices.
1012	///
1013	/// This accesses an iterator range over the partitions of the alloca's
1014	/// slices. It computes these partitions on the fly based on the overlapping
1015	/// offsets of the slices and the ability to split them. It will visit "empty"
1016	/// partitions to cover regions of the alloca only accessed via split
1017	/// slices.
1018	iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
1019	return make_range(x: partition_iterator (begin(), end()),
1020	y: partition_iterator (end(), end()));
1021	}
1022
1023	static Value *foldSelectInst(SelectInst &SI) {
1024	// If the condition being selected on is a constant or the same value is
1025	// being selected between, fold the select. Yes this does (rarely) happen
1026	// early on.
1027	if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: SI.getCondition()))
1028	return SI.getOperand(i_nocapture: `1` + CI->isZero());
1029	if (SI.getOperand(i_nocapture: `1`) == SI.getOperand(i_nocapture: `2`))
1030	return SI.getOperand(i_nocapture: `1`);
1031
1032	return nullptr;
1033	}
1034
1035	/// A helper that folds a PHI node or a select.
1036	static Value *foldPHINodeOrSelectInst(Instruction &I) {
1037	if (PHINode *PN = dyn_cast<PHINode>(Val: &I)) {
1038	// If PN merges together the same value, return that value.
1039	return PN->hasConstantValue();
1040	}
1041	return foldSelectInst(SI&: cast<SelectInst>(Val&: I));
1042	}
1043
1044	/// Builder for the alloca slices.
1045	///
1046	/// This class builds a set of alloca slices by recursively visiting the uses
1047	/// of an alloca and making a slice for each load and store at each offset.
1048	class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1049	friend class PtrUseVisitor<SliceBuilder>;
1050	friend class InstVisitor<SliceBuilder>;
1051
1052	using Base = PtrUseVisitor<SliceBuilder>;
1053
1054	const uint64_t AllocSize;
1055	AllocaSlices &AS;
1056
1057	SmallDenseMap<Instruction , unsigned*> MemTransferSliceMap;
1058	SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes;
1059
1060	/// Set to de-duplicate dead instructions found in the use walk.
1061	SmallPtrSet<Instruction *, `4`> VisitedDeadInsts;
1062
1063	public:
1064	SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1065	: PtrUseVisitor<SliceBuilder>(DL),
1066	AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1067
1068	private:
1069	void markAsDead(Instruction &I) {
1070	if (VisitedDeadInsts.insert(Ptr: &I).second)
1071	AS.DeadUsers.push_back(Elt: &I);
1072	}
1073
1074	void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1075	bool IsSplittable = false) {
1076	// Completely skip uses which have a zero size or start either before or
1077	// past the end of the allocation.
1078	if (Size == `0` \|\| Offset.uge(RHS: AllocSize)) {
1079	LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1080	<< Offset
1081	<< " which has zero size or starts outside of the "
1082	<< AllocSize << " byte alloca:\n"
1083	<< " alloca: " << AS.AI << "\n"
1084	<< " use: " << I << "\n");
1085	return markAsDead(I);
1086	}
1087
1088	uint64_t BeginOffset = Offset.getZExtValue();
1089	uint64_t EndOffset = BeginOffset + Size;
1090
1091	// Clamp the end offset to the end of the allocation. Note that this is
1092	// formulated to handle even the case where "BeginOffset + Size" overflows.
1093	// This may appear superficially to be something we could ignore entirely,
1094	// but that is not so! There may be widened loads or PHI-node uses where
1095	// some instructions are dead but not others. We can't completely ignore
1096	// them, and so have to record at least the information here.
1097	assert(AllocSize >= BeginOffset); // Established above.
1098	if (Size > AllocSize - BeginOffset) {
1099	LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1100	<< Offset << " to remain within the " << AllocSize
1101	<< " byte alloca:\n"
1102	<< " alloca: " << AS.AI << "\n"
1103	<< " use: " << I << "\n");
1104	EndOffset = AllocSize;
1105	}
1106
1107	AS.Slices.push_back(Elt: Slice (BeginOffset, EndOffset, U, IsSplittable));
1108	}
1109
1110	void visitBitCastInst(BitCastInst &BC) {
1111	if (BC.use_empty())
1112	return markAsDead(I&: BC);
1113
1114	return Base::visitBitCastInst(BC);
1115	}
1116
1117	void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1118	if (ASC.use_empty())
1119	return markAsDead(I&: ASC);
1120
1121	return Base::visitAddrSpaceCastInst(ASC);
1122	}
1123
1124	void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1125	if (GEPI.use_empty())
1126	return markAsDead(I&: GEPI);
1127
1128	return Base::visitGetElementPtrInst(GEPI);
1129	}
1130
1131	void handleLoadOrStore(Type Ty, Instruction &I, const* APInt &Offset,
1132	uint64_t Size, bool IsVolatile) {
1133	// We allow splitting of non-volatile loads and stores where the type is an
1134	// integer type. These may be used to implement 'memcpy' or other "transfer
1135	// of bits" patterns.
1136	bool IsSplittable =
1137	Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1138
1139	insertUse(I, Offset, Size, IsSplittable);
1140	}
1141
1142	void visitLoadInst(LoadInst &LI) {
1143	assert((!LI.isSimple() \|\| LI.getType()->isSingleValueType()) &&
1144	"All simple FCA loads should have been pre-split");
1145
1146	// If there is a load with an unknown offset, we can still perform store
1147	// to load forwarding for other known-offset loads.
1148	if (!IsOffsetKnown)
1149	return PI.setEscapedReadOnly(&LI);
1150
1151	TypeSize Size = DL.getTypeStoreSize(Ty: LI.getType());
1152	if (Size.isScalable()) {
1153	unsigned VScale = LI.getFunction()->getVScaleValue();
1154	if (!VScale)
1155	return PI.setAborted(&LI);
1156
1157	Size = TypeSize::getFixed(ExactSize: Size.getKnownMinValue() * VScale);
1158	}
1159
1160	return handleLoadOrStore(Ty: LI.getType(), I&: LI, Offset, Size: Size.getFixedValue(),
1161	IsVolatile: LI.isVolatile());
1162	}
1163
1164	void visitStoreInst(StoreInst &SI) {
1165	Value *ValOp = SI.getValueOperand();
1166	if (ValOp == *U)
1167	return PI.setEscapedAndAborted(&SI);
1168	if (!IsOffsetKnown)
1169	return PI.setAborted(&SI);
1170
1171	TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValOp->getType());
1172	if (StoreSize.isScalable()) {
1173	unsigned VScale = SI.getFunction()->getVScaleValue();
1174	if (!VScale)
1175	return PI.setAborted(&SI);
1176
1177	StoreSize = TypeSize::getFixed(ExactSize: StoreSize.getKnownMinValue() * VScale);
1178	}
1179
1180	uint64_t Size = StoreSize.getFixedValue();
1181
1182	// If this memory access can be shown to statically* extend outside the*
1183	// bounds of the allocation, it's behavior is undefined, so simply
1184	// ignore it. Note that this is more strict than the generic clamping
1185	// behavior of insertUse. We also try to handle cases which might run the
1186	// risk of overflow.
1187	// FIXME: We should instead consider the pointer to have escaped if this
1188	// function is being instrumented for addressing bugs or race conditions.
1189	if (Size > AllocSize \|\| Offset.ugt(RHS: AllocSize - Size)) {
1190	LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1191	<< Offset << " which extends past the end of the "
1192	<< AllocSize << " byte alloca:\n"
1193	<< " alloca: " << AS.AI << "\n"
1194	<< " use: " << SI << "\n");
1195	return markAsDead(I&: SI);
1196	}
1197
1198	assert((!SI.isSimple() \|\| ValOp->getType()->isSingleValueType()) &&
1199	"All simple FCA stores should have been pre-split");
1200	handleLoadOrStore(Ty: ValOp->getType(), I&: SI, Offset, Size, IsVolatile: SI.isVolatile());
1201	}
1202
1203	void visitMemSetInst(MemSetInst &II) {
1204	assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1205	ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1206	if ((Length && Length->getValue() == `0`) \|\|
1207	(IsOffsetKnown && Offset.uge(RHS: AllocSize)))
1208	// Zero-length mem transfer intrinsics can be ignored entirely.
1209	return markAsDead(I&: II);
1210
1211	if (!IsOffsetKnown)
1212	return PI.setAborted(&II);
1213
1214	insertUse(I&: II, Offset,
1215	Size: Length ? Length->getLimitedValue()
1216	: AllocSize - Offset.getLimitedValue(),
1217	IsSplittable: (bool)Length);
1218	}
1219
1220	void visitMemTransferInst(MemTransferInst &II) {
1221	ConstantInt *Length = dyn_cast<ConstantInt>(Val: II.getLength());
1222	if (Length && Length->getValue() == `0`)
1223	// Zero-length mem transfer intrinsics can be ignored entirely.
1224	return markAsDead(I&: II);
1225
1226	// Because we can visit these intrinsics twice, also check to see if the
1227	// first time marked this instruction as dead. If so, skip it.
1228	if (VisitedDeadInsts.count(Ptr: &II))
1229	return;
1230
1231	if (!IsOffsetKnown)
1232	return PI.setAborted(&II);
1233
1234	// This side of the transfer is completely out-of-bounds, and so we can
1235	// nuke the entire transfer. However, we also need to nuke the other side
1236	// if already added to our partitions.
1237	// FIXME: Yet another place we really should bypass this when
1238	// instrumenting for ASan.
1239	if (Offset.uge(RHS: AllocSize)) {
1240	auto MTPI = MemTransferSliceMap.find(Val: &II);
1241	if (MTPI != MemTransferSliceMap.end())
1242	AS.Slices [MTPI ->second].kill();
1243	return markAsDead(I&: II);
1244	}
1245
1246	uint64_t RawOffset = Offset.getLimitedValue();
1247	uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1248
1249	// Check for the special case where the same exact value is used for both
1250	// source and dest.
1251	if (U == II.getRawDest() && U == II.getRawSource()) {
1252	// For non-volatile transfers this is a no-op.
1253	if (!II.isVolatile())
1254	return markAsDead(I&: II);
1255
1256	return insertUse(I&: II, Offset, Size, /IsSplittable=/false);
1257	}
1258
1259	// If we have seen both source and destination for a mem transfer, then
1260	// they both point to the same alloca.
1261	bool Inserted;
1262	SmallDenseMap<Instruction , unsigned*>::iterator MTPI;
1263	std::tie(args&: MTPI, args&: Inserted) =
1264	MemTransferSliceMap.insert(KV: std::make_pair(x: &II, y: AS.Slices.size()));
1265	unsigned PrevIdx = MTPI ->second;
1266	if (!Inserted) {
1267	Slice &PrevP = AS.Slices [PrevIdx];
1268
1269	// Check if the begin offsets match and this is a non-volatile transfer.
1270	// In that case, we can completely elide the transfer.
1271	if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1272	PrevP.kill();
1273	return markAsDead(I&: II);
1274	}
1275
1276	// Otherwise we have an offset transfer within the same alloca. We can't
1277	// split those.
1278	PrevP.makeUnsplittable();
1279	}
1280
1281	// Insert the use now that we've fixed up the splittable nature.
1282	insertUse(I&: II, Offset, Size, /IsSplittable=/Inserted && Length);
1283
1284	// Check that we ended up with a valid index in the map.
1285	assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1286	"Map index doesn't point back to a slice with this user.");
1287	}
1288
1289	// Disable SRoA for any intrinsics except for lifetime invariants.
1290	// FIXME: What about debug intrinsics? This matches old behavior, but
1291	// doesn't make sense.
1292	void visitIntrinsicInst(IntrinsicInst &II) {
1293	if (II.isDroppable()) {
1294	AS.DeadUseIfPromotable.push_back(Elt: U);
1295	return;
1296	}
1297
1298	if (!IsOffsetKnown)
1299	return PI.setAborted(&II);
1300
1301	if (II.isLifetimeStartOrEnd()) {
1302	insertUse(I&: II, Offset, Size: AllocSize, IsSplittable: true);
1303	return;
1304	}
1305
1306	Base::visitIntrinsicInst(II);
1307	}
1308
1309	Instruction hasUnsafePHIOrSelectUse(Instruction Root, uint64_t &Size) {
1310	// We consider any PHI or select that results in a direct load or store of
1311	// the same offset to be a viable use for slicing purposes. These uses
1312	// are considered unsplittable and the size is the maximum loaded or stored
1313	// size.
1314	SmallPtrSet<Instruction *, `4`> Visited;
1315	SmallVector<std::pair<Instruction , Instruction >, `4`> Uses;
1316	Visited.insert(Ptr: Root);
1317	Uses.push_back(Elt: std::make_pair(x: cast<Instruction>(Val&: *U), y&: Root));
1318	const DataLayout &DL = Root->getDataLayout();
1319	// If there are no loads or stores, the access is dead. We mark that as
1320	// a size zero access.
1321	Size = `0`;
1322	do {
1323	Instruction I, UsedI;
1324	std::tie(args&: UsedI, args&: I) = Uses.pop_back_val();
1325
1326	if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
1327	TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
1328	if (LoadSize.isScalable()) {
1329	PI.setAborted(LI);
1330	return nullptr;
1331	}
1332	Size = std::max(a: Size, b: LoadSize.getFixedValue());
1333	continue;
1334	}
1335	if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
1336	Value *Op = SI->getOperand(i_nocapture: `0`);
1337	if (Op == UsedI)
1338	return SI;
1339	TypeSize StoreSize = DL.getTypeStoreSize(Ty: Op->getType());
1340	if (StoreSize.isScalable()) {
1341	PI.setAborted(SI);
1342	return nullptr;
1343	}
1344	Size = std::max(a: Size, b: StoreSize.getFixedValue());
1345	continue;
1346	}
1347
1348	if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
1349	if (!GEP->hasAllZeroIndices())
1350	return GEP;
1351	} else if (!isa<BitCastInst>(Val: I) && !isa<PHINode>(Val: I) &&
1352	!isa<SelectInst>(Val: I) && !isa<AddrSpaceCastInst>(Val: I)) {
1353	return I;
1354	}
1355
1356	for (User *U : I->users())
1357	if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
1358	Uses.push_back(Elt: std::make_pair(x&: I, y: cast<Instruction>(Val: U)));
1359	} while (!Uses.empty());
1360
1361	return nullptr;
1362	}
1363
1364	void visitPHINodeOrSelectInst(Instruction &I) {
1365	assert(isa<PHINode>(I) \|\| isa<SelectInst>(I));
1366	if (I.use_empty())
1367	return markAsDead(I);
1368
1369	// If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1370	// instructions in this BB, which may be required during rewriting. Bail out
1371	// on these cases.
1372	if (isa<PHINode>(Val: I) && !I.getParent()->hasInsertionPt())
1373	return PI.setAborted(&I);
1374
1375	// TODO: We could use simplifyInstruction here to fold PHINodes and
1376	// SelectInsts. However, doing so requires to change the current
1377	// dead-operand-tracking mechanism. For instance, suppose neither loading
1378	// from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1379	// trap either. However, if we simply replace %U with undef using the
1380	// current dead-operand-tracking mechanism, "load (select undef, undef,
1381	// %other)" may trap because the select may return the first operand
1382	// "undef".
1383	if (Value *Result = foldPHINodeOrSelectInst(I)) {
1384	if (Result == *U)
1385	// If the result of the constant fold will be the pointer, recurse
1386	// through the PHI/select as if we had RAUW'ed it.
1387	enqueueUsers(I);
1388	else
1389	// Otherwise the operand to the PHI/select is dead, and we can replace
1390	// it with poison.
1391	AS.DeadOperands.push_back(Elt: U);
1392
1393	return;
1394	}
1395
1396	if (!IsOffsetKnown)
1397	return PI.setAborted(&I);
1398
1399	// See if we already have computed info on this node.
1400	uint64_t &Size = PHIOrSelectSizes [&I];
1401	if (!Size) {
1402	// This is a new PHI/Select, check for an unsafe use of it.
1403	if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(Root: &I, Size))
1404	return PI.setAborted(UnsafeI);
1405	}
1406
1407	// For PHI and select operands outside the alloca, we can't nuke the entire
1408	// phi or select -- the other side might still be relevant, so we special
1409	// case them here and use a separate structure to track the operands
1410	// themselves which should be replaced with poison.
1411	// FIXME: This should instead be escaped in the event we're instrumenting
1412	// for address sanitization.
1413	if (Offset.uge(RHS: AllocSize)) {
1414	AS.DeadOperands.push_back(Elt: U);
1415	return;
1416	}
1417
1418	insertUse(I, Offset, Size);
1419	}
1420
1421	void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(I&: PN); }
1422
1423	void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(I&: SI); }
1424
1425	/// Disable SROA entirely if there are unhandled users of the alloca.
1426	void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1427
1428	void visitCallBase(CallBase &CB) {
1429	// If the call operand is read-only and only does a read-only or address
1430	// capture, then we mark it as EscapedReadOnly.
1431	if (CB.isDataOperand(U) &&
1432	!capturesFullProvenance(CC: CB.getCaptureInfo(OpNo: U->getOperandNo())) &&
1433	CB.onlyReadsMemory(OpNo: U->getOperandNo())) {
1434	PI.setEscapedReadOnly(&CB);
1435	return;
1436	}
1437
1438	Base::visitCallBase(CB);
1439	}
1440	};
1441
1442	AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1443	:
1444	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1445	AI(AI),
1446	#endif
1447	PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1448	SliceBuilder PB(DL, AI, *this);
1449	SliceBuilder::PtrInfo PtrI = PB.visitPtr(I&: AI);
1450	if (PtrI.isEscaped() \|\| PtrI.isAborted()) {
1451	// FIXME: We should sink the escape vs. abort info into the caller nicely,
1452	// possibly by just storing the PtrInfo in the AllocaSlices.
1453	PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1454	: PtrI.getAbortingInst();
1455	assert(PointerEscapingInstr && "Did not track a bad instruction");
1456	return;
1457	}
1458	PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1459
1460	llvm::erase_if(C&: Slices, P: [](const Slice &S) { return S.isDead(); });
1461
1462	// Sort the uses. This arranges for the offsets to be in ascending order,
1463	// and the sizes to be in descending order.
1464	llvm::stable_sort(Range&: Slices);
1465	}
1466
1467	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1468
1469	void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1470	StringRef Indent) const {
1471	printSlice(OS, I, Indent);
1472	OS << "\n";
1473	printUse(OS, I, Indent);
1474	}
1475
1476	void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1477	StringRef Indent) const {
1478	OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1479	<< " slice #" << (I - begin())
1480	<< (I->isSplittable() ? " (splittable)" : "");
1481	}
1482
1483	void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1484	StringRef Indent) const {
1485	OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1486	}
1487
1488	void AllocaSlices::print(raw_ostream &OS) const {
1489	if (PointerEscapingInstr) {
1490	OS << "Can't analyze slices for alloca: " << AI << "\n"
1491	<< " A pointer to this alloca escaped by:\n"
1492	<< " " << *PointerEscapingInstr << "\n";
1493	return;
1494	}
1495
1496	if (PointerEscapingInstrReadOnly)
1497	OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1498
1499	OS << "Slices of alloca: " << AI << "\n";
1500	for (const_iterator I = begin(), E = end(); I != E; ++I)
1501	print(OS, I);
1502	}
1503
1504	LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1505	print(dbgs(), I);
1506	}
1507	LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1508
1509	#endif // !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1510
1511	/// Find a common load/store type used through a pointer PHI or select.
1512	///
1513	/// Look through a PHI or select to see if all of its users are loads or stores
1514	/// of one common type. Whether those accesses can be speculated does not affect
1515	/// the type they use and is checked separately when attempting promotion.
1516	static Type *findCommonTypeThroughPHIOrSelect(Instruction &I) {
1517	assert((isa<PHINode, SelectInst>(I)) && "expected a PHI or select");
1518	Type Ty = nullptr*;
1519
1520	for (User *U : I.users()) {
1521	Type UserTy = nullptr*;
1522	if (auto *LI = dyn_cast<LoadInst>(Val: U))
1523	UserTy = LI->getType();
1524	else if (auto *Store = dyn_cast<StoreInst>(Val: U))
1525	// Slice building rejects stores of the PHI-or-select-derived pointer, so
1526	// it must be the store's pointer operand here.
1527	UserTy = Store->getValueOperand()->getType();
1528
1529	if (!UserTy \|\| (Ty && Ty != UserTy))
1530	return nullptr;
1531	Ty = UserTy;
1532	}
1533
1534	return Ty;
1535	}
1536
1537	/// Walk the range of a partitioning looking for a common type to cover this
1538	/// sequence of slices.
1539	static std::pair<Type , IntegerType >
1540	findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1541	uint64_t EndOffset) {
1542	Type Ty = nullptr*;
1543	bool TyIsCommon = true;
1544	IntegerType ITy = nullptr*;
1545
1546	// Note that we need to look at every* alloca slice's Use to ensure we*
1547	// always get consistent results regardless of the order of slices.
1548	for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1549	Use *U = I->getUse();
1550	if (isa<IntrinsicInst>(Val: *U->getUser()))
1551	continue;
1552	if (I->beginOffset() != B->beginOffset() \|\| I->endOffset() != EndOffset)
1553	continue;
1554
1555	Type UserTy = nullptr*;
1556	if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
1557	UserTy = LI->getType();
1558	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
1559	UserTy = SI->getValueOperand()->getType();
1560	} else if (isa<PHINode, SelectInst>(Val: U->getUser())) {
1561	UserTy =
1562	findCommonTypeThroughPHIOrSelect(I&: *cast<Instruction>(Val: U->getUser()));
1563	}
1564
1565	if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(Val: UserTy)) {
1566	// If the type is larger than the partition, skip it. We only encounter
1567	// this for split integer operations where we want to use the type of the
1568	// entity causing the split. Also skip if the type is not a byte width
1569	// multiple.
1570	if (UserITy->getBitWidth() % `8` != `0` \|\|
1571	UserITy->getBitWidth() / `8` > (EndOffset - B->beginOffset()))
1572	continue;
1573
1574	// Track the largest bitwidth integer type used in this way in case there
1575	// is no common type.
1576	if (!ITy \|\| ITy->getBitWidth() < UserITy->getBitWidth())
1577	ITy = UserITy;
1578	}
1579
1580	// To avoid depending on the order of slices, Ty and TyIsCommon must not
1581	// depend on types skipped above.
1582	if (!UserTy \|\| (Ty && Ty != UserTy))
1583	TyIsCommon = false; // Give up on anything but an iN type.
1584	else
1585	Ty = UserTy;
1586	}
1587
1588	return {TyIsCommon ? Ty : nullptr, ITy};
1589	}
1590
1591	/// PHI instructions that use an alloca and are subsequently loaded can be
1592	/// rewritten to load both input pointers in the pred blocks and then PHI the
1593	/// results, allowing the load of the alloca to be promoted.
1594	/// From this:
1595	/// %P2 = phi [i32 %Alloca, i32* %Other]*
1596	/// %V = load i32 %P2*
1597	/// to:
1598	/// %V1 = load i32 %Alloca -> will be mem2reg'd*
1599	/// ...
1600	/// %V2 = load i32 %Other*
1601	/// ...
1602	/// %V = phi [i32 %V1, i32 %V2]
1603	///
1604	/// We can do this to a PHI if its only uses are loads and if any loads moved
1605	/// across other outgoing edges can be executed unconditionally.
1606	///
1607	/// FIXME: This should be hoisted into a generic utility, likely in
1608	/// Transforms/Util/Local.h
1609	static bool isSafePHIToSpeculate(PHINode &PN) {
1610	const DataLayout &DL = PN.getDataLayout();
1611
1612	// For now, we can only do this promotion if the load is in the same block
1613	// as the PHI, and if there are no stores between the phi and load.
1614	// TODO: Allow recursive phi users.
1615	BasicBlock *BB = PN.getParent();
1616	Align MaxAlign;
1617	uint64_t APWidth = DL.getIndexTypeSizeInBits(Ty: PN.getType());
1618	Type LoadType = nullptr*;
1619	for (User *U : PN.users()) {
1620	LoadInst *LI = dyn_cast<LoadInst>(Val: U);
1621	if (!LI \|\| !LI->isSimple())
1622	return false;
1623
1624	// For now we only allow loads in the same block as the PHI. This is
1625	// a common case that happens when instcombine merges two loads through
1626	// a PHI.
1627	if (LI->getParent() != BB)
1628	return false;
1629
1630	if (LoadType) {
1631	if (LoadType != LI->getType())
1632	return false;
1633	} else {
1634	LoadType = LI->getType();
1635	}
1636
1637	// Ensure that there are no instructions between the PHI and the load that
1638	// could store.
1639	for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1640	if (BBI ->mayWriteToMemory())
1641	return false;
1642
1643	MaxAlign = std::max(a: MaxAlign, b: LI->getAlign());
1644	}
1645
1646	if (!LoadType)
1647	return false;
1648
1649	APInt LoadSize =
1650	APInt (APWidth, DL.getTypeStoreSize(Ty: LoadType).getFixedValue());
1651
1652	// We can only transform this if it is safe to push the loads into the
1653	// predecessor blocks. The only thing to watch out for is that we can't put
1654	// a possibly trapping load in the predecessor if it is a critical edge.
1655	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1656	Instruction *TI = PN.getIncomingBlock(i: Idx)->getTerminator();
1657	Value *InVal = PN.getIncomingValue(i: Idx);
1658
1659	// If the value is produced by the terminator of the predecessor (an
1660	// invoke) or it has side-effects, there is no valid place to put a load
1661	// in the predecessor.
1662	if (TI == InVal \|\| TI->mayHaveSideEffects())
1663	return false;
1664
1665	// If the predecessor has a single successor, then the edge isn't
1666	// critical.
1667	if (TI->getNumSuccessors() == `1`)
1668	continue;
1669
1670	// If this pointer is always safe to load, or if we can prove that there
1671	// is already a load in the block, then we can move the load to the pred
1672	// block.
1673	if (isSafeToLoadUnconditionally(V: InVal, Alignment: MaxAlign, Size: LoadSize,
1674	SQ: SimplifyQuery (DL, TI)))
1675	continue;
1676
1677	return false;
1678	}
1679
1680	return true;
1681	}
1682
1683	/// Check whether a single store through PN can be moved onto each incoming
1684	/// edge.
1685	static StoreInst getPHIStoreToRewrite(PHINode &PN, bool* PreserveCFG,
1686	DominatorTree &DT) {
1687	// TODO: Support multiple stores and mixed load/store users.
1688	// TODO: Look through other instructions, such as other phis or addrspacecasts
1689	if (!PN.hasOneUse())
1690	return nullptr;
1691
1692	auto *SI = dyn_cast<StoreInst>(Val: PN.user_back());
1693	if (!SI \|\| SI->getPointerOperand() != &PN)
1694	return nullptr;
1695
1696	if (SI->isVolatile())
1697	return nullptr;
1698
1699	BasicBlock *BB = PN.getParent();
1700	// TODO: Allow a harmless prefix between the PHIs and the store.
1701	if (&*BB->getFirstNonPHIOrPseudoProbe() != SI)
1702	return nullptr;
1703
1704	Value *StoredValue = SI->getValueOperand();
1705	SmallPtrSet<BasicBlock *, `4`> SeenPreds;
1706	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1707	BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1708
1709	// Only one store is needed for duplicate edges from the same predecessor.
1710	if (!SeenPreds.insert(Ptr: Pred).second)
1711	continue;
1712
1713	// TODO: Support other terminators.
1714	Instruction *TI = Pred->getTerminator();
1715	if (!isa<CondBrInst, UncondBrInst>(Val: TI))
1716	return nullptr;
1717
1718	if (Pred == BB)
1719	return nullptr;
1720
1721	// TODO: If StoredValue is another PHI in BB, use its corresponding
1722	// incoming value instead of requiring it to dominate every predecessor.
1723	if (!DT.dominates(Def: StoredValue, User: TI))
1724	return nullptr;
1725
1726	if (TI->getNumSuccessors() == `1`)
1727	continue;
1728
1729	// If the predecessor has more than one successor, then we will need to
1730	// split it so that we can insert the store only on the path to this BB.
1731	if (PreserveCFG \|\| !BB->canSplitPredecessors())
1732	return nullptr;
1733	}
1734
1735	return SI;
1736	}
1737
1738	static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1739	LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1740
1741	LoadInst *SomeLoad = cast<LoadInst>(Val: PN.user_back());
1742	Type *LoadTy = SomeLoad->getType();
1743	IRB.SetInsertPoint(&PN);
1744	PHINode *NewPN = IRB.CreatePHI(Ty: LoadTy, NumReservedValues: PN.getNumIncomingValues(),
1745	Name: PN.getName() + ".sroa.speculated");
1746
1747	// Get the AA tags and alignment to use from one of the loads. It does not
1748	// matter which one we get and if any differ.
1749	AAMDNodes AATags = SomeLoad->getAAMetadata();
1750	Align Alignment = SomeLoad->getAlign();
1751
1752	// Rewrite all loads of the PN to use the new PHI.
1753	while (!PN.use_empty()) {
1754	LoadInst *LI = cast<LoadInst>(Val: PN.user_back());
1755	LI->replaceAllUsesWith(V: NewPN);
1756	LI->eraseFromParent();
1757	}
1758
1759	// Inject loads into all of the pred blocks.
1760	DenseMap<BasicBlock , Value > InjectedLoads;
1761	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1762	BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1763	Value *InVal = PN.getIncomingValue(i: Idx);
1764
1765	// A PHI node is allowed to have multiple (duplicated) entries for the same
1766	// basic block, as long as the value is the same. So if we already injected
1767	// a load in the predecessor, then we should reuse the same load for all
1768	// duplicated entries.
1769	if (Value *V = InjectedLoads.lookup(Val: Pred)) {
1770	NewPN->addIncoming(V, BB: Pred);
1771	continue;
1772	}
1773
1774	Instruction *TI = Pred->getTerminator();
1775	IRB.SetInsertPoint(TI);
1776
1777	LoadInst *Load = IRB.CreateAlignedLoad(
1778	Ty: LoadTy, Ptr: InVal, Align: Alignment,
1779	Name: (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1780	++NumLoadsSpeculated;
1781	if (AATags)
1782	Load->setAAMetadata(AATags);
1783	NewPN->addIncoming(V: Load, BB: Pred);
1784	InjectedLoads [Pred] = Load;
1785	}
1786
1787	LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1788	PN.eraseFromParent();
1789	}
1790
1791	/// Move a store through a pointer PHI onto each of the PHI's incoming edges.
1792	/// Returns whether this required modifying the CFG.
1793	static bool rewritePHINodeStore(PHINode &PN, StoreInst &SI, DomTreeUpdater &DTU,
1794	SmallSetVector<AllocaInst *, `16`> &Worklist) {
1795	LLVM_DEBUG(dbgs() << " original: " << PN << "\n"
1796	<< " " << SI << "\n");
1797
1798	// Splitting one edge rewrites all PHIs in the destination block. Snapshot
1799	// the original predecessor/value pairs before making any CFG changes.
1800	SmallVector<std::pair<BasicBlock , Value >, `4`> IncomingValues;
1801	SmallPtrSet<BasicBlock *, `4`> SeenPreds;
1802	for (unsigned Idx = `0`, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1803	BasicBlock *Pred = PN.getIncomingBlock(i: Idx);
1804	if (!SeenPreds.insert(Ptr: Pred).second)
1805	continue;
1806	Value *InVal = PN.getIncomingValue(i: Idx);
1807	IncomingValues.emplace_back(Args&: Pred, Args&: InVal);
1808
1809	// Revisit every alloca exposed by removing the pointer PHI.
1810	if (auto *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: InVal)))
1811	Worklist.insert(X: AI);
1812	}
1813
1814	bool CFGChanged = false;
1815	BasicBlock *BB = PN.getParent();
1816	for (auto [Pred, InVal] : IncomingValues) {
1817	BasicBlock *StoreBB = Pred;
1818	if (Pred->getTerminator()->getNumSuccessors() != `1`) {
1819	StoreBB = SplitBlockPredecessors(BB, Preds: {Pred}, Suffix: ".sroa.store", DTU: &DTU);
1820	assert(StoreBB && "store edge was not checked for splitting");
1821	CFGChanged = true;
1822	}
1823
1824	auto *NewStore = cast<StoreInst>(Val: SI.clone());
1825	NewStore->setOperand(i_nocapture: StoreInst::getPointerOperandIndex(), Val_nocapture: InVal);
1826	NewStore->insertBefore(InsertPos: StoreBB->getTerminator()->getIterator());
1827	++NumStoresPredicated;
1828	LLVM_DEBUG(dbgs() << " to: " << *NewStore << "\n");
1829	}
1830
1831	SI.eraseFromParent();
1832	PN.eraseFromParent();
1833	return CFGChanged;
1834	}
1835
1836	SelectHandSpeculativity &
1837	SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1838	if (isTrueVal)
1839	Bitfield::set<SelectHandSpeculativity::TrueVal>(Packed&: Storage, Value: true);
1840	else
1841	Bitfield::set<SelectHandSpeculativity::FalseVal>(Packed&: Storage, Value: true);
1842	return *this;
1843	}
1844
1845	bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1846	return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Packed: Storage)
1847	: Bitfield::get<SelectHandSpeculativity::FalseVal>(Packed: Storage);
1848	}
1849
1850	bool SelectHandSpeculativity::areAllSpeculatable() const {
1851	return isSpeculatable(/isTrueVal=/true) &&
1852	isSpeculatable(/isTrueVal=/false);
1853	}
1854
1855	bool SelectHandSpeculativity::areAnySpeculatable() const {
1856	return isSpeculatable(/isTrueVal=/true) \|\|
1857	isSpeculatable(/isTrueVal=/false);
1858	}
1859	bool SelectHandSpeculativity::areNoneSpeculatable() const {
1860	return !areAnySpeculatable();
1861	}
1862
1863	static SelectHandSpeculativity
1864	isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) {
1865	assert(LI.isSimple() && "Only for simple loads");
1866	SelectHandSpeculativity Spec;
1867
1868	const DataLayout &DL = SI.getDataLayout();
1869	for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1870	if (isSafeToLoadUnconditionally(V: Value, Ty: LI.getType(), Alignment: LI.getAlign(),
1871	SQ: SimplifyQuery (DL, &LI)))
1872	Spec.setAsSpeculatable(/isTrueVal=/Value == SI.getTrueValue());
1873	else if (PreserveCFG)
1874	return Spec;
1875
1876	return Spec;
1877	}
1878
1879	std::optional<RewriteableMemOps>
1880	SROA::isSafeSelectToSpeculateOrPredicate(SelectInst &SI, bool PreserveCFG) {
1881	RewriteableMemOps Ops;
1882
1883	for (User *U : SI.users()) {
1884	if (auto *Store = dyn_cast<StoreInst>(Val: U)) {
1885	// We don't care about atomicity of stores as they're always predicated.
1886	if (Store->isVolatile() \|\| PreserveCFG)
1887	return {}; // Give up on this `select`.
1888	Ops.emplace_back(Args&: Store);
1889	continue;
1890	}
1891
1892	auto *LI = dyn_cast<LoadInst>(Val: U);
1893
1894	if (!LI \|\| LI->isVolatile())
1895	return {}; // Give up on this `select`.
1896
1897	PossiblySpeculatableLoad Load(LI);
1898
1899	// Atomic loads need to be predicated: speculation emits plain loads, which
1900	// would drop the atomicity of the load when one of the hands is non-local,
1901	// introducing a race.
1902	// We could allow this when both hands are allocas (atomicity doesn't mean
1903	// anything on allocas), but for now we give up if we're not allowed to
1904	// predicate.
1905	if (!LI->isSimple()) { // or `LI->isAtomic()`, `isVolatile` already checked
1906	if (PreserveCFG)
1907	return {}; // Give up on this `select`.
1908	Ops.emplace_back(Args&: Load);
1909	continue;
1910	}
1911
1912	SelectHandSpeculativity Spec =
1913	isSafeLoadOfSelectToSpeculate(LI&: *LI, SI, PreserveCFG);
1914	if (PreserveCFG && !Spec.areAllSpeculatable())
1915	return {}; // Give up on this `select`.
1916
1917	Load.setInt(Spec);
1918	Ops.emplace_back(Args&: Load);
1919	}
1920
1921	return Ops;
1922	}
1923
1924	static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI,
1925	IRBuilderTy &IRB) {
1926	LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1927
1928	Value *TV = SI.getTrueValue();
1929	Value *FV = SI.getFalseValue();
1930	// Replace the given load of the select with a select of two loads.
1931
1932	assert(LI.isSimple() && "We only speculate simple loads");
1933
1934	IRB.SetInsertPoint(&LI);
1935
1936	LoadInst *TL =
1937	IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: TV, Align: LI.getAlign(),
1938	Name: LI.getName() + ".sroa.speculate.load.true");
1939	LoadInst *FL =
1940	IRB.CreateAlignedLoad(Ty: LI.getType(), Ptr: FV, Align: LI.getAlign(),
1941	Name: LI.getName() + ".sroa.speculate.load.false");
1942	NumLoadsSpeculated += `2`;
1943
1944	// Transfer alignment and AA info if present.
1945	TL->setAlignment(LI.getAlign());
1946	FL->setAlignment(LI.getAlign());
1947
1948	AAMDNodes Tags = LI.getAAMetadata();
1949	if (Tags) {
1950	TL->setAAMetadata(Tags);
1951	FL->setAAMetadata(Tags);
1952	}
1953
1954	Value *V = IRB.CreateSelect(C: SI.getCondition(), True: TL, False: FL,
1955	Name: LI.getName() + ".sroa.speculated", MDFrom: &SI);
1956
1957	LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1958	LI.replaceAllUsesWith(V);
1959	}
1960
1961	template <typename T>
1962	static void rewriteMemOpOfSelect(SelectInst &SI, T &I,
1963	SelectHandSpeculativity Spec,
1964	DomTreeUpdater &DTU) {
1965	assert((isa<LoadInst>(I) \|\| isa<StoreInst>(I)) && "Only for load and store!");
1966	LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1967	BasicBlock *Head = I.getParent();
1968	Instruction ThenTerm = nullptr*;
1969	Instruction ElseTerm = nullptr*;
1970	if (Spec.areNoneSpeculatable())
1971	SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1972	SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU);
1973	else {
1974	SplitBlockAndInsertIfThen(SI.getCondition(), &I, /Unreachable=/false,
1975	SI.getMetadata(KindID: LLVMContext::MD_prof), &DTU,
1976	/LI=/nullptr, /ThenBlock=/nullptr);
1977	if (Spec.isSpeculatable(/isTrueVal=/true))
1978	cast<CondBrInst>(Val: Head->getTerminator())->swapSuccessors();
1979	}
1980	auto *HeadBI = cast<CondBrInst>(Val: Head->getTerminator());
1981	Spec = {}; // Do not use `Spec` beyond this point.
1982	BasicBlock *Tail = I.getParent();
1983	Tail->setName(Head->getName() + ".cont");
1984	PHINode *PN;
1985	if (isa<LoadInst>(I))
1986	PN = PHINode::Create(Ty: I.getType(), NumReservedValues: `2`, NameStr: "", InsertBefore: I.getIterator());
1987	for (BasicBlock *SuccBB : successors(BB: Head)) {
1988	bool IsThen = SuccBB == HeadBI->getSuccessor(i: `0`);
1989	int SuccIdx = IsThen ? `0` : `1`;
1990	auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1991	auto &CondMemOp = cast<T>(*I.clone());
1992	if (NewMemOpBB != Head) {
1993	NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1994	if (isa<LoadInst>(I))
1995	++NumLoadsPredicated;
1996	else
1997	++NumStoresPredicated;
1998	} else {
1999	CondMemOp.dropUBImplyingAttrsAndMetadata();
2000	++NumLoadsSpeculated;
2001	}
2002	CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
2003	Value *Ptr = SI.getOperand(i_nocapture: `1` + SuccIdx);
2004	CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
2005	if (isa<LoadInst>(I)) {
2006	CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
2007	PN->addIncoming(V: &CondMemOp, BB: NewMemOpBB);
2008	} else
2009	LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
2010	}
2011	if (isa<LoadInst>(I)) {
2012	PN->takeName(V: &I);
2013	LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
2014	I.replaceAllUsesWith(PN);
2015	}
2016	}
2017
2018	static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I,
2019	SelectHandSpeculativity Spec,
2020	DomTreeUpdater &DTU) {
2021	if (auto *LI = dyn_cast<LoadInst>(Val: &I))
2022	rewriteMemOpOfSelect(SI&: SelInst, I&: *LI, Spec, DTU);
2023	else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
2024	rewriteMemOpOfSelect(SI&: SelInst, I&: *SI, Spec, DTU);
2025	else
2026	llvm_unreachable_internal(msg: "Only for load and store.");
2027	}
2028
2029	static bool rewriteSelectInstMemOps(SelectInst &SI,
2030	const RewriteableMemOps &Ops,
2031	IRBuilderTy &IRB, DomTreeUpdater *DTU) {
2032	bool CFGChanged = false;
2033	LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
2034
2035	for (const RewriteableMemOp &Op : Ops) {
2036	SelectHandSpeculativity Spec;
2037	Instruction *I;
2038	if (auto *const *US = std::get_if<UnspeculatableStore>(ptr: &Op)) {
2039	I = *US;
2040	} else {
2041	auto PSL = std::get<PossiblySpeculatableLoad>(v: Op);
2042	I = PSL.getPointer();
2043	Spec = PSL.getInt();
2044	}
2045	if (Spec.areAllSpeculatable()) {
2046	speculateSelectInstLoads(SI, LI&: cast<LoadInst>(Val&: *I), IRB);
2047	} else {
2048	assert(DTU && "Should not get here when not allowed to modify the CFG!");
2049	rewriteMemOpOfSelect(SelInst&: SI, I&: I, Spec, DTU&: DTU);
2050	CFGChanged = true;
2051	}
2052	I->eraseFromParent();
2053	}
2054
2055	for (User *U : make_early_inc_range(Range: SI.users()))
2056	cast<BitCastInst>(Val: U)->eraseFromParent();
2057	SI.eraseFromParent();
2058	return CFGChanged;
2059	}
2060
2061	/// Compute an adjusted pointer from Ptr by Offset bytes where the
2062	/// resulting pointer has PointerTy.
2063	static Value getAdjustedPtr(IRBuilderTy &IRB, const* DataLayout &DL, Value *Ptr,
2064	APInt Offset, Type *PointerTy,
2065	const Twine &NamePrefix) {
2066	if (Offset != `0`)
2067	Ptr = IRB.CreateInBoundsPtrAdd(Ptr, Offset: IRB.getInt(AI: Offset),
2068	Name: NamePrefix + "sroa_idx");
2069	return IRB.CreatePointerBitCastOrAddrSpaceCast(V: Ptr, DestTy: PointerTy,
2070	Name: NamePrefix + "sroa_cast");
2071	}
2072
2073	/// Compute the adjusted alignment for a load or store from an offset.
2074	static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
2075	return commonAlignment(A: getLoadStoreAlignment(I), Offset);
2076	}
2077
2078	/// Test whether we can convert a value from the old to the new type.
2079	///
2080	/// This predicate should be used to guard calls to convertValue in order to
2081	/// ensure that we only try to convert viable values. The strategy is that we
2082	/// will peel off single element struct and array wrappings to get to an
2083	/// underlying value, and convert that value.
2084	static bool canConvertValue(const DataLayout &DL, Type OldTy, Type NewTy,
2085	unsigned VScale = `0`) {
2086	if (OldTy == NewTy)
2087	return true;
2088
2089	// For integer types, we can't handle any bit-width differences. This would
2090	// break both vector conversions with extension and introduce endianness
2091	// issues when in conjunction with loads and stores.
2092	if (isa<IntegerType>(Val: OldTy) && isa<IntegerType>(Val: NewTy)) {
2093	assert(cast<IntegerType>(OldTy)->getBitWidth() !=
2094	cast<IntegerType>(NewTy)->getBitWidth() &&
2095	"We can't have the same bitwidth for different int types");
2096	return false;
2097	}
2098
2099	TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
2100	TypeSize OldSize = DL.getTypeSizeInBits(Ty: OldTy);
2101
2102	if ((isa<ScalableVectorType>(Val: NewTy) && isa<FixedVectorType>(Val: OldTy)) \|\|
2103	(isa<ScalableVectorType>(Val: OldTy) && isa<FixedVectorType>(Val: NewTy))) {
2104	// Conversion is only possible when the size of scalable vectors is known.
2105	if (!VScale)
2106	return false;
2107
2108	// For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
2109	// a single domain (either fixed or scalable). Any additional conversion
2110	// between fixed and scalable types is handled through integer types.
2111	auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
2112	auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
2113
2114	if (isa<ScalableVectorType>(Val: NewTy)) {
2115	if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: NewVTy), EltTy: OldVTy))
2116	return false;
2117
2118	NewSize = TypeSize::getFixed(ExactSize: NewSize.getKnownMinValue() * VScale);
2119	} else {
2120	if (!VectorType::getWithSizeAndScalar(SizeTy: cast<VectorType>(Val: OldVTy), EltTy: NewVTy))
2121	return false;
2122
2123	OldSize = TypeSize::getFixed(ExactSize: OldSize.getKnownMinValue() * VScale);
2124	}
2125	}
2126
2127	if (NewSize != OldSize)
2128	return false;
2129	if (!NewTy->isSingleValueType() \|\| !OldTy->isSingleValueType())
2130	return false;
2131
2132	// We can convert pointers to integers and vice-versa. Same for vectors
2133	// of pointers and integers.
2134	OldTy = OldTy->getScalarType();
2135	NewTy = NewTy->getScalarType();
2136	if (NewTy->isPointerTy() \|\| OldTy->isPointerTy()) {
2137	if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
2138	unsigned OldAS = OldTy->getPointerAddressSpace();
2139	unsigned NewAS = NewTy->getPointerAddressSpace();
2140	// Convert pointers if they are pointers from the same address space or
2141	// different integral (not non-integral) address spaces with the same
2142	// pointer size.
2143	return OldAS == NewAS \|\|
2144	(!DL.isNonIntegralAddressSpace(AddrSpace: OldAS) &&
2145	!DL.isNonIntegralAddressSpace(AddrSpace: NewAS) &&
2146	DL.getPointerSize(AS: OldAS) == DL.getPointerSize(AS: NewAS));
2147	}
2148
2149	// We can convert integers to integral pointers, but not to non-integral
2150	// pointers.
2151	if (OldTy->isIntegerTy())
2152	return !DL.isNonIntegralPointerType(Ty: NewTy);
2153
2154	// We can convert integral pointers to integers, but non-integral pointers
2155	// need to remain pointers.
2156	if (!DL.isNonIntegralPointerType(Ty: OldTy))
2157	return NewTy->isIntegerTy();
2158
2159	return false;
2160	}
2161
2162	if (OldTy->isTargetExtTy() \|\| NewTy->isTargetExtTy())
2163	return false;
2164
2165	return true;
2166	}
2167
2168	/// Test whether the given slice use can be promoted to a vector.
2169	///
2170	/// This function is called to test each entry in a partition which is slated
2171	/// for a single slice.
2172	static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2173	VectorType *Ty,
2174	uint64_t ElementSize,
2175	const DataLayout &DL,
2176	unsigned VScale) {
2177	// First validate the slice offsets.
2178	uint64_t BeginOffset =
2179	std::max(a: S.beginOffset(), b: P.beginOffset()) - P.beginOffset();
2180	uint64_t BeginIndex = BeginOffset / ElementSize;
2181	if (BeginIndex * ElementSize != BeginOffset \|\|
2182	BeginIndex >= cast<FixedVectorType>(Val: Ty)->getNumElements())
2183	return false;
2184	uint64_t EndOffset = std::min(a: S.endOffset(), b: P.endOffset()) - P.beginOffset();
2185	uint64_t EndIndex = EndOffset / ElementSize;
2186	if (EndIndex * ElementSize != EndOffset \|\|
2187	EndIndex > cast<FixedVectorType>(Val: Ty)->getNumElements())
2188	return false;
2189
2190	assert(EndIndex > BeginIndex && "Empty vector!");
2191	uint64_t NumElements = EndIndex - BeginIndex;
2192	Type *SliceTy = (NumElements == `1`)
2193	? Ty->getElementType()
2194	: FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElements);
2195
2196	Type *SplitIntTy =
2197	Type::getIntNTy(C&: Ty->getContext(), N: NumElements * ElementSize * `8`);
2198
2199	Use *U = S.getUse();
2200
2201	if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2202	if (MI->isVolatile())
2203	return false;
2204	if (!S.isSplittable())
2205	return false; // Skip any unsplittable intrinsics.
2206	if (isa<MemSetInst>(Val: MI)) {
2207	Type SplatTy = Type::getIntNTy(C&: Ty->getContext(), N: ElementSize `8`);
2208	if (!canConvertValue(DL, OldTy: SplatTy, NewTy: Ty->getElementType(), VScale))
2209	return false;
2210	}
2211	} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2212	if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2213	return false;
2214	} else if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2215	if (LI->isVolatile())
2216	return false;
2217	Type *LTy = LI->getType();
2218	// Disable vector promotion when there are loads or stores of an FCA.
2219	if (LTy->isStructTy())
2220	return false;
2221	if (P.beginOffset() > S.beginOffset() \|\| P.endOffset() < S.endOffset()) {
2222	assert(LTy->isIntegerTy());
2223	LTy = SplitIntTy;
2224	}
2225	if (!canConvertValue(DL, OldTy: SliceTy, NewTy: LTy, VScale))
2226	return false;
2227	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2228	if (SI->isVolatile())
2229	return false;
2230	Type *STy = SI->getValueOperand()->getType();
2231	// Disable vector promotion when there are loads or stores of an FCA.
2232	if (STy->isStructTy())
2233	return false;
2234	if (P.beginOffset() > S.beginOffset() \|\| P.endOffset() < S.endOffset()) {
2235	assert(STy->isIntegerTy());
2236	STy = SplitIntTy;
2237	}
2238	if (!canConvertValue(DL, OldTy: STy, NewTy: SliceTy, VScale))
2239	return false;
2240	} else {
2241	return false;
2242	}
2243
2244	return true;
2245	}
2246
2247	/// Test whether any vector type in \p CandidateTys is viable for promotion.
2248	///
2249	/// This implements the necessary checking for \c isVectorPromotionViable over
2250	/// all slices of the alloca for the given VectorType.
2251	static VectorType *
2252	checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
2253	SmallVectorImpl<VectorType *> &CandidateTys,
2254	bool HaveCommonEltTy, Type *CommonEltTy,
2255	bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2256	VectorType CommonVecPtrTy, unsigned* VScale) {
2257	// If we didn't find a vector type, nothing to do here.
2258	if (CandidateTys.empty())
2259	return nullptr;
2260
2261	// Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2262	// then we should choose it, not some other alternative.
2263	// But, we can't perform a no-op pointer address space change via bitcast,
2264	// so if we didn't have a common pointer element type, bail.
2265	if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2266	return nullptr;
2267
2268	// Try to pick the "best" element type out of the choices.
2269	if (!HaveCommonEltTy && HaveVecPtrTy) {
2270	// If there was a pointer element type, there's really only one choice.
2271	CandidateTys.clear();
2272	CandidateTys.push_back(Elt: CommonVecPtrTy);
2273	} else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2274	// Integer-ify vector types.
2275	for (VectorType *&VTy : CandidateTys) {
2276	if (!VTy->getElementType()->isIntegerTy())
2277	VTy = cast<VectorType>(Val: VTy->getWithNewType(EltTy: IntegerType::getIntNTy(
2278	C&: VTy->getContext(), N: VTy->getScalarSizeInBits())));
2279	}
2280
2281	// Rank the remaining candidate vector types. This is easy because we know
2282	// they're all integer vectors. We sort by ascending number of elements.
2283	auto RankVectorTypesComp = [&DL](VectorType RHSTy, VectorType LHSTy) {
2284	(void)DL;
2285	assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2286	DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2287	"Cannot have vector types of different sizes!");
2288	assert(RHSTy->getElementType()->isIntegerTy() &&
2289	"All non-integer types eliminated!");
2290	assert(LHSTy->getElementType()->isIntegerTy() &&
2291	"All non-integer types eliminated!");
2292	return cast<FixedVectorType>(Val: RHSTy)->getNumElements() <
2293	cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2294	};
2295	auto RankVectorTypesEq = [&DL](VectorType RHSTy, VectorType LHSTy) {
2296	(void)DL;
2297	assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2298	DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2299	"Cannot have vector types of different sizes!");
2300	assert(RHSTy->getElementType()->isIntegerTy() &&
2301	"All non-integer types eliminated!");
2302	assert(LHSTy->getElementType()->isIntegerTy() &&
2303	"All non-integer types eliminated!");
2304	return cast<FixedVectorType>(Val: RHSTy)->getNumElements() ==
2305	cast<FixedVectorType>(Val: LHSTy)->getNumElements();
2306	};
2307	llvm::sort(C&: CandidateTys, Comp: RankVectorTypesComp);
2308	CandidateTys.erase(CS: llvm::unique(R&: CandidateTys, P: RankVectorTypesEq),
2309	CE: CandidateTys.end());
2310	} else {
2311	// The only way to have the same element type in every vector type is to
2312	// have the same vector type. Check that and remove all but one.
2313	#ifndef NDEBUG
2314	for (VectorType *VTy : CandidateTys) {
2315	assert(VTy->getElementType() == CommonEltTy &&
2316	"Unaccounted for element type!");
2317	assert(VTy == CandidateTys[`0`] &&
2318	"Different vector types with the same element type!");
2319	}
2320	#endif
2321	CandidateTys.resize(N: `1`);
2322	}
2323
2324	// FIXME: hack. Do we have a named constant for this?
2325	// SDAG SDNode can't have more than 65535 operands.
2326	llvm::erase_if(C&: CandidateTys, P: [](VectorType *VTy) {
2327	return cast<FixedVectorType>(Val: VTy)->getNumElements() >
2328	std::numeric_limits<unsigned short>::max();
2329	});
2330
2331	// Find a vector type viable for promotion by iterating over all slices.
2332	auto VTy = llvm::find_if(Range&: CandidateTys, P: [&](VectorType VTy) -> bool {
2333	uint64_t ElementSize =
2334	DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2335
2336	// While the definition of LLVM vectors is bitpacked, we don't support sizes
2337	// that aren't byte sized.
2338	if (ElementSize % `8`)
2339	return false;
2340	assert((DL.getTypeSizeInBits(VTy).getFixedValue() % `8`) == `0` &&
2341	"vector size not a multiple of element size?");
2342	ElementSize /= `8`;
2343
2344	for (const Slice &S : P)
2345	if (!isVectorPromotionViableForSlice(P, S, Ty: VTy, ElementSize, DL, VScale))
2346	return false;
2347
2348	for (const Slice *S : P.splitSliceTails())
2349	if (!isVectorPromotionViableForSlice(P, S: *S, Ty: VTy, ElementSize, DL, VScale))
2350	return false;
2351
2352	return true;
2353	});
2354	return VTy != CandidateTys.end() ? VTy : nullptr*;
2355	}
2356
2357	static VectorType *createAndCheckVectorTypesForPromotion(
2358	SetVector<Type > &OtherTys, ArrayRef<VectorType > CandidateTysCopy,
2359	function_ref<void(Type *)> CheckCandidateType, Partition &P,
2360	const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2361	bool &HaveCommonEltTy, Type &CommonEltTy, bool* &HaveVecPtrTy,
2362	bool &HaveCommonVecPtrTy, VectorType &CommonVecPtrTy, unsigned* VScale) {
2363	[[maybe_unused]] VectorType *OriginalElt =
2364	CandidateTysCopy.size() ? CandidateTysCopy [`0`] : nullptr;
2365	// Consider additional vector types where the element type size is a
2366	// multiple of load/store element size.
2367	for (Type *Ty : OtherTys) {
2368	if (!VectorType::isValidElementType(ElemTy: Ty))
2369	continue;
2370	unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2371	// Make a copy of CandidateTys and iterate through it, because we
2372	// might append to CandidateTys in the loop.
2373	for (VectorType *const VTy : CandidateTysCopy) {
2374	// The elements in the copy should remain invariant throughout the loop
2375	assert(CandidateTysCopy[`0`] == OriginalElt && "Different Element");
2376	unsigned VectorSize = DL.getTypeSizeInBits(Ty: VTy).getFixedValue();
2377	unsigned ElementSize =
2378	DL.getTypeSizeInBits(Ty: VTy->getElementType()).getFixedValue();
2379	if (TypeSize != VectorSize && TypeSize != ElementSize &&
2380	VectorSize % TypeSize == `0`) {
2381	VectorType NewVTy = VectorType::get(ElementType: Ty, NumElements: VectorSize / TypeSize, Scalable: false*);
2382	CheckCandidateType (NewVTy);
2383	}
2384	}
2385	}
2386
2387	return checkVectorTypesForPromotion(
2388	P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2389	HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2390	}
2391
2392	/// Test whether the given alloca partitioning and range of slices can be
2393	/// promoted to a vector.
2394	///
2395	/// This is a quick test to check whether we can rewrite a particular alloca
2396	/// partition (and its newly formed alloca) into a vector alloca with only
2397	/// whole-vector loads and stores such that it could be promoted to a vector
2398	/// SSA value. We only can ensure this for a limited set of operations, and we
2399	/// don't want to do the rewrites unless we are confident that the result will
2400	/// be promotable, so we have an early test here.
2401	static VectorType isVectorPromotionViable(Partition &P, const* DataLayout &DL,
2402	unsigned VScale) {
2403	// Collect the candidate types for vector-based promotion. Also track whether
2404	// we have different element types.
2405	SmallVector<VectorType *, `4`> CandidateTys;
2406	SetVector<Type *> LoadStoreTys;
2407	SetVector<Type *> DeferredTys;
2408	Type CommonEltTy = nullptr*;
2409	VectorType CommonVecPtrTy = nullptr*;
2410	bool HaveVecPtrTy = false;
2411	bool HaveCommonEltTy = true;
2412	bool HaveCommonVecPtrTy = true;
2413	auto CheckCandidateType = [&](Type *Ty) {
2414	if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
2415	// Return if bitcast to vectors is different for total size in bits.
2416	if (!CandidateTys.empty()) {
2417	VectorType *V = CandidateTys [`0`];
2418	if (DL.getTypeSizeInBits(Ty: VTy).getFixedValue() !=
2419	DL.getTypeSizeInBits(Ty: V).getFixedValue()) {
2420	CandidateTys.clear();
2421	return;
2422	}
2423	}
2424	CandidateTys.push_back(Elt: VTy);
2425	Type *EltTy = VTy->getElementType();
2426
2427	if (!CommonEltTy)
2428	CommonEltTy = EltTy;
2429	else if (CommonEltTy != EltTy)
2430	HaveCommonEltTy = false;
2431
2432	if (EltTy->isPointerTy()) {
2433	HaveVecPtrTy = true;
2434	if (!CommonVecPtrTy)
2435	CommonVecPtrTy = VTy;
2436	else if (CommonVecPtrTy != VTy)
2437	HaveCommonVecPtrTy = false;
2438	}
2439	}
2440	};
2441
2442	// Put load and store types into a set for de-duplication.
2443	for (const Slice &S : P) {
2444	Type *Ty;
2445	if (auto *LI = dyn_cast<LoadInst>(Val: S.getUse()->getUser()))
2446	Ty = LI->getType();
2447	else if (auto *SI = dyn_cast<StoreInst>(Val: S.getUse()->getUser()))
2448	Ty = SI->getValueOperand()->getType();
2449	else
2450	continue;
2451
2452	auto CandTy = Ty->getScalarType();
2453	if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() \|\|
2454	S.endOffset() != P.endOffset())) {
2455	DeferredTys.insert(X: Ty);
2456	continue;
2457	}
2458
2459	LoadStoreTys.insert(X: Ty);
2460	// Consider any loads or stores that are the exact size of the slice.
2461	if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2462	CheckCandidateType (Ty);
2463	}
2464
2465	SmallVector<VectorType *, `4`> CandidateTysCopy = CandidateTys;
2466	if (auto *VTy = createAndCheckVectorTypesForPromotion(
2467	OtherTys&: LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2468	CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2469	HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2470	return VTy;
2471
2472	CandidateTys.clear();
2473	return createAndCheckVectorTypesForPromotion(
2474	OtherTys&: DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2475	HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2476	CommonVecPtrTy, VScale);
2477	}
2478
2479	/// Test whether a slice of an alloca is valid for integer widening.
2480	///
2481	/// This implements the necessary checking for the \c isIntegerWideningViable
2482	/// test below on a single slice of the alloca.
2483	static bool isIntegerWideningViableForSlice(const Slice &S,
2484	uint64_t AllocBeginOffset,
2485	Type *AllocaTy,
2486	const DataLayout &DL,
2487	bool &WholeAllocaOp) {
2488	uint64_t Size = DL.getTypeStoreSize(Ty: AllocaTy).getFixedValue();
2489
2490	uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2491	uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2492
2493	Use *U = S.getUse();
2494
2495	// Lifetime intrinsics operate over the whole alloca whose sizes are usually
2496	// larger than other load/store slices (RelEnd > Size). But lifetime are
2497	// always promotable and should not impact other slices' promotability of the
2498	// partition.
2499	if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: U->getUser())) {
2500	if (II->isLifetimeStartOrEnd() \|\| II->isDroppable())
2501	return true;
2502	}
2503
2504	// We can't reasonably handle cases where the load or store extends past
2505	// the end of the alloca's type and into its padding.
2506	if (RelEnd > Size)
2507	return false;
2508
2509	if (LoadInst *LI = dyn_cast<LoadInst>(Val: U->getUser())) {
2510	if (LI->isVolatile())
2511	return false;
2512	// We can't handle loads that extend past the allocated memory.
2513	TypeSize LoadSize = DL.getTypeStoreSize(Ty: LI->getType());
2514	if (!LoadSize.isFixed() \|\| LoadSize.getFixedValue() > Size)
2515	return false;
2516	// So far, AllocaSliceRewriter does not support widening split slice tails
2517	// in rewriteIntegerLoad.
2518	if (S.beginOffset() < AllocBeginOffset)
2519	return false;
2520	// Note that we don't count vector loads or stores as whole-alloca
2521	// operations which enable integer widening because we would prefer to use
2522	// vector widening instead.
2523	if (!isa<VectorType>(Val: LI->getType()) && RelBegin == `0` && RelEnd == Size)
2524	WholeAllocaOp = true;
2525	if (IntegerType *ITy = dyn_cast<IntegerType>(Val: LI->getType())) {
2526	if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2527	return false;
2528	} else if (RelBegin != `0` \|\| RelEnd != Size \|\|
2529	!canConvertValue(DL, OldTy: AllocaTy, NewTy: LI->getType())) {
2530	// Non-integer loads need to be convertible from the alloca type so that
2531	// they are promotable.
2532	return false;
2533	}
2534	} else if (StoreInst *SI = dyn_cast<StoreInst>(Val: U->getUser())) {
2535	Type *ValueTy = SI->getValueOperand()->getType();
2536	if (SI->isVolatile())
2537	return false;
2538	// We can't handle stores that extend past the allocated memory.
2539	TypeSize StoreSize = DL.getTypeStoreSize(Ty: ValueTy);
2540	if (!StoreSize.isFixed() \|\| StoreSize.getFixedValue() > Size)
2541	return false;
2542	// So far, AllocaSliceRewriter does not support widening split slice tails
2543	// in rewriteIntegerStore.
2544	if (S.beginOffset() < AllocBeginOffset)
2545	return false;
2546	// Note that we don't count vector loads or stores as whole-alloca
2547	// operations which enable integer widening because we would prefer to use
2548	// vector widening instead.
2549	if (!isa<VectorType>(Val: ValueTy) && RelBegin == `0` && RelEnd == Size)
2550	WholeAllocaOp = true;
2551	if (IntegerType *ITy = dyn_cast<IntegerType>(Val: ValueTy)) {
2552	if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(Ty: ITy).getFixedValue())
2553	return false;
2554	} else if (RelBegin != `0` \|\| RelEnd != Size \|\|
2555	!canConvertValue(DL, OldTy: ValueTy, NewTy: AllocaTy)) {
2556	// Non-integer stores need to be convertible to the alloca type so that
2557	// they are promotable.
2558	return false;
2559	}
2560	} else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(Val: U->getUser())) {
2561	if (MI->isVolatile() \|\| !isa<Constant>(Val: MI->getLength()))
2562	return false;
2563	if (!S.isSplittable())
2564	return false; // Skip any unsplittable intrinsics.
2565	} else {
2566	return false;
2567	}
2568
2569	return true;
2570	}
2571
2572	/// Test whether the given alloca partition's integer operations can be
2573	/// widened to promotable ones.
2574	///
2575	/// This is a quick test to check whether we can rewrite the integer loads and
2576	/// stores to a particular alloca into wider loads and stores and be able to
2577	/// promote the resulting alloca.
2578	static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2579	const DataLayout &DL) {
2580	uint64_t SizeInBits = DL.getTypeSizeInBits(Ty: AllocaTy).getFixedValue();
2581	// Don't create integer types larger than the maximum bitwidth.
2582	if (SizeInBits > IntegerType::MAX_INT_BITS)
2583	return false;
2584
2585	// Don't try to handle allocas with bit-padding.
2586	if (SizeInBits != DL.getTypeStoreSizeInBits(Ty: AllocaTy).getFixedValue())
2587	return false;
2588
2589	// We need to ensure that an integer type with the appropriate bitwidth can
2590	// be converted to the alloca type, whatever that is. We don't want to force
2591	// the alloca itself to have an integer type if there is a more suitable one.
2592	Type *IntTy = Type::getIntNTy(C&: AllocaTy->getContext(), N: SizeInBits);
2593	if (!canConvertValue(DL, OldTy: AllocaTy, NewTy: IntTy) \|\|
2594	!canConvertValue(DL, OldTy: IntTy, NewTy: AllocaTy))
2595	return false;
2596
2597	// While examining uses, we ensure that the alloca has a covering load or
2598	// store. We don't want to widen the integer operations only to fail to
2599	// promote due to some other unsplittable entry (which we may make splittable
2600	// later). However, if there are only splittable uses, go ahead and assume
2601	// that we cover the alloca.
2602	// FIXME: We shouldn't consider split slices that happen to start in the
2603	// partition here...
2604	bool WholeAllocaOp = P.empty() && DL.isLegalInteger(Width: SizeInBits);
2605
2606	for (const Slice &S : P)
2607	if (!isIntegerWideningViableForSlice(S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2608	WholeAllocaOp))
2609	return false;
2610
2611	for (const Slice *S : P.splitSliceTails())
2612	if (!isIntegerWideningViableForSlice(S: *S, AllocBeginOffset: P.beginOffset(), AllocaTy, DL,
2613	WholeAllocaOp))
2614	return false;
2615
2616	return WholeAllocaOp;
2617	}
2618
2619	static Value extractInteger(const* DataLayout &DL, IRBuilderTy &IRB, Value *V,
2620	IntegerType *Ty, uint64_t Offset,
2621	const Twine &Name) {
2622	LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2623	IntegerType *IntTy = cast<IntegerType>(Val: V->getType());
2624	assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2625	DL.getTypeStoreSize(IntTy).getFixedValue() &&
2626	"Element extends past full value");
2627	uint64_t ShAmt = `8` * Offset;
2628	if (DL.isBigEndian())
2629	ShAmt = `8` * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2630	DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2631	if (ShAmt) {
2632	V = IRB.CreateLShr(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2633	LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2634	}
2635	assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2636	"Cannot extract to a larger integer!");
2637	if (Ty != IntTy) {
2638	V = IRB.CreateTrunc(V, DestTy: Ty, Name: Name + ".trunc");
2639	LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2640	}
2641	return V;
2642	}
2643
2644	static Value insertInteger(const* DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2645	Value V, uint64_t Offset, const* Twine &Name) {
2646	IntegerType *IntTy = cast<IntegerType>(Val: Old->getType());
2647	IntegerType *Ty = cast<IntegerType>(Val: V->getType());
2648	assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2649	"Cannot insert a larger integer!");
2650	LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2651	if (Ty != IntTy) {
2652	V = IRB.CreateZExt(V, DestTy: IntTy, Name: Name + ".ext");
2653	LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2654	}
2655	assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2656	DL.getTypeStoreSize(IntTy).getFixedValue() &&
2657	"Element store outside of alloca store");
2658	uint64_t ShAmt = `8` * Offset;
2659	if (DL.isBigEndian())
2660	ShAmt = `8` * (DL.getTypeStoreSize(Ty: IntTy).getFixedValue() -
2661	DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2662	if (ShAmt) {
2663	V = IRB.CreateShl(LHS: V, RHS: ShAmt, Name: Name + ".shift");
2664	LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2665	}
2666
2667	if (ShAmt \|\| Ty->getBitWidth() < IntTy->getBitWidth()) {
2668	APInt Mask = ~Ty->getMask().zext(width: IntTy->getBitWidth()).shl(shiftAmt: ShAmt);
2669	Old = IRB.CreateAnd(LHS: Old, RHS: Mask, Name: Name + ".mask");
2670	LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2671	V = IRB.CreateOr(LHS: Old, RHS: V, Name: Name + ".insert");
2672	LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2673	}
2674	return V;
2675	}
2676
2677	static Value extractVector(IRBuilderTy &IRB, Value V, unsigned BeginIndex,
2678	unsigned EndIndex, const Twine &Name) {
2679	auto *VecTy = cast<FixedVectorType>(Val: V->getType());
2680	unsigned NumElements = EndIndex - BeginIndex;
2681	assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2682
2683	if (NumElements == VecTy->getNumElements())
2684	return V;
2685
2686	if (NumElements == `1`) {
2687	V = IRB.CreateExtractElement(Vec: V, Idx: BeginIndex, Name: Name + ".extract");
2688	LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2689	return V;
2690	}
2691
2692	auto Mask = llvm::to_vector<`8`>(Range: llvm::seq<int>(Begin: BeginIndex, End: EndIndex));
2693	V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".extract");
2694	LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2695	return V;
2696	}
2697
2698	static Value insertVector(IRBuilderTy &IRB, Value Old, Value *V,
2699	unsigned BeginIndex, const Twine &Name) {
2700	VectorType *VecTy = cast<VectorType>(Val: Old->getType());
2701	assert(VecTy && "Can only insert a vector into a vector");
2702
2703	VectorType *Ty = dyn_cast<VectorType>(Val: V->getType());
2704	if (!Ty) {
2705	// Single element to insert.
2706	V = IRB.CreateInsertElement(Vec: Old, NewElt: V, Idx: BeginIndex, Name: Name + ".insert");
2707	LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2708	return V;
2709	}
2710
2711	unsigned NumSubElements = cast<FixedVectorType>(Val: Ty)->getNumElements();
2712	unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
2713
2714	assert(NumSubElements <= NumElements && "Too many elements!");
2715	if (NumSubElements == NumElements) {
2716	assert(V->getType() == VecTy && "Vector type mismatch");
2717	return V;
2718	}
2719	unsigned EndIndex = BeginIndex + NumSubElements;
2720
2721	// When inserting a smaller vector into the larger to store, we first
2722	// use a shuffle vector to widen it with undef elements, and then
2723	// a second shuffle vector to select between the loaded vector and the
2724	// incoming vector.
2725	SmallVector<int, `8`> Mask;
2726	Mask.reserve(N: NumElements);
2727	for (unsigned Idx = `0`; Idx != NumElements; ++Idx)
2728	if (Idx >= BeginIndex && Idx < EndIndex)
2729	Mask.push_back(Elt: Idx - BeginIndex);
2730	else
2731	Mask.push_back(Elt: -`1`);
2732	V = IRB.CreateShuffleVector(V, Mask, Name: Name + ".expand");
2733	LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2734
2735	Mask.clear();
2736	for (unsigned Idx = `0`; Idx != NumElements; ++Idx)
2737	if (Idx >= BeginIndex && Idx < EndIndex)
2738	Mask.push_back(Elt: Idx);
2739	else
2740	Mask.push_back(Elt: Idx + NumElements);
2741	V = IRB.CreateShuffleVector(V1: V, V2: Old, Mask, Name: Name + "blend");
2742	LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2743	return V;
2744	}
2745
2746	/// This function takes two vector values and combines them into a single vector
2747	/// by concatenating their elements. The function handles:
2748	///
2749	/// 1. Element type mismatch: If either vector's element type differs from
2750	/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2751	/// preserving the total bit width (adjusting the number of elements
2752	/// accordingly).
2753	///
2754	/// 2. Size mismatch: After transforming the vectors to have the desired element
2755	/// type, if the two vectors have different numbers of elements, the smaller
2756	/// vector is extended with poison values to match the size of the larger
2757	/// vector before concatenation.
2758	///
2759	/// 3. Concatenation: The vectors are merged using a shuffle operation that
2760	/// places all elements of V0 first, followed by all elements of V1.
2761	///
2762	/// \param V0 The first vector to merge (must be a vector type)
2763	/// \param V1 The second vector to merge (must be a vector type)
2764	/// \param DL The data layout for size calculations
2765	/// \param NewAIEltTy The desired element type for the result vector
2766	/// \param Builder IRBuilder for creating new instructions
2767	/// \return A new vector containing all elements from V0 followed by all
2768	/// elements from V1
2769	static Value mergeTwoVectors(Value V0, Value V1, const* DataLayout &DL,
2770	Type *NewAIEltTy, IRBuilder<> &Builder) {
2771	// V0 and V1 are vectors
2772	// Create a new vector type with combined elements
2773	// Use ShuffleVector to concatenate the vectors
2774	auto *VecType0 = cast<FixedVectorType>(Val: V0->getType());
2775	auto *VecType1 = cast<FixedVectorType>(Val: V1->getType());
2776
2777	// If V0/V1 element types are different from NewAllocaElementType,
2778	// we need to introduce bitcasts before merging them
2779	auto BitcastIfNeeded = [&](Value &V, FixedVectorType &VecType,
2780	const char *DebugName) {
2781	Type *EltType = VecType->getElementType();
2782	if (EltType != NewAIEltTy) {
2783	// Calculate new number of elements to maintain same bit width
2784	unsigned TotalBits =
2785	VecType->getNumElements() * DL.getTypeSizeInBits(Ty: EltType);
2786	unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(Ty: NewAIEltTy);
2787
2788	auto *NewVecType = FixedVectorType::get(ElementType: NewAIEltTy, NumElts: NewNumElts);
2789	V = Builder.CreateBitCast(V, DestTy: NewVecType);
2790	VecType = NewVecType;
2791	LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2792	}
2793	};
2794
2795	BitcastIfNeeded (V0, VecType0, "V0");
2796	BitcastIfNeeded (V1, VecType1, "V1");
2797
2798	unsigned NumElts0 = VecType0->getNumElements();
2799	unsigned NumElts1 = VecType1->getNumElements();
2800
2801	SmallVector<int, `16`> ShuffleMask;
2802
2803	if (NumElts0 == NumElts1) {
2804	for (unsigned i = `0`; i < NumElts0 + NumElts1; ++i)
2805	ShuffleMask.push_back(Elt: i);
2806	} else {
2807	// If two vectors have different sizes, we need to extend
2808	// the smaller vector to the size of the larger vector.
2809	unsigned SmallSize = std::min(a: NumElts0, b: NumElts1);
2810	unsigned LargeSize = std::max(a: NumElts0, b: NumElts1);
2811	bool IsV0Smaller = NumElts0 < NumElts1;
2812	Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2813	SmallVector<int, `16`> ExtendMask;
2814	for (unsigned i = `0`; i < SmallSize; ++i)
2815	ExtendMask.push_back(Elt: i);
2816	for (unsigned i = SmallSize; i < LargeSize; ++i)
2817	ExtendMask.push_back(Elt: PoisonMaskElem);
2818	ExtendedVec = Builder.CreateShuffleVector(
2819	V1: ExtendedVec, V2: PoisonValue::get(T: ExtendedVec->getType()), Mask: ExtendMask);
2820	LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2821	for (unsigned i = `0`; i < NumElts0; ++i)
2822	ShuffleMask.push_back(Elt: i);
2823	for (unsigned i = `0`; i < NumElts1; ++i)
2824	ShuffleMask.push_back(Elt: LargeSize + i);
2825	}
2826
2827	return Builder.CreateShuffleVector(V1: V0, V2: V1, Mask: ShuffleMask);
2828	}
2829
2830	namespace {
2831
2832	/// Visitor to rewrite instructions using p particular slice of an alloca
2833	/// to use a new alloca.
2834	///
2835	/// Also implements the rewriting to vector-based accesses when the partition
2836	/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2837	/// lives here.
2838	class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2839	// Befriend the base class so it can delegate to private visit methods.
2840	friend class InstVisitor<AllocaSliceRewriter, bool>;
2841
2842	using Base = InstVisitor<AllocaSliceRewriter, bool>;
2843
2844	const DataLayout &DL;
2845	AllocaSlices &AS;
2846	SROA &Pass;
2847	AllocaInst &OldAI, &NewAI;
2848	const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2849	Type *NewAllocaTy;
2850
2851	// This is a convenience and flag variable that will be null unless the new
2852	// alloca's integer operations should be widened to this integer type due to
2853	// passing isIntegerWideningViable above. If it is non-null, the desired
2854	// integer type will be stored here for easy access during rewriting.
2855	IntegerType *IntTy;
2856
2857	// If we are rewriting an alloca partition which can be written as pure
2858	// vector operations, we stash extra information here. When VecTy is
2859	// non-null, we have some strict guarantees about the rewritten alloca:
2860	// - The new alloca is exactly the size of the vector type here.
2861	// - The accesses all either map to the entire vector or to a single
2862	// element.
2863	// - The set of accessing instructions is only one of those handled above
2864	// in isVectorPromotionViable. Generally these are the same access kinds
2865	// which are promotable via mem2reg.
2866	VectorType *VecTy;
2867	Type *ElementTy;
2868	uint64_t ElementSize;
2869
2870	// The original offset of the slice currently being rewritten relative to
2871	// the original alloca.
2872	uint64_t BeginOffset = `0`;
2873	uint64_t EndOffset = `0`;
2874
2875	// The new offsets of the slice currently being rewritten relative to the
2876	// original alloca.
2877	uint64_t NewBeginOffset = `0`, NewEndOffset = `0`;
2878
2879	uint64_t SliceSize = `0`;
2880	bool IsSplittable = false;
2881	bool IsSplit = false;
2882	Use OldUse = nullptr*;
2883	Instruction OldPtr = nullptr*;
2884
2885	// Track post-rewrite users which are PHI nodes and Selects.
2886	SmallSetVector<PHINode *, `8`> &PHIUsers;
2887	SmallSetVector<SelectInst *, `8`> &SelectUsers;
2888
2889	// Utility IR builder, whose name prefix is setup for each visited use, and
2890	// the insertion point is set to point to the user.
2891	IRBuilderTy IRB;
2892
2893	// Return the new alloca, addrspacecasted if required to avoid changing the
2894	// addrspace of a volatile access.
2895	Value getPtrToNewAI(unsigned* AddrSpace, bool IsVolatile) {
2896	if (!IsVolatile \|\| AddrSpace == NewAI.getType()->getPointerAddressSpace())
2897	return &NewAI;
2898
2899	Type *AccessTy = IRB.getPtrTy(AddrSpace);
2900	return IRB.CreateAddrSpaceCast(V: &NewAI, DestTy: AccessTy);
2901	}
2902
2903	public:
2904	AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2905	AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2906	uint64_t NewAllocaBeginOffset,
2907	uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2908	VectorType *PromotableVecTy,
2909	SmallSetVector<PHINode *, `8`> &PHIUsers,
2910	SmallSetVector<SelectInst *, `8`> &SelectUsers)
2911	: DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2912	NewAllocaBeginOffset(NewAllocaBeginOffset),
2913	NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2914	IntTy(IsIntegerPromotable
2915	? Type::getIntNTy(
2916	C&: NewAI.getContext(),
2917	N: DL.getTypeSizeInBits(Ty: NewAllocaTy).getFixedValue())
2918	: nullptr),
2919	VecTy(PromotableVecTy),
2920	ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2921	ElementSize(VecTy ? DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / `8`
2922	: `0`),
2923	PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2924	IRB (*NewAI.getModule(), ConstantFolder ()) {
2925	if (VecTy) {
2926	assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % `8`) == `0` &&
2927	"Only multiple-of-8 sized vector elements are viable");
2928	++NumVectorized;
2929	}
2930	assert((!IntTy && !VecTy) \|\| (IntTy && !VecTy) \|\| (!IntTy && VecTy));
2931	}
2932
2933	bool visit(AllocaSlices::const_iterator I) {
2934	bool CanSROA = true;
2935	BeginOffset = I->beginOffset();
2936	EndOffset = I->endOffset();
2937	IsSplittable = I->isSplittable();
2938	IsSplit =
2939	BeginOffset < NewAllocaBeginOffset \|\| EndOffset > NewAllocaEndOffset;
2940	LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2941	LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2942	LLVM_DEBUG(dbgs() << "\n");
2943
2944	// Compute the intersecting offset range.
2945	assert(BeginOffset < NewAllocaEndOffset);
2946	assert(EndOffset > NewAllocaBeginOffset);
2947	NewBeginOffset = std::max(a: BeginOffset, b: NewAllocaBeginOffset);
2948	NewEndOffset = std::min(a: EndOffset, b: NewAllocaEndOffset);
2949
2950	SliceSize = NewEndOffset - NewBeginOffset;
2951	LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2952	<< ") NewBegin:(" << NewBeginOffset << ", "
2953	<< NewEndOffset << ") NewAllocaBegin:("
2954	<< NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2955	<< ")\n");
2956	assert(IsSplit \|\| NewBeginOffset == BeginOffset);
2957	OldUse = I->getUse();
2958	OldPtr = cast<Instruction>(Val: OldUse->get());
2959
2960	Instruction *OldUserI = cast<Instruction>(Val: OldUse->getUser());
2961	IRB.SetInsertPoint(OldUserI);
2962	IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2963	// Avoid materializing the name prefix when it is discarded anyway.
2964	if (!IRB.getContext().shouldDiscardValueNames())
2965	IRB.getInserter().SetNamePrefix(Twine (NewAI.getName()) + "." +
2966	Twine (BeginOffset) + ".");
2967
2968	CanSROA &= visit(I: cast<Instruction>(Val: OldUse->getUser()));
2969	if (VecTy \|\| IntTy)
2970	assert(CanSROA);
2971	return CanSROA;
2972	}
2973
2974	/// Attempts to rewrite a partition using tree-structured merge optimization.
2975	///
2976	/// This function handles two patterns. Both produce an O(log n) tree of
2977	/// shufflevectors in place of the linear expand+blend chain that SROA would
2978	/// otherwise emit for each partial store.
2979	///
2980	/// Pattern 1 (stores-only):
2981	/// Multiple non-overlapping partial stores completely fill the alloca
2982	/// and there is exactly one full-width load coming after the stores.
2983	/// The stores are tree-merged into a single vector and stored once.
2984	///
2985	/// Example transformation:
2986	/// Before: (stores do not have to be in order)
2987	/// %alloca = alloca <8 x float>
2988	/// store <2 x float> %val0, ptr %alloca ; offset 0-1
2989	/// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2990	/// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2991	/// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2992	/// %r = load <8 x float>, ptr %alloca
2993	///
2994	/// After: tree of shufflevectors producing <8 x float> directly.
2995	///
2996	/// Pattern 2 (init + RMW, possibly multi-round):
2997	/// A single full-width init store, followed by partial loads and
2998	/// partial stores that read-modify-write the alloca one or more
2999	/// times, optionally followed by a full-width load. The only
3000	/// structural requirement is that the distinct [begin, end) ranges
3001	/// touched by the partial loads and stores, taken together, tile
3002	/// the alloca disjointly.
3003	///
3004	/// We keep a map from each slice range to the SSA value that
3005	/// currently lives there, `SliceValues[r] -> Value`:*
3006	/// - initialize each entry to the corresponding piece of the
3007	/// init store's value (via a shufflevector picking the
3008	/// range's elements out of the init value),
3009	/// - walk partial loads and stores in block order,
3010	/// - for a partial load at range r: RAUW with `SliceValues[r]`,
3011	/// - for a partial store at range r: update `SliceValues[r]` to
3012	/// the stored value and drop the store.
3013	/// At the end, the final `SliceValues[r]` entries are tree-merged
3014	/// (in range order) into a single store to the alloca, and the
3015	/// optional full-width load is replaced by a load of the alloca.
3016	///
3017	/// Because the ranges are disjoint by construction, a store at one
3018	/// range cannot affect another range's tracked value, so a single
3019	/// block-order walk correctly tracks the memory state at each
3020	/// range. The algorithm handles multi-round RMW, partial loads
3021	/// and stores interleaved in any order, read-only slices (the
3022	/// tracked value stays at the init extract), and write-only
3023	/// slices (the tracked value never flows into a load).
3024	///
3025	/// \param P The partition to analyze and potentially rewrite
3026	/// \return An optional vector of values that were deleted during the
3027	/// rewrite, or std::nullopt if the partition cannot be optimized.
3028	std::optional<SmallVector<Value *, `4`>>
3029	rewriteTreeStructuredMerge(Partition &P) {
3030	// No tail slices that overlap with the partition
3031	if (P.splitSliceTails().size() > `0`)
3032	return std::nullopt;
3033
3034	// Structure to hold store information
3035	struct StoreInfo {
3036	StoreInst *Store;
3037	uint64_t BeginOffset;
3038	uint64_t EndOffset;
3039	Value *StoredValue;
3040	StoreInfo(StoreInst SI, uint64_t Begin, uint64_t End, Value Val)
3041	: Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
3042	};
3043	struct LoadInfo {
3044	LoadInst *Load;
3045	uint64_t BeginOffset;
3046	uint64_t EndOffset;
3047	};
3048
3049	SmallVector<StoreInfo, `4`> StoreInfos; // partial stores only
3050	SmallVector<LoadInfo, `4`> LoadInfos; // partial loads only
3051	LoadInst FullLoad = nullptr; // optional full-width load*
3052	StoreInst InitStore = nullptr; // optional full-width init store*
3053
3054	// If the new alloca is a fixed vector type, we use its element type as the
3055	// allocated element type, otherwise we use i8 as the allocated element
3056	Type *AllocatedEltTy =
3057	isa<FixedVectorType>(Val: NewAllocaTy)
3058	? cast<FixedVectorType>(Val: NewAllocaTy)->getElementType()
3059	: Type::getInt8Ty(C&: NewAI.getContext());
3060	unsigned AllocatedEltTySize = DL.getTypeSizeInBits(Ty: AllocatedEltTy);
3061
3062	// Helper to check if a type is
3063	// 1. A fixed vector type
3064	// 2. The element type is not a pointer
3065	// 3. The element type size is byte-aligned
3066	// We only handle the cases that the ld/st meet these conditions
3067	auto IsTypeValidForTreeStructuredMerge = [&](Type Ty) -> bool* {
3068	auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: Ty);
3069	return FixedVecTy &&
3070	DL.getTypeSizeInBits(Ty: FixedVecTy->getElementType()) % `8` == `0` &&
3071	!FixedVecTy->getElementType()->isPointerTy();
3072	};
3073
3074	for (Slice &S : P) {
3075	auto *User = cast<Instruction>(Val: S.getUse()->getUser());
3076	// A "full-width" slice spans the entire alloca; it's either the single
3077	// init store (Pattern 2) or the single final load (both patterns).
3078	bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
3079	S.endOffset() == NewAllocaEndOffset);
3080	if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
3081	// Only handle simple (non-volatile, non-atomic) loads.
3082	if (!LI->isSimple() \|\|
3083	!IsTypeValidForTreeStructuredMerge(LI->getType()))
3084	return std::nullopt;
3085	if (IsFullWidth) {
3086	// We accept at most one full-width load (the "final" load, after
3087	// all the partial stores).
3088	if (FullLoad)
3089	return std::nullopt;
3090	FullLoad = LI;
3091	} else {
3092	// Partial load (RMW pattern only).
3093	LoadInfos.push_back(Elt: {.Load: LI, .BeginOffset: S.beginOffset(), .EndOffset: S.endOffset()});
3094	}
3095	} else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
3096	// Do not handle the case if
3097	// 1. The store does not meet the conditions in the helper function
3098	// 2. The store is not simple — we drop stores as part of the
3099	// rewrite, so volatile stores (which must be kept) and atomic
3100	// stores (which carry memory-ordering semantics) are unsound
3101	// to replace with SSA bookkeeping.
3102	// 3. The total store size is not a multiple of the allocated
3103	// element type size (required so the tree merge can produce a
3104	// vector whose element type matches the alloca).
3105	if (!SI->isSimple() \|\| !IsTypeValidForTreeStructuredMerge(
3106	SI->getValueOperand()->getType()))
3107	return std::nullopt;
3108	auto *StVecTy = cast<FixedVectorType>(Val: SI->getValueOperand()->getType());
3109	unsigned NumElts = StVecTy->getNumElements();
3110	unsigned EltSize = DL.getTypeSizeInBits(Ty: StVecTy->getElementType());
3111	if (NumElts * EltSize % AllocatedEltTySize != `0`)
3112	return std::nullopt;
3113	if (IsFullWidth) {
3114	// At most one full-width store is allowed — it's the init store
3115	// for the RMW pattern.
3116	if (InitStore)
3117	return std::nullopt;
3118	InitStore = SI;
3119	} else {
3120	StoreInfos.emplace_back(Args&: SI, Args: S.beginOffset(), Args: S.endOffset(),
3121	Args: SI->getValueOperand());
3122	}
3123	} else {
3124	// If we have instructions other than load and store, we cannot do
3125	// the tree structured merge.
3126	return std::nullopt;
3127	}
3128	}
3129
3130	// Need at least two partial stores to benefit from tree-merging; a
3131	// single store is already optimal as-is. This applies to both patterns
3132	// below, so check it before classifying.
3133	if (StoreInfos.size() < `2`)
3134	return std::nullopt;
3135
3136	// Classify the pattern by looking at what we collected:
3137	// Pattern 1 (stores-only): only partial stores + exactly one full load.
3138	// Pattern 2 (RMW): one full init store + partial loads + partial stores
3139	// (+ optional full final load). RMW also needs VecTy to be set
3140	// because we use getIndex() to convert byte offsets to element
3141	// indices, which requires a promoted vector alloca.
3142	bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
3143	bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
3144	if (!IsRMWPattern && !IsStoresOnlyPattern)
3145	return std::nullopt;
3146
3147	// All partial stores must live in the same basic block — the tree merge
3148	// is built in a single BB using block-order ordering (comesBefore).
3149	BasicBlock *StoreBB = StoreInfos [`0`].Store->getParent();
3150	for (auto &Info : StoreInfos)
3151	if (Info.Store->getParent() != StoreBB)
3152	return std::nullopt;
3153
3154	SmallVector<Value *, `4`> DeletedValues;
3155
3156	// Helper: pairwise tree-merge a list of vectors into a single vector.
3157	// At each iteration we merge each adjacent pair via mergeTwoVectors,
3158	// collect the merged values into Next, and (if Vals had odd length)
3159	// carry the trailing element through unchanged. Loop until one value
3160	// remains — the fully-merged vector.
3161	auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3162	IRBuilder<> &B) -> Value * {
3163	LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3164	while (Vals.size() > `1`) {
3165	SmallVector<Value *, `8`> Next;
3166	for (unsigned I = `0`, E = Vals.size(); I + `1` < E; I += `2`) {
3167	Value *M =
3168	mergeTwoVectors(V0: Vals [I], V1: Vals [I + `1`], DL, NewAIEltTy: AllocatedEltTy, Builder&: B);
3169	LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3170	Next.push_back(Elt: M);
3171	}
3172	if (Vals.size() % `2` == `1`)
3173	Next.push_back(Elt: Vals.back());
3174	Vals = std::move(Next);
3175	}
3176	return Vals [`0`];
3177	};
3178
3179	// Replace a full-width load with a load of the freshly-merged alloca.
3180	// The merge stored a value of type Merged->getType() into NewAI; we load
3181	// that same type back so every access to NewAI stays consistently typed
3182	// (otherwise the alloca is no longer promotable).
3183	auto ReplaceFullLoad = [&](LoadInst LoadToReplace, Value Merged) {
3184	IRBuilder<> LoadBuilder(LoadToReplace);
3185	Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3186	Ty: Merged->getType(), Ptr: &NewAI, Align: getSliceAlign(),
3187	isVolatile: LoadToReplace->isVolatile(),
3188	Name: LoadToReplace->getName() + ".sroa.new.load");
3189	if (NewLoad->getType() != LoadToReplace->getType())
3190	NewLoad = LoadBuilder.CreateBitCast(V: NewLoad, DestTy: LoadToReplace->getType());
3191	LoadToReplace->replaceAllUsesWith(V: NewLoad);
3192	DeletedValues.push_back(Elt: LoadToReplace);
3193	};
3194
3195	if (IsStoresOnlyPattern) {
3196	// Stores should not overlap and should cover the whole alloca.
3197	// Sort by begin offset to verify this with a single linear scan.
3198	llvm::sort(C&: StoreInfos, Comp: [](const StoreInfo &A, const StoreInfo &B) {
3199	return A.BeginOffset < B.BeginOffset;
3200	});
3201	// Check for gap or overlap: each begin offset must equal the previous
3202	// end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3203	// NewAllocaEndOffset) exactly.
3204	uint64_t Expected = NewAllocaBeginOffset;
3205	for (auto &Info : StoreInfos) {
3206	if (Info.BeginOffset != Expected)
3207	return std::nullopt;
3208	Expected = Info.EndOffset;
3209	}
3210	// Stores cover the entire alloca (no trailing gap either).
3211	if (Expected != NewAllocaEndOffset)
3212	return std::nullopt;
3213
3214	// The load should not be in the middle of the stores.
3215	// Note:
3216	// If the load is in a different basic block from the stores, we can
3217	// still do the tree-structured merge. We don't have store->load
3218	// forwarding here — the merged vector is stored back to NewAI and
3219	// the new load loads from NewAI. The forwarding will be handled
3220	// later when NewAI is promoted.
3221	BasicBlock *LoadBB = FullLoad->getParent();
3222	if (LoadBB == StoreBB) {
3223	for (auto &Info : StoreInfos)
3224	if (!Info.Store->comesBefore(Other: FullLoad))
3225	return std::nullopt;
3226	}
3227
3228	LLVM_DEBUG({
3229	dbgs() << "Tree structured merge rewrite (stores-only):\n";
3230	dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3231	for (auto [I, Info] : enumerate(StoreInfos)) {
3232	dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3233	<< Info.EndOffset << ") \tStore: " << *Info.Store
3234	<< "\tValue: " << *Info.StoredValue << "\n";
3235	}
3236	});
3237
3238	// StoreInfos is sorted by offset, not by block order. Anchoring to
3239	// StoreInfos.back().Store (last by offset) can place shuffles before
3240	// operands that appear later in the block (invalid SSA). Insert before
3241	// FullLoad when it shares the store block (after all stores, before
3242	// any later IR in that block). Otherwise insert before the store
3243	// block's terminator so the merge runs after every store and any
3244	// trailing instructions in that block.
3245	IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(Val: FullLoad)
3246	: StoreBB->getTerminator());
3247	SmallVector<Value *, `8`> Vals;
3248	for (const auto &Info : StoreInfos) {
3249	DeletedValues.push_back(Elt: Info.Store);
3250	Vals.push_back(Elt: Info.StoredValue);
3251	}
3252	// Merge all stored values and store the merged value into the alloca.
3253	Value *Merged = TreeMerge(Vals, Builder);
3254	Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3255
3256	// Replace the original load with a load of the newly-merged alloca.
3257	ReplaceFullLoad(FullLoad, Merged);
3258	return DeletedValues;
3259	}
3260
3261	// RMW pattern handling starts from here.
3262	// Like StoreBB above: keep the init store, all partial loads and all
3263	// partial stores in one basic block so we can reason about ordering
3264	// with comesBefore and build SSA without PHIs.
3265	if (InitStore->getParent() != StoreBB)
3266	return std::nullopt;
3267	if (any_of(Range&: LoadInfos, P: [&](const LoadInfo &I) {
3268	return I.Load->getParent() != StoreBB;
3269	}))
3270	return std::nullopt;
3271	// FullLoad (if any) is allowed to live in a different basic block. See
3272	// the note on the stores-only path: we don't do store->load forwarding
3273	// directly — the merged vector is stored to NewAI and the new load
3274	// loads from NewAI, so cross-BB ordering is resolved later when NewAI
3275	// is promoted.
3276
3277	// Collect the combined partial-load/partial-store accesses sorted
3278	// by block order. Used both for ordering checks and for the rewrite
3279	// walk below.
3280	struct Access {
3281	Instruction *Inst;
3282	uint64_t BeginOffset, EndOffset;
3283	bool IsStore;
3284	};
3285	SmallVector<Access, `16`> Accesses;
3286	Accesses.reserve(N: LoadInfos.size() + StoreInfos.size());
3287	for (const auto &L : LoadInfos)
3288	Accesses.push_back(Elt: {.Inst: L.Load, .BeginOffset: L.BeginOffset, .EndOffset: L.EndOffset, .IsStore: false});
3289	for (const auto &S : StoreInfos)
3290	Accesses.push_back(Elt: {.Inst: S.Store, .BeginOffset: S.BeginOffset, .EndOffset: S.EndOffset, .IsStore: true});
3291	llvm::sort(C&: Accesses, Comp: [](const Access &A, const Access &B) {
3292	return A.Inst->comesBefore(Other: B.Inst);
3293	});
3294
3295	// Ordering constraint 1: InitStore must come before every partial
3296	// access — they read/write the RMW state initialised by InitStore.
3297	// Accesses is sorted by block order, so the first element is the
3298	// earliest; checking it is enough.
3299	if (!InitStore->comesBefore(Other: Accesses.front().Inst))
3300	return std::nullopt;
3301	// Ordering constraint 2: when FullLoad shares the block with the
3302	// partial accesses, it must come after every one of them — otherwise
3303	// it could read a stale value. Accesses is sorted, so the last
3304	// element is the latest; checking it is enough. If FullLoad is in
3305	// another block, mem2reg forwards the merged store to it.
3306	if (FullLoad && FullLoad->getParent() == StoreBB &&
3307	!Accesses.back().Inst->comesBefore(Other: FullLoad))
3308	return std::nullopt;
3309
3310	// Coverage check: the distinct [begin, end) ranges touched by the
3311	// partial loads and stores must tile the alloca disjointly. That is
3312	// the only precondition the per-range SliceValues tracking below
3313	// needs — a disjoint tile guarantees the entries don't alias each
3314	// other. We don't check per-range load/store counts: a range with
3315	// only loads ends with SliceValues[r] = the init extract
3316	// (contributed to the final tree-merge), and a range with only
3317	// stores ends with SliceValues[r] = its last stored value. Both are
3318	// correct.
3319	using SliceRange = std::pair<uint64_t, uint64_t>;
3320	SmallVector<SliceRange, `8`> SortedRanges;
3321	SortedRanges.reserve(N: Accesses.size());
3322	for (auto &Acc : Accesses)
3323	SortedRanges.emplace_back(Args&: Acc.BeginOffset, Args&: Acc.EndOffset);
3324	llvm::sort(C&: SortedRanges);
3325	SortedRanges.erase(CS: llvm::unique(R&: SortedRanges), CE: SortedRanges.end());
3326	// Disjoint + contiguous tile of the whole alloca.
3327	uint64_t Expected = NewAllocaBeginOffset;
3328	for (auto &Range : SortedRanges) {
3329	if (Range.first != Expected)
3330	return std::nullopt;
3331	Expected = Range.second;
3332	}
3333	if (Expected != NewAllocaEndOffset)
3334	return std::nullopt;
3335
3336	LLVM_DEBUG({
3337	dbgs() << "Tree structured merge rewrite (RMW):\n";
3338	dbgs() << " Init store: " << *InitStore << "\n";
3339	if (FullLoad)
3340	dbgs() << " Final load: " << *FullLoad << "\n";
3341	dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3342	for (auto &Range : SortedRanges)
3343	dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3344	});
3345
3346	// Initialize SliceValues: one SSA value per slice range, tracking
3347	// the value the alloca currently holds at that range. Each entry
3348	// starts at the corresponding piece of the init store, obtained by
3349	// bitcasting the init value to the alloca's vector type (if needed)
3350	// and extracting the slice's sub-range.
3351	IRB.SetInsertPoint(InitStore->getNextNode());
3352	Value *InitVec = InitStore->getValueOperand();
3353	if (InitVec->getType() != NewAllocaTy)
3354	InitVec = IRB.CreateBitCast(V: InitVec, DestTy: NewAllocaTy, Name: "init.cast");
3355	DenseMap<SliceRange, Value *> SliceValues;
3356	for (auto &Range : SortedRanges) {
3357	unsigned BeginIdx = getIndex(Offset: Range.first);
3358	unsigned EndIdx = getIndex(Offset: Range.second);
3359	SliceValues [Range] = IRB.CreateShuffleVector(
3360	V: InitVec, Mask: createSequentialMask(Start: BeginIdx, NumInts: EndIdx - BeginIdx, NumUndefs: `0`),
3361	Name: "init.extract");
3362	}
3363	// The init store itself becomes dead — its value is consumed via the
3364	// extracts above.
3365	DeletedValues.push_back(Elt: InitStore);
3366
3367	// Walk accesses in block order:
3368	// - partial load at range r: replace with SliceValues[r] (bitcast
3369	// if the load's type differs from the current tracked value's
3370	// type, e.g. because a previous store wrote a vector with a
3371	// different element type);
3372	// - partial store at range r: update SliceValues[r] to the stored
3373	// value and drop the store.
3374	for (auto &Acc : Accesses) {
3375	SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3376	if (!Acc.IsStore) {
3377	Value *V = SliceValues [Range];
3378	if (V->getType() != Acc.Inst->getType()) {
3379	IRB.SetInsertPoint(cast<LoadInst>(Val: Acc.Inst));
3380	V = IRB.CreateBitCast(V, DestTy: Acc.Inst->getType());
3381	}
3382	Acc.Inst->replaceAllUsesWith(V);
3383	} else {
3384	SliceValues [Range] = cast<StoreInst>(Val: Acc.Inst)->getValueOperand();
3385	}
3386	DeletedValues.push_back(Elt: Acc.Inst);
3387	}
3388
3389	// Tree-merge the final per-range values (in range order) into the
3390	// alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3391	// shares the partial-access block) or otherwise to the block's
3392	// terminator — never to a partial access, since those are queued for
3393	// deletion. Both anchors are guaranteed to dominate every SliceValues
3394	// entry: each one is either an init extract (before any access) or a
3395	// stored value defined before its (now-deleted) store.
3396	IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3397	? cast<Instruction>(Val: FullLoad)
3398	: StoreBB->getTerminator());
3399	SmallVector<Value *, `8`> Vals;
3400	for (auto &Range : SortedRanges)
3401	Vals.push_back(Elt: SliceValues [Range]);
3402	Value *Merged = TreeMerge(Vals, Builder);
3403	Builder.CreateAlignedStore(Val: Merged, Ptr: &NewAI, Align: getSliceAlign());
3404
3405	// Replace the optional final full-width load with a load of the newly
3406	// merged alloca. Later promotion will forward the store above to it.
3407	if (FullLoad)
3408	ReplaceFullLoad(FullLoad, Merged);
3409
3410	return DeletedValues;
3411	}
3412
3413	private:
3414	// Make sure the other visit overloads are visible.
3415	using Base::visit;
3416
3417	// Every instruction which can end up as a user must have a rewrite rule.
3418	bool visitInstruction(Instruction &I) {
3419	LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3420	llvm_unreachable("No rewrite rule for this instruction!");
3421	}
3422
3423	Value getNewAllocaSlicePtr(IRBuilderTy &IRB, Type PointerTy) {
3424	// Note that the offset computation can use BeginOffset or NewBeginOffset
3425	// interchangeably for unsplit slices.
3426	assert(IsSplit \|\| BeginOffset == NewBeginOffset);
3427	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3428
3429	StringRef OldName = OldPtr->getName();
3430	// Skip through the last '.sroa.' component of the name.
3431	size_t LastSROAPrefix = OldName.rfind(Str: ".sroa.");
3432	if (LastSROAPrefix != StringRef::npos) {
3433	OldName = OldName.substr(Start: LastSROAPrefix + strlen(s: ".sroa."));
3434	// Look for an SROA slice index.
3435	size_t IndexEnd = OldName.find_first_not_of(Chars: "0123456789");
3436	if (IndexEnd != StringRef::npos && OldName [IndexEnd] == `'.'`) {
3437	// Strip the index and look for the offset.
3438	OldName = OldName.substr(Start: IndexEnd + `1`);
3439	size_t OffsetEnd = OldName.find_first_not_of(Chars: "0123456789");
3440	if (OffsetEnd != StringRef::npos && OldName [OffsetEnd] == `'.'`)
3441	// Strip the offset.
3442	OldName = OldName.substr(Start: OffsetEnd + `1`);
3443	}
3444	}
3445	// Strip any SROA suffixes as well.
3446	OldName = OldName.substr(Start: `0`, N: OldName.find(Str: ".sroa_"));
3447
3448	return getAdjustedPtr(IRB, DL, Ptr: &NewAI,
3449	Offset: APInt (DL.getIndexTypeSizeInBits(Ty: PointerTy), Offset),
3450	PointerTy, NamePrefix: Twine (OldName) + ".");
3451	}
3452
3453	/// Compute suitable alignment to access this slice of the new
3454	/// alloca.
3455	///
3456	/// You can optionally pass a type to this routine and if that type's ABI
3457	/// alignment is itself suitable, this will return zero.
3458	Align getSliceAlign() {
3459	return commonAlignment(A: NewAI.getAlign(),
3460	Offset: NewBeginOffset - NewAllocaBeginOffset);
3461	}
3462
3463	unsigned getIndex(uint64_t Offset) {
3464	assert(VecTy && "Can only call getIndex when rewriting a vector");
3465	uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3466	assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3467	uint32_t Index = RelOffset / ElementSize;
3468	assert(Index * ElementSize == RelOffset);
3469	return Index;
3470	}
3471
3472	void deleteIfTriviallyDead(Value *V) {
3473	Instruction *I = cast<Instruction>(Val: V);
3474	if (isInstructionTriviallyDead(I))
3475	Pass.DeadInsts.push_back(Elt: I);
3476	}
3477
3478	Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3479	unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3480	unsigned EndIndex = getIndex(Offset: NewEndOffset);
3481	assert(EndIndex > BeginIndex && "Empty vector!");
3482
3483	LoadInst *Load =
3484	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3485
3486	Load->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3487	LLVMContext::MD_access_group});
3488	return extractVector(IRB, V: Load, BeginIndex, EndIndex, Name: "vec");
3489	}
3490
3491	Value *rewriteIntegerLoad(LoadInst &LI) {
3492	assert(IntTy && "We cannot insert an integer to the alloca");
3493	assert(!LI.isVolatile());
3494	Value *V =
3495	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3496	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: IntTy);
3497	assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3498	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3499	if (Offset > `0` \|\| NewEndOffset < NewAllocaEndOffset) {
3500	IntegerType ExtractTy = Type::getIntNTy(C&: LI.getContext(), N: SliceSize `8`);
3501	V = extractInteger(DL, IRB, V, Ty: ExtractTy, Offset, Name: "extract");
3502	}
3503	// It is possible that the extracted type is not the load type. This
3504	// happens if there is a load past the end of the alloca, and as
3505	// a consequence the slice is narrower but still a candidate for integer
3506	// lowering. To handle this case, we just zero extend the extracted
3507	// integer.
3508	assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * `8` &&
3509	"Can only handle an extract for an overly wide load");
3510	if (cast<IntegerType>(Val: LI.getType())->getBitWidth() > SliceSize * `8`)
3511	V = IRB.CreateZExt(V, DestTy: LI.getType());
3512	return V;
3513	}
3514
3515	bool visitLoadInst(LoadInst &LI) {
3516	LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3517	Value *OldOp = LI.getOperand(i_nocapture: `0`);
3518	assert(OldOp == OldPtr);
3519
3520	AAMDNodes AATags = LI.getAAMetadata();
3521
3522	unsigned AS = LI.getPointerAddressSpace();
3523
3524	Type TargetTy = IsSplit ? Type::getIntNTy(C&: LI.getContext(), N: SliceSize `8`)
3525	: LI.getType();
3526	bool IsPtrAdjusted = false;
3527	Value *V;
3528	if (VecTy) {
3529	V = rewriteVectorizedLoadInst(LI);
3530	} else if (IntTy && LI.getType()->isIntegerTy()) {
3531	V = rewriteIntegerLoad(LI);
3532	} else if (NewBeginOffset == NewAllocaBeginOffset &&
3533	NewEndOffset == NewAllocaEndOffset &&
3534	(canConvertValue(DL, OldTy: NewAllocaTy, NewTy: TargetTy) \|\|
3535	(NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3536	DL.getTypeStoreSize(Ty: TargetTy).getFixedValue() > SliceSize &&
3537	!LI.isVolatile()))) {
3538	Value *NewPtr =
3539	getPtrToNewAI(AddrSpace: LI.getPointerAddressSpace(), IsVolatile: LI.isVolatile());
3540	LoadInst *NewLI = IRB.CreateAlignedLoad(
3541	Ty: NewAllocaTy, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3542	if (LI.isVolatile())
3543	NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3544	if (NewLI->isAtomic())
3545	NewLI->setAlignment(LI.getAlign());
3546
3547	// Copy any metadata that is valid for the new load. This may require
3548	// conversion to a different kind of metadata, e.g. !nonnull might change
3549	// to !range or vice versa.
3550	copyMetadataForLoad(Dest&: *NewLI, Source: LI);
3551
3552	// Do this after copyMetadataForLoad() to preserve the TBAA shift.
3553	if (AATags)
3554	NewLI->setAAMetadata(AATags.adjustForAccess(
3555	Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3556
3557	// Try to preserve nonnull metadata
3558	V = NewLI;
3559
3560	// If this is an integer load past the end of the slice (which means the
3561	// bytes outside the slice are undef or this load is dead) just forcibly
3562	// fix the integer size with correct handling of endianness.
3563	if (auto *AITy = dyn_cast<IntegerType>(Val: NewAllocaTy))
3564	if (auto *TITy = dyn_cast<IntegerType>(Val: TargetTy))
3565	if (AITy->getBitWidth() < TITy->getBitWidth()) {
3566	V = IRB.CreateZExt(V, DestTy: TITy, Name: "load.ext");
3567	if (DL.isBigEndian())
3568	V = IRB.CreateShl(LHS: V, RHS: TITy->getBitWidth() - AITy->getBitWidth(),
3569	Name: "endian_shift");
3570	}
3571	} else {
3572	Type *LTy = IRB.getPtrTy(AddrSpace: AS);
3573	LoadInst *NewLI =
3574	IRB.CreateAlignedLoad(Ty: TargetTy, Ptr: getNewAllocaSlicePtr(IRB, PointerTy: LTy),
3575	Align: getSliceAlign(), isVolatile: LI.isVolatile(), Name: LI.getName());
3576
3577	if (AATags)
3578	NewLI->setAAMetadata(AATags.adjustForAccess(
3579	Offset: NewBeginOffset - BeginOffset, AccessTy: NewLI->getType(), DL));
3580
3581	if (LI.isVolatile())
3582	NewLI->setAtomic(Ordering: LI.getOrdering(), SSID: LI.getSyncScopeID());
3583	NewLI->copyMetadata(SrcInst: LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3584	LLVMContext::MD_access_group});
3585
3586	V = NewLI;
3587	IsPtrAdjusted = true;
3588	}
3589	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: TargetTy);
3590
3591	if (IsSplit) {
3592	assert(!LI.isVolatile());
3593	assert(LI.getType()->isIntegerTy() &&
3594	"Only integer type loads and stores are split");
3595	assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3596	"Split load isn't smaller than original load");
3597	assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3598	"Non-byte-multiple bit width");
3599	// Move the insertion point just past the load so that we can refer to it.
3600	BasicBlock::iterator LIIt = std::next(x: LI.getIterator());
3601	// Ensure the insertion point comes before any debug-info immediately
3602	// after the load, so that variable values referring to the load are
3603	// dominated by it.
3604	LIIt.setHeadBit(true);
3605	IRB.SetInsertPoint(LIIt);
3606	// Create a placeholder value with the same type as LI to use as the
3607	// basis for the new value. This allows us to replace the uses of LI with
3608	// the computed value, and then replace the placeholder with LI, leaving
3609	// LI only used for this computation.
3610	Value *Placeholder =
3611	new LoadInst (LI.getType(), PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AS)), "",
3612	false, Align (`1`));
3613	V = insertInteger(DL, IRB, Old: Placeholder, V, Offset: NewBeginOffset - BeginOffset,
3614	Name: "insert");
3615	LI.replaceAllUsesWith(V);
3616	Placeholder->replaceAllUsesWith(V: &LI);
3617	Placeholder->deleteValue();
3618	} else {
3619	LI.replaceAllUsesWith(V);
3620	}
3621
3622	Pass.DeadInsts.push_back(Elt: &LI);
3623	deleteIfTriviallyDead(V: OldOp);
3624	LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3625	return !LI.isVolatile() && !IsPtrAdjusted;
3626	}
3627
3628	bool rewriteVectorizedStoreInst(Value V, StoreInst &SI, Value OldOp,
3629	AAMDNodes AATags) {
3630	// Capture V for the purpose of debug-info accounting once it's converted
3631	// to a vector store.
3632	Value *OrigV = V;
3633	if (V->getType() != VecTy) {
3634	unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3635	unsigned EndIndex = getIndex(Offset: NewEndOffset);
3636	assert(EndIndex > BeginIndex && "Empty vector!");
3637	unsigned NumElements = EndIndex - BeginIndex;
3638	assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3639	"Too many elements!");
3640	Type *SliceTy = (NumElements == `1`)
3641	? ElementTy
3642	: FixedVectorType::get(ElementType: ElementTy, NumElts: NumElements);
3643	if (V->getType() != SliceTy)
3644	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: SliceTy);
3645
3646	// Mix in the existing elements.
3647	Value *Old =
3648	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
3649	V = insertVector(IRB, Old, V, BeginIndex, Name: "vec");
3650	}
3651	StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3652	Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3653	LLVMContext::MD_access_group});
3654	if (AATags)
3655	Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3656	AccessTy: V->getType(), DL));
3657	Pass.DeadInsts.push_back(Elt: &SI);
3658
3659	// NOTE: Careful to use OrigV rather than V.
3660	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &SI,
3661	Inst: Store, Dest: Store->getPointerOperand(), Value: OrigV, DL);
3662	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3663	return true;
3664	}
3665
3666	bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3667	assert(IntTy && "We cannot extract an integer from the alloca");
3668	assert(!SI.isVolatile());
3669	if (DL.getTypeSizeInBits(Ty: V->getType()).getFixedValue() !=
3670	IntTy->getBitWidth()) {
3671	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3672	Name: "oldload");
3673	Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3674	assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3675	uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3676	V = insertInteger(DL, IRB, Old, V: SI.getValueOperand(), Offset, Name: "insert");
3677	}
3678	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3679	StoreInst *Store = IRB.CreateAlignedStore(Val: V, Ptr: &NewAI, Align: NewAI.getAlign());
3680	Store->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3681	LLVMContext::MD_access_group});
3682	if (AATags)
3683	Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3684	AccessTy: V->getType(), DL));
3685
3686	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &SI,
3687	Inst: Store, Dest: Store->getPointerOperand(),
3688	Value: Store->getValueOperand(), DL);
3689
3690	Pass.DeadInsts.push_back(Elt: &SI);
3691	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3692	return true;
3693	}
3694
3695	bool visitStoreInst(StoreInst &SI) {
3696	LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3697	Value *OldOp = SI.getOperand(i_nocapture: `1`);
3698	assert(OldOp == OldPtr);
3699
3700	AAMDNodes AATags = SI.getAAMetadata();
3701	Value *V = SI.getValueOperand();
3702
3703	// Strip all inbounds GEPs and pointer casts to try to dig out any root
3704	// alloca that should be re-examined after promoting this alloca.
3705	if (V->getType()->isPointerTy())
3706	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: V->stripInBoundsOffsets()))
3707	Pass.PostPromotionWorklist.insert(X: AI);
3708
3709	TypeSize StoreSize = DL.getTypeStoreSize(Ty: V->getType());
3710	if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3711	assert(!SI.isVolatile());
3712	assert(V->getType()->isIntegerTy() &&
3713	"Only integer type loads and stores are split");
3714	assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3715	"Non-byte-multiple bit width");
3716	IntegerType NarrowTy = Type::getIntNTy(C&: SI.getContext(), N: SliceSize `8`);
3717	V = extractInteger(DL, IRB, V, Ty: NarrowTy, Offset: NewBeginOffset - BeginOffset,
3718	Name: "extract");
3719	}
3720
3721	if (VecTy)
3722	return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3723	if (IntTy && V->getType()->isIntegerTy())
3724	return rewriteIntegerStore(V, SI, AATags);
3725
3726	StoreInst *NewSI;
3727	if (NewBeginOffset == NewAllocaBeginOffset &&
3728	NewEndOffset == NewAllocaEndOffset &&
3729	canConvertValue(DL, OldTy: V->getType(), NewTy: NewAllocaTy)) {
3730	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3731	Value *NewPtr =
3732	getPtrToNewAI(AddrSpace: SI.getPointerAddressSpace(), IsVolatile: SI.isVolatile());
3733
3734	NewSI =
3735	IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: SI.isVolatile());
3736	} else {
3737	unsigned AS = SI.getPointerAddressSpace();
3738	Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: IRB.getPtrTy(AddrSpace: AS));
3739	NewSI =
3740	IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: getSliceAlign(), isVolatile: SI.isVolatile());
3741	}
3742	NewSI->copyMetadata(SrcInst: SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
3743	LLVMContext::MD_access_group});
3744	if (AATags)
3745	NewSI->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3746	AccessTy: V->getType(), DL));
3747	if (SI.isVolatile())
3748	NewSI->setAtomic(Ordering: SI.getOrdering(), SSID: SI.getSyncScopeID());
3749	if (NewSI->isAtomic())
3750	NewSI->setAlignment(SI.getAlign());
3751
3752	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &SI,
3753	Inst: NewSI, Dest: NewSI->getPointerOperand(),
3754	Value: NewSI->getValueOperand(), DL);
3755
3756	Pass.DeadInsts.push_back(Elt: &SI);
3757	deleteIfTriviallyDead(V: OldOp);
3758
3759	LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3760	return NewSI->getPointerOperand() == &NewAI &&
3761	NewSI->getValueOperand()->getType() == NewAllocaTy &&
3762	!SI.isVolatile();
3763	}
3764
3765	/// Compute an integer value from splatting an i8 across the given
3766	/// number of bytes.
3767	///
3768	/// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3769	/// call this routine.
3770	/// FIXME: Heed the advice above.
3771	///
3772	/// \param V The i8 value to splat.
3773	/// \param Size The number of bytes in the output (assuming i8 is one byte)
3774	Value getIntegerSplat(Value V, unsigned Size) {
3775	assert(Size > `0` && "Expected a positive number of bytes.");
3776	IntegerType *VTy = cast<IntegerType>(Val: V->getType());
3777	assert(VTy->getBitWidth() == `8` && "Expected an i8 value for the byte");
3778	if (Size == `1`)
3779	return V;
3780
3781	Type SplatIntTy = Type::getIntNTy(C&: VTy->getContext(), N: Size `8`);
3782	V = IRB.CreateMul(
3783	LHS: IRB.CreateZExt(V, DestTy: SplatIntTy, Name: "zext"),
3784	RHS: IRB.CreateUDiv(LHS: Constant::getAllOnesValue(Ty: SplatIntTy),
3785	RHS: IRB.CreateZExt(V: Constant::getAllOnesValue(Ty: V->getType()),
3786	DestTy: SplatIntTy)),
3787	Name: "isplat");
3788	return V;
3789	}
3790
3791	/// Compute a vector splat for a given element value.
3792	Value getVectorSplat(Value V, unsigned NumElements) {
3793	V = IRB.CreateVectorSplat(NumElts: NumElements, V, Name: "vsplat");
3794	LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3795	return V;
3796	}
3797
3798	bool visitMemSetInst(MemSetInst &II) {
3799	LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3800	assert(II.getRawDest() == OldPtr);
3801
3802	AAMDNodes AATags = II.getAAMetadata();
3803
3804	// If the memset has a variable size, it cannot be split, just adjust the
3805	// pointer to the new alloca.
3806	if (!isa<ConstantInt>(Val: II.getLength())) {
3807	assert(!IsSplit);
3808	assert(NewBeginOffset == BeginOffset);
3809	II.setDest(getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()));
3810	II.setDestAlignment(getSliceAlign());
3811	// In theory we should call migrateDebugInfo here. However, we do not
3812	// emit dbg.assign intrinsics for mem intrinsics storing through non-
3813	// constant geps, or storing a variable number of bytes.
3814	assert(at::getDVRAssignmentMarkers(&II).empty() &&
3815	"AT: Unexpected link to non-const GEP");
3816	deleteIfTriviallyDead(V: OldPtr);
3817	return false;
3818	}
3819
3820	// Record this instruction for deletion.
3821	Pass.DeadInsts.push_back(Elt: &II);
3822
3823	Type *ScalarTy = NewAllocaTy->getScalarType();
3824
3825	const bool CanContinue = [&]() {
3826	if (VecTy \|\| IntTy)
3827	return true;
3828	if (BeginOffset > NewAllocaBeginOffset \|\| EndOffset < NewAllocaEndOffset)
3829	return false;
3830	// Length must be in range for FixedVectorType.
3831	auto *C = cast<ConstantInt>(Val: II.getLength());
3832	const uint64_t Len = C->getLimitedValue();
3833	if (Len > std::numeric_limits<unsigned>::max())
3834	return false;
3835	auto *Int8Ty = IntegerType::getInt8Ty(C&: NewAI.getContext());
3836	auto *SrcTy = FixedVectorType::get(ElementType: Int8Ty, NumElts: Len);
3837	return canConvertValue(DL, OldTy: SrcTy, NewTy: NewAllocaTy) &&
3838	DL.isLegalInteger(Width: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue());
3839	}();
3840
3841	// If this doesn't map cleanly onto the alloca type, and that type isn't
3842	// a single value type, just emit a memset.
3843	if (!CanContinue) {
3844	Type *SizeTy = II.getLength()->getType();
3845	unsigned Sz = NewEndOffset - NewBeginOffset;
3846	Constant *Size = ConstantInt::get(Ty: SizeTy, V: Sz);
3847	MemIntrinsic *New = cast<MemIntrinsic>(Val: IRB.CreateMemSet(
3848	Ptr: getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType()), Val: II.getValue(), Size,
3849	Align: MaybeAlign (getSliceAlign()), isVolatile: II.isVolatile()));
3850	if (AATags)
3851	New->setAAMetadata(
3852	AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset, AccessSize: Sz));
3853
3854	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &II,
3855	Inst: New, Dest: New->getRawDest(), Value: nullptr, DL);
3856
3857	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3858	return false;
3859	}
3860
3861	// If we can represent this as a simple value, we have to build the actual
3862	// value to store, which requires expanding the byte present in memset to
3863	// a sensible representation for the alloca type. This is essentially
3864	// splatting the byte to a sufficiently wide integer, splatting it across
3865	// any desired vector width, and bitcasting to the final type.
3866	Value *V;
3867
3868	if (VecTy) {
3869	// If this is a memset of a vectorized alloca, insert it.
3870	assert(ElementTy == ScalarTy);
3871
3872	unsigned BeginIndex = getIndex(Offset: NewBeginOffset);
3873	unsigned EndIndex = getIndex(Offset: NewEndOffset);
3874	assert(EndIndex > BeginIndex && "Empty vector!");
3875	unsigned NumElements = EndIndex - BeginIndex;
3876	assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3877	"Too many elements!");
3878
3879	Value *Splat = getIntegerSplat(
3880	V: II.getValue(), Size: DL.getTypeSizeInBits(Ty: ElementTy).getFixedValue() / `8`);
3881	Splat = IRB.CreateBitPreservingCastChain(DL, V: Splat, NewTy: ElementTy);
3882	if (NumElements > `1`)
3883	Splat = getVectorSplat(V: Splat, NumElements);
3884
3885	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
3886	Name: "oldload");
3887	V = insertVector(IRB, Old, V: Splat, BeginIndex, Name: "vec");
3888	} else if (IntTy) {
3889	// If this is a memset on an alloca where we can widen stores, insert the
3890	// set integer.
3891	assert(!II.isVolatile());
3892
3893	uint64_t Size = NewEndOffset - NewBeginOffset;
3894	V = getIntegerSplat(V: II.getValue(), Size);
3895
3896	if (IntTy && (NewBeginOffset != NewAllocaBeginOffset \|\|
3897	NewEndOffset != NewAllocaEndOffset)) {
3898	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI,
3899	Align: NewAI.getAlign(), Name: "oldload");
3900	Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
3901	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3902	V = insertInteger(DL, IRB, Old, V, Offset, Name: "insert");
3903	} else {
3904	assert(V->getType() == IntTy &&
3905	"Wrong type for an alloca wide integer!");
3906	}
3907	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3908	} else {
3909	// Established these invariants above.
3910	assert(NewBeginOffset == NewAllocaBeginOffset);
3911	assert(NewEndOffset == NewAllocaEndOffset);
3912
3913	V = getIntegerSplat(V: II.getValue(),
3914	Size: DL.getTypeSizeInBits(Ty: ScalarTy).getFixedValue() / `8`);
3915	if (VectorType *AllocaVecTy = dyn_cast<VectorType>(Val: NewAllocaTy))
3916	V = getVectorSplat(
3917	V, NumElements: cast<FixedVectorType>(Val: AllocaVecTy)->getNumElements());
3918
3919	V = IRB.CreateBitPreservingCastChain(DL, V, NewTy: NewAllocaTy);
3920	}
3921
3922	Value *NewPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
3923	StoreInst *New =
3924	IRB.CreateAlignedStore(Val: V, Ptr: NewPtr, Align: NewAI.getAlign(), isVolatile: II.isVolatile());
3925	New->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
3926	LLVMContext::MD_access_group});
3927	if (AATags)
3928	New->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
3929	AccessTy: V->getType(), DL));
3930
3931	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &II,
3932	Inst: New, Dest: New->getPointerOperand(), Value: V, DL);
3933
3934	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3935	return !II.isVolatile();
3936	}
3937
3938	bool visitMemTransferInst(MemTransferInst &II) {
3939	// Rewriting of memory transfer instructions can be a bit tricky. We break
3940	// them into two categories: split intrinsics and unsplit intrinsics.
3941
3942	LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3943
3944	AAMDNodes AATags = II.getAAMetadata();
3945
3946	bool IsDest = &II.getRawDestUse() == OldUse;
3947	assert((IsDest && II.getRawDest() == OldPtr) \|\|
3948	(!IsDest && II.getRawSource() == OldPtr));
3949
3950	Align SliceAlign = getSliceAlign();
3951	// For unsplit intrinsics, we simply modify the source and destination
3952	// pointers in place. This isn't just an optimization, it is a matter of
3953	// correctness. With unsplit intrinsics we may be dealing with transfers
3954	// within a single alloca before SROA ran, or with transfers that have
3955	// a variable length. We may also be dealing with memmove instead of
3956	// memcpy, and so simply updating the pointers is the necessary for us to
3957	// update both source and dest of a single call.
3958	if (!IsSplittable) {
3959	Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
3960	if (IsDest) {
3961	// Update the address component of linked dbg.assigns.
3962	for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(Inst: &II)) {
3963	if (llvm::is_contained(Range: DbgAssign->location_ops(), Element: II.getDest()) \|\|
3964	DbgAssign->getAddress() == II.getDest())
3965	DbgAssign->replaceVariableLocationOp(OldValue: II.getDest(), NewValue: AdjustedPtr);
3966	}
3967	II.setDest(AdjustedPtr);
3968	II.setDestAlignment(SliceAlign);
3969	} else {
3970	II.setSource(AdjustedPtr);
3971	II.setSourceAlignment(SliceAlign);
3972	}
3973
3974	LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3975	deleteIfTriviallyDead(V: OldPtr);
3976	return false;
3977	}
3978	// For split transfer intrinsics we have an incredibly useful assurance:
3979	// the source and destination do not reside within the same alloca, and at
3980	// least one of them does not escape. This means that we can replace
3981	// memmove with memcpy, and we don't need to worry about all manner of
3982	// downsides to splitting and transforming the operations.
3983
3984	// If this doesn't map cleanly onto the alloca type, and that type isn't
3985	// a single value type, just emit a memcpy.
3986	bool EmitMemCpy =
3987	!VecTy && !IntTy &&
3988	(BeginOffset > NewAllocaBeginOffset \|\| EndOffset < NewAllocaEndOffset \|\|
3989	SliceSize != DL.getTypeStoreSize(Ty: NewAllocaTy).getFixedValue() \|\|
3990	!DL.typeSizeEqualsStoreSize(Ty: NewAllocaTy) \|\|
3991	!NewAllocaTy->isSingleValueType());
3992
3993	// If we're just going to emit a memcpy, the alloca hasn't changed, and the
3994	// size hasn't been shrunk based on analysis of the viable range, this is
3995	// a no-op.
3996	if (EmitMemCpy && &OldAI == &NewAI) {
3997	// Ensure the start lines up.
3998	assert(NewBeginOffset == BeginOffset);
3999
4000	// Rewrite the size as needed.
4001	if (NewEndOffset != EndOffset)
4002	II.setLength(NewEndOffset - NewBeginOffset);
4003	return false;
4004	}
4005	// Record this instruction for deletion.
4006	Pass.DeadInsts.push_back(Elt: &II);
4007
4008	// Strip all inbounds GEPs and pointer casts to try to dig out any root
4009	// alloca that should be re-examined after rewriting this instruction.
4010	Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
4011	if (AllocaInst *AI =
4012	dyn_cast<AllocaInst>(Val: OtherPtr->stripInBoundsOffsets())) {
4013	assert(AI != &OldAI && AI != &NewAI &&
4014	"Splittable transfers cannot reach the same alloca on both ends.");
4015	Pass.Worklist.insert(X: AI);
4016	}
4017
4018	Type *OtherPtrTy = OtherPtr->getType();
4019	unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
4020
4021	// Compute the relative offset for the other pointer within the transfer.
4022	unsigned OffsetWidth = DL.getIndexSizeInBits(AS: OtherAS);
4023	APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
4024	Align OtherAlign =
4025	(IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
4026	OtherAlign =
4027	commonAlignment(A: OtherAlign, Offset: OtherOffset.zextOrTrunc(width: `64`).getZExtValue());
4028
4029	if (EmitMemCpy) {
4030	// Compute the other pointer, folding as much as possible to produce
4031	// a single, simple GEP in most cases.
4032	OtherPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
4033	NamePrefix: OtherPtr->getName() + ".");
4034
4035	Value *OurPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4036	Type *SizeTy = II.getLength()->getType();
4037	Constant *Size = ConstantInt::get(Ty: SizeTy, V: NewEndOffset - NewBeginOffset);
4038
4039	Value DestPtr, SrcPtr;
4040	MaybeAlign DestAlign, SrcAlign;
4041	// Note: IsDest is true iff we're copying into the new alloca slice
4042	if (IsDest) {
4043	DestPtr = OurPtr;
4044	DestAlign = SliceAlign;
4045	SrcPtr = OtherPtr;
4046	SrcAlign = OtherAlign;
4047	} else {
4048	DestPtr = OtherPtr;
4049	DestAlign = OtherAlign;
4050	SrcPtr = OurPtr;
4051	SrcAlign = SliceAlign;
4052	}
4053	CallInst *New = IRB.CreateMemCpy(Dst: DestPtr, DstAlign: DestAlign, Src: SrcPtr, SrcAlign,
4054	Size, isVolatile: II.isVolatile());
4055	if (AATags)
4056	New->setAAMetadata(AATags.shift(Offset: NewBeginOffset - BeginOffset));
4057
4058	APInt Offset(DL.getIndexTypeSizeInBits(Ty: DestPtr->getType()), `0`);
4059	if (IsDest) {
4060	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`,
4061	OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
4062	} else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4063	Val: DestPtr->stripAndAccumulateConstantOffsets(
4064	DL, Offset, /AllowNonInbounds/ true))) {
4065	migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * `8`,
4066	SliceSizeInBits: SliceSize * `8`, OldInst: &II, Inst: New, Dest: DestPtr, Value: nullptr, DL);
4067	}
4068	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4069	return false;
4070	}
4071
4072	bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
4073	NewEndOffset == NewAllocaEndOffset;
4074	uint64_t Size = NewEndOffset - NewBeginOffset;
4075	unsigned BeginIndex = VecTy ? getIndex(Offset: NewBeginOffset) : `0`;
4076	unsigned EndIndex = VecTy ? getIndex(Offset: NewEndOffset) : `0`;
4077	unsigned NumElements = EndIndex - BeginIndex;
4078	IntegerType *SubIntTy =
4079	IntTy ? Type::getIntNTy(C&: IntTy->getContext(), N: Size * `8`) : nullptr;
4080
4081	// Reset the other pointer type to match the register type we're going to
4082	// use, but using the address space of the original other pointer.
4083	Type *OtherTy;
4084	if (VecTy && !IsWholeAlloca) {
4085	if (NumElements == `1`)
4086	OtherTy = VecTy->getElementType();
4087	else
4088	OtherTy = FixedVectorType::get(ElementType: VecTy->getElementType(), NumElts: NumElements);
4089	} else if (IntTy && !IsWholeAlloca) {
4090	OtherTy = SubIntTy;
4091	} else {
4092	OtherTy = NewAllocaTy;
4093	}
4094
4095	Value *AdjPtr = getAdjustedPtr(IRB, DL, Ptr: OtherPtr, Offset: OtherOffset, PointerTy: OtherPtrTy,
4096	NamePrefix: OtherPtr->getName() + ".");
4097	MaybeAlign SrcAlign = OtherAlign;
4098	MaybeAlign DstAlign = SliceAlign;
4099	if (!IsDest)
4100	std::swap(a&: SrcAlign, b&: DstAlign);
4101
4102	Value *SrcPtr;
4103	Value *DstPtr;
4104
4105	if (IsDest) {
4106	DstPtr = getPtrToNewAI(AddrSpace: II.getDestAddressSpace(), IsVolatile: II.isVolatile());
4107	SrcPtr = AdjPtr;
4108	} else {
4109	DstPtr = AdjPtr;
4110	SrcPtr = getPtrToNewAI(AddrSpace: II.getSourceAddressSpace(), IsVolatile: II.isVolatile());
4111	}
4112
4113	Value *Src;
4114	if (VecTy && !IsWholeAlloca && !IsDest) {
4115	Src =
4116	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
4117	Src = extractVector(IRB, V: Src, BeginIndex, EndIndex, Name: "vec");
4118	} else if (IntTy && !IsWholeAlloca && !IsDest) {
4119	Src =
4120	IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(), Name: "load");
4121	Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: IntTy);
4122	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4123	Src = extractInteger(DL, IRB, V: Src, Ty: SubIntTy, Offset, Name: "extract");
4124	} else {
4125	LoadInst *Load = IRB.CreateAlignedLoad(Ty: OtherTy, Ptr: SrcPtr, Align: SrcAlign,
4126	isVolatile: II.isVolatile(), Name: "copyload");
4127	Load->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4128	LLVMContext::MD_access_group});
4129	if (AATags)
4130	Load->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4131	AccessTy: Load->getType(), DL));
4132	Src = Load;
4133	}
4134
4135	if (VecTy && !IsWholeAlloca && IsDest) {
4136	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4137	Name: "oldload");
4138	Src = insertVector(IRB, Old, V: Src, BeginIndex, Name: "vec");
4139	} else if (IntTy && !IsWholeAlloca && IsDest) {
4140	Value *Old = IRB.CreateAlignedLoad(Ty: NewAllocaTy, Ptr: &NewAI, Align: NewAI.getAlign(),
4141	Name: "oldload");
4142	Old = IRB.CreateBitPreservingCastChain(DL, V: Old, NewTy: IntTy);
4143	uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4144	Src = insertInteger(DL, IRB, Old, V: Src, Offset, Name: "insert");
4145	Src = IRB.CreateBitPreservingCastChain(DL, V: Src, NewTy: NewAllocaTy);
4146	}
4147
4148	StoreInst *Store = cast<StoreInst>(
4149	Val: IRB.CreateAlignedStore(Val: Src, Ptr: DstPtr, Align: DstAlign, isVolatile: II.isVolatile()));
4150	Store->copyMetadata(SrcInst: II, WL: {LLVMContext::MD_mem_parallel_loop_access,
4151	LLVMContext::MD_access_group});
4152	if (AATags)
4153	Store->setAAMetadata(AATags.adjustForAccess(Offset: NewBeginOffset - BeginOffset,
4154	AccessTy: Src->getType(), DL));
4155
4156	APInt Offset(DL.getIndexTypeSizeInBits(Ty: DstPtr->getType()), `0`);
4157	if (IsDest) {
4158
4159	migrateDebugInfo(OldAlloca: &OldAI, IsSplit, OldAllocaOffsetInBits: NewBeginOffset * `8`, SliceSizeInBits: SliceSize * `8`, OldInst: &II,
4160	Inst: Store, Dest: DstPtr, Value: Src, DL);
4161	} else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4162	Val: DstPtr->stripAndAccumulateConstantOffsets(
4163	DL, Offset, /AllowNonInbounds/ true))) {
4164	migrateDebugInfo(OldAlloca: Base, IsSplit, OldAllocaOffsetInBits: Offset.getZExtValue() * `8`, SliceSizeInBits: SliceSize * `8`,
4165	OldInst: &II, Inst: Store, Dest: DstPtr, Value: Src, DL);
4166	}
4167
4168	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4169	return !II.isVolatile();
4170	}
4171
4172	bool visitIntrinsicInst(IntrinsicInst &II) {
4173	assert((II.isLifetimeStartOrEnd() \|\| II.isDroppable()) &&
4174	"Unexpected intrinsic!");
4175	LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4176
4177	// Record this instruction for deletion.
4178	Pass.DeadInsts.push_back(Elt: &II);
4179
4180	if (II.isDroppable()) {
4181	assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4182	// TODO For now we forget assumed information, this can be improved.
4183	OldPtr->dropDroppableUsesIn(Usr&: II);
4184	return true;
4185	}
4186
4187	assert(II.getArgOperand(`0`) == OldPtr);
4188	Type *PointerTy = IRB.getPtrTy(AddrSpace: OldPtr->getType()->getPointerAddressSpace());
4189	Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4190	Value *New;
4191	if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4192	New = IRB.CreateLifetimeStart(Ptr);
4193	else
4194	New = IRB.CreateLifetimeEnd(Ptr);
4195
4196	(void)New;
4197	LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4198
4199	return true;
4200	}
4201
4202	void fixLoadStoreAlign(Instruction &Root) {
4203	// This algorithm implements the same visitor loop as
4204	// hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4205	// or store found.
4206	SmallPtrSet<Instruction *, `4`> Visited;
4207	SmallVector<Instruction *, `4`> Uses;
4208	Visited.insert(Ptr: &Root);
4209	Uses.push_back(Elt: &Root);
4210	do {
4211	Instruction *I = Uses.pop_back_val();
4212
4213	if (LoadInst *LI = dyn_cast<LoadInst>(Val: I)) {
4214	LI->setAlignment(std::min(a: LI->getAlign(), b: getSliceAlign()));
4215	continue;
4216	}
4217	if (StoreInst *SI = dyn_cast<StoreInst>(Val: I)) {
4218	SI->setAlignment(std::min(a: SI->getAlign(), b: getSliceAlign()));
4219	continue;
4220	}
4221
4222	assert(isa<BitCastInst>(I) \|\| isa<AddrSpaceCastInst>(I) \|\|
4223	isa<PHINode>(I) \|\| isa<SelectInst>(I) \|\|
4224	isa<GetElementPtrInst>(I));
4225	for (User *U : I->users())
4226	if (Visited.insert(Ptr: cast<Instruction>(Val: U)).second)
4227	Uses.push_back(Elt: cast<Instruction>(Val: U));
4228	} while (!Uses.empty());
4229	}
4230
4231	bool visitPHINode(PHINode &PN) {
4232	LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4233	assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4234	assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4235
4236	// We would like to compute a new pointer in only one place, but have it be
4237	// as local as possible to the PHI. To do that, we re-use the location of
4238	// the old pointer, which necessarily must be in the right position to
4239	// dominate the PHI.
4240	IRBuilderBase::InsertPointGuard Guard(IRB);
4241	if (isa<PHINode>(Val: OldPtr))
4242	IRB.SetInsertPoint(OldPtr->getParent()->getFirstInsertionPt());
4243	else
4244	IRB.SetInsertPoint(OldPtr);
4245	IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4246
4247	Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4248	// Replace the operands which were using the old pointer.
4249	std::replace(first: PN.op_begin(), last: PN.op_end(), old_value: cast<Value>(Val: OldPtr), new_value: NewPtr);
4250
4251	LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4252	deleteIfTriviallyDead(V: OldPtr);
4253
4254	// Fix the alignment of any loads or stores using this PHI node.
4255	fixLoadStoreAlign(Root&: PN);
4256
4257	// PHIs can't be promoted on their own, but often can be speculated. We
4258	// check the speculation outside of the rewriter so that we see the
4259	// fully-rewritten alloca.
4260	PHIUsers.insert(X: &PN);
4261	return true;
4262	}
4263
4264	bool visitSelectInst(SelectInst &SI) {
4265	LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4266	assert((SI.getTrueValue() == OldPtr \|\| SI.getFalseValue() == OldPtr) &&
4267	"Pointer isn't an operand!");
4268	assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4269	assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4270
4271	Value *NewPtr = getNewAllocaSlicePtr(IRB, PointerTy: OldPtr->getType());
4272	// Replace the operands which were using the old pointer.
4273	if (SI.getOperand(i_nocapture: `1`) == OldPtr)
4274	SI.setOperand(i_nocapture: `1`, Val_nocapture: NewPtr);
4275	if (SI.getOperand(i_nocapture: `2`) == OldPtr)
4276	SI.setOperand(i_nocapture: `2`, Val_nocapture: NewPtr);
4277
4278	LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4279	deleteIfTriviallyDead(V: OldPtr);
4280
4281	// Fix the alignment of any loads or stores using this select.
4282	fixLoadStoreAlign(Root&: SI);
4283
4284	// Selects can't be promoted on their own, but often can be speculated. We
4285	// check the speculation outside of the rewriter so that we see the
4286	// fully-rewritten alloca.
4287	SelectUsers.insert(X: &SI);
4288	return true;
4289	}
4290	};
4291
4292	/// Visitor to rewrite aggregate loads and stores as scalar.
4293	///
4294	/// This pass aggressively rewrites all aggregate loads and stores on
4295	/// a particular pointer (or any pointer derived from it which we can identify)
4296	/// with scalar loads and stores.
4297	class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4298	// Befriend the base class so it can delegate to private visit methods.
4299	friend class InstVisitor<AggLoadStoreRewriter, bool>;
4300
4301	/// Queue of pointer uses to analyze and potentially rewrite.
4302	SmallVector<Use *, `8`> Queue;
4303
4304	/// Set to prevent us from cycling with phi nodes and loops.
4305	SmallPtrSet<User *, `8`> Visited;
4306
4307	/// The current pointer use being rewritten. This is used to dig up the used
4308	/// value (as opposed to the user).
4309	Use U = nullptr*;
4310
4311	/// Used to calculate offsets, and hence alignment, of subobjects.
4312	const DataLayout &DL;
4313
4314	IRBuilderTy &IRB;
4315
4316	public:
4317	AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4318	: DL(DL), IRB(IRB) {}
4319
4320	/// Rewrite loads and stores through a pointer and all pointers derived from
4321	/// it.
4322	bool rewrite(Instruction &I) {
4323	LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4324	enqueueUsers(I);
4325	bool Changed = false;
4326	while (!Queue.empty()) {
4327	U = Queue.pop_back_val();
4328	Changed \|= visit(I: cast<Instruction>(Val: U->getUser()));
4329	}
4330	return Changed;
4331	}
4332
4333	private:
4334	/// Enqueue all the users of the given instruction for further processing.
4335	/// This uses a set to de-duplicate users.
4336	void enqueueUsers(Instruction &I) {
4337	for (Use &U : I.uses())
4338	if (Visited.insert(Ptr: U.getUser()).second)
4339	Queue.push_back(Elt: &U);
4340	}
4341
4342	// Conservative default is to not rewrite anything.
4343	bool visitInstruction(Instruction &I) { return false; }
4344
4345	/// Generic recursive split emission class.
4346	template <typename Derived> class OpSplitter {
4347	protected:
4348	/// The builder used to form new instructions.
4349	IRBuilderTy &IRB;
4350
4351	/// The indices which to be used with insert- or extractvalue to select the
4352	/// appropriate value within the aggregate.
4353	SmallVector<unsigned, `4`> Indices;
4354
4355	/// The indices to a GEP instruction which will move Ptr to the correct slot
4356	/// within the aggregate.
4357	SmallVector<Value *, `4`> GEPIndices;
4358
4359	/// The base pointer of the original op, used as a base for GEPing the
4360	/// split operations.
4361	Value *Ptr;
4362
4363	/// The base pointee type being GEPed into.
4364	Type *BaseTy;
4365
4366	/// Known alignment of the base pointer.
4367	Align BaseAlign;
4368
4369	/// To calculate offset of each component so we can correctly deduce
4370	/// alignments.
4371	const DataLayout &DL;
4372
4373	/// Initialize the splitter with an insertion point, Ptr and start with a
4374	/// single zero GEP index.
4375	OpSplitter(Instruction InsertionPoint, Value Ptr, Type *BaseTy,
4376	Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4377	: IRB(IRB), GEPIndices (`1`, IRB.getInt32(C: `0`)), Ptr(Ptr), BaseTy(BaseTy),
4378	BaseAlign (BaseAlign), DL(DL) {
4379	IRB.SetInsertPoint(InsertionPoint);
4380	}
4381
4382	public:
4383	/// Generic recursive split emission routine.
4384	///
4385	/// This method recursively splits an aggregate op (load or store) into
4386	/// scalar or vector ops. It splits recursively until it hits a single value
4387	/// and emits that single value operation via the template argument.
4388	///
4389	/// The logic of this routine relies on GEPs and insertvalue and
4390	/// extractvalue all operating with the same fundamental index list, merely
4391	/// formatted differently (GEPs need actual values).
4392	///
4393	/// \param Ty The type being split recursively into smaller ops.
4394	/// \param Agg The aggregate value being built up or stored, depending on
4395	/// whether this is splitting a load or a store respectively.
4396	void emitSplitOps(Type Ty, Value &Agg, const Twine &Name) {
4397	if (Ty->isSingleValueType()) {
4398	unsigned Offset = DL.getIndexedOffsetInType(ElemTy: BaseTy, Indices: GEPIndices);
4399	return static_cast<Derived >(this*)->emitFunc(
4400	Ty, Agg, commonAlignment(A: BaseAlign, Offset), Name);
4401	}
4402
4403	if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
4404	unsigned OldSize = Indices.size();
4405	(void)OldSize;
4406	for (unsigned Idx = `0`, Size = ATy->getNumElements(); Idx != Size;
4407	++Idx) {
4408	assert(Indices.size() == OldSize && "Did not return to the old size");
4409	Indices.push_back(Elt: Idx);
4410	GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4411	emitSplitOps(Ty: ATy->getElementType(), Agg, Name: Name + "." + Twine (Idx));
4412	GEPIndices.pop_back();
4413	Indices.pop_back();
4414	}
4415	return;
4416	}
4417
4418	if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4419	unsigned OldSize = Indices.size();
4420	(void)OldSize;
4421	for (unsigned Idx = `0`, Size = STy->getNumElements(); Idx != Size;
4422	++Idx) {
4423	assert(Indices.size() == OldSize && "Did not return to the old size");
4424	Indices.push_back(Elt: Idx);
4425	GEPIndices.push_back(Elt: IRB.getInt32(C: Idx));
4426	emitSplitOps(Ty: STy->getElementType(N: Idx), Agg, Name: Name + "." + Twine (Idx));
4427	GEPIndices.pop_back();
4428	Indices.pop_back();
4429	}
4430	return;
4431	}
4432
4433	llvm_unreachable("Only arrays and structs are aggregate loadable types");
4434	}
4435	};
4436
4437	struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4438	AAMDNodes AATags;
4439	// A vector to hold the split components that we want to emit
4440	// separate fake uses for.
4441	SmallVector<Value *, `4`> Components;
4442	// A vector to hold all the fake uses of the struct that we are splitting.
4443	// Usually there should only be one, but we are handling the general case.
4444	SmallVector<Instruction *, `1`> FakeUses;
4445
4446	LoadOpSplitter(Instruction InsertionPoint, Value Ptr, Type *BaseTy,
4447	AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4448	IRBuilderTy &IRB)
4449	: OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4450	IRB),
4451	AATags (AATags) {}
4452
4453	/// Emit a leaf load of a single value. This is called at the leaves of the
4454	/// recursive emission to actually load values.
4455	void emitFunc(Type Ty, Value &Agg, Align Alignment, const Twine &Name) {
4456	assert(Ty->isSingleValueType());
4457	// Load the single value and insert it using the indices.
4458	Value *GEP =
4459	IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4460	LoadInst *Load =
4461	IRB.CreateAlignedLoad(Ty, Ptr: GEP, Align: Alignment, Name: Name + ".load");
4462
4463	APInt Offset(
4464	DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), `0`);
4465	if (AATags &&
4466	GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset))
4467	Load->setAAMetadata(
4468	AATags.adjustForAccess(Offset: Offset.getZExtValue(), AccessTy: Load->getType(), DL));
4469	// Record the load so we can generate a fake use for this aggregate
4470	// component.
4471	Components.push_back(Elt: Load);
4472
4473	Agg = IRB.CreateInsertValue(Agg, Val: Load, Idxs: Indices, Name: Name + ".insert");
4474	LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4475	}
4476
4477	// Stash the fake uses that use the value generated by this instruction.
4478	void recordFakeUses(LoadInst &LI) {
4479	for (Use &U : LI.uses())
4480	if (auto *II = dyn_cast<IntrinsicInst>(Val: U.getUser()))
4481	if (II->getIntrinsicID() == Intrinsic::fake_use)
4482	FakeUses.push_back(Elt: II);
4483	}
4484
4485	// Replace all fake uses of the aggregate with a series of fake uses, one
4486	// for each split component.
4487	void emitFakeUses() {
4488	for (Instruction *I : FakeUses) {
4489	IRB.SetInsertPoint(I);
4490	for (auto *V : Components)
4491	IRB.CreateIntrinsic(ID: Intrinsic::fake_use, Args: {V});
4492	I->eraseFromParent();
4493	}
4494	}
4495	};
4496
4497	bool visitLoadInst(LoadInst &LI) {
4498	assert(LI.getPointerOperand() == *U);
4499	if (!LI.isSimple() \|\| LI.getType()->isSingleValueType())
4500	return false;
4501
4502	// We have an aggregate being loaded, split it apart.
4503	LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4504	LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4505	getAdjustedAlignment(I: &LI, Offset: `0`), DL, IRB);
4506	Splitter.recordFakeUses(LI);
4507	Value *V = PoisonValue::get(T: LI.getType());
4508	Splitter.emitSplitOps(Ty: LI.getType(), Agg&: V, Name: LI.getName() + ".fca");
4509	Splitter.emitFakeUses();
4510	Visited.erase(Ptr: &LI);
4511	LI.replaceAllUsesWith(V);
4512	LI.eraseFromParent();
4513	return true;
4514	}
4515
4516	struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4517	StoreOpSplitter(Instruction InsertionPoint, Value Ptr, Type *BaseTy,
4518	AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4519	const DataLayout &DL, IRBuilderTy &IRB)
4520	: OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4521	DL, IRB),
4522	AATags (AATags), AggStore(AggStore) {}
4523	AAMDNodes AATags;
4524	StoreInst *AggStore;
4525	/// Emit a leaf store of a single value. This is called at the leaves of the
4526	/// recursive emission to actually produce stores.
4527	void emitFunc(Type Ty, Value &Agg, Align Alignment, const Twine &Name) {
4528	assert(Ty->isSingleValueType());
4529	// Extract the single value and store it using the indices.
4530	//
4531	// The gep and extractvalue values are factored out of the CreateStore
4532	// call to make the output independent of the argument evaluation order.
4533	Value *ExtractValue =
4534	IRB.CreateExtractValue(Agg, Idxs: Indices, Name: Name + ".extract");
4535	Value *InBoundsGEP =
4536	IRB.CreateInBoundsGEP(Ty: BaseTy, Ptr, IdxList: GEPIndices, Name: Name + ".gep");
4537	StoreInst *Store =
4538	IRB.CreateAlignedStore(Val: ExtractValue, Ptr: InBoundsGEP, Align: Alignment);
4539
4540	APInt Offset(
4541	DL.getIndexSizeInBits(AS: Ptr->getType()->getPointerAddressSpace()), `0`);
4542	GEPOperator::accumulateConstantOffset(SourceType: BaseTy, Index: GEPIndices, DL, Offset);
4543	if (AATags) {
4544	Store->setAAMetadata(AATags.adjustForAccess(
4545	Offset: Offset.getZExtValue(), AccessTy: ExtractValue->getType(), DL));
4546	}
4547
4548	// migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4549	// If we cannot (because there's an intervening non-const or unbounded
4550	// gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4551	// this instruction.
4552	Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets();
4553	if (auto *OldAI = dyn_cast<AllocaInst>(Val: Base)) {
4554	uint64_t SizeInBits =
4555	DL.getTypeSizeInBits(Ty: Store->getValueOperand()->getType());
4556	migrateDebugInfo(OldAlloca: OldAI, /IsSplit/ true, OldAllocaOffsetInBits: Offset.getZExtValue() * `8`,
4557	SliceSizeInBits: SizeInBits, OldInst: AggStore, Inst: Store,
4558	Dest: Store->getPointerOperand(), Value: Store->getValueOperand(),
4559	DL);
4560	} else {
4561	assert(at::getDVRAssignmentMarkers(Store).empty() &&
4562	"AT: unexpected debug.assign linked to store through "
4563	"unbounded GEP");
4564	}
4565	LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4566	}
4567	};
4568
4569	bool visitStoreInst(StoreInst &SI) {
4570	if (!SI.isSimple() \|\| SI.getPointerOperand() != *U)
4571	return false;
4572	Value *V = SI.getValueOperand();
4573	if (V->getType()->isSingleValueType())
4574	return false;
4575
4576	// We have an aggregate being stored, split it apart.
4577	LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4578	StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4579	getAdjustedAlignment(I: &SI, Offset: `0`), DL, IRB);
4580	Splitter.emitSplitOps(Ty: V->getType(), Agg&: V, Name: V->getName() + ".fca");
4581	Visited.erase(Ptr: &SI);
4582	// The stores replacing SI each have markers describing fragments of the
4583	// assignment so delete the assignment markers linked to SI.
4584	at::deleteAssignmentMarkers(Inst: &SI);
4585	SI.eraseFromParent();
4586	return true;
4587	}
4588
4589	bool visitBitCastInst(BitCastInst &BC) {
4590	enqueueUsers(I&: BC);
4591	return false;
4592	}
4593
4594	bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4595	enqueueUsers(I&: ASC);
4596	return false;
4597	}
4598
4599	// Unfold gep (select cond, ptr1, ptr2), idx
4600	// => select cond, gep(ptr1, idx), gep(ptr2, idx)
4601	// and gep ptr, (select cond, idx1, idx2)
4602	// => select cond, gep(ptr, idx1), gep(ptr, idx2)
4603	// We also allow for i1 zext indices, which are equivalent to selects.
4604	bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4605	// Check whether the GEP has exactly one select operand and all indices
4606	// will become constant after the transform.
4607	Instruction *Sel = dyn_cast<SelectInst>(Val: GEPI.getPointerOperand());
4608	for (Value *Op : GEPI.indices()) {
4609	if (auto *SI = dyn_cast<SelectInst>(Val: Op)) {
4610	if (Sel)
4611	return false;
4612
4613	Sel = SI;
4614	if (!isa<ConstantInt>(Val: SI->getTrueValue()) \|\|
4615	!isa<ConstantInt>(Val: SI->getFalseValue()))
4616	return false;
4617	continue;
4618	}
4619	if (auto *ZI = dyn_cast<ZExtInst>(Val: Op)) {
4620	if (Sel)
4621	return false;
4622	Sel = ZI;
4623	if (!ZI->getSrcTy()->isIntegerTy(BitWidth: `1`))
4624	return false;
4625	continue;
4626	}
4627
4628	if (!isa<ConstantInt>(Val: Op))
4629	return false;
4630	}
4631
4632	if (!Sel)
4633	return false;
4634
4635	LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4636	dbgs() << " original: " << *Sel << "\n";
4637	dbgs() << " " << GEPI << "\n";);
4638
4639	auto GetNewOps = [&](Value *SelOp) {
4640	SmallVector<Value *> NewOps;
4641	for (Value *Op : GEPI.operands())
4642	if (Op == Sel)
4643	NewOps.push_back(Elt: SelOp);
4644	else
4645	NewOps.push_back(Elt: Op);
4646	return NewOps;
4647	};
4648
4649	Value Cond, True, *False;
4650	Instruction MDFrom = nullptr*;
4651	if (auto *SI = dyn_cast<SelectInst>(Val: Sel)) {
4652	Cond = SI->getCondition();
4653	True = SI->getTrueValue();
4654	False = SI->getFalseValue();
4655	MDFrom = SI;
4656	} else {
4657	Cond = Sel->getOperand(i: `0`);
4658	True = ConstantInt::get(Ty: Sel->getType(), V: `1`);
4659	False = ConstantInt::get(Ty: Sel->getType(), V: `0`);
4660	}
4661	SmallVector<Value *> TrueOps = GetNewOps(True);
4662	SmallVector<Value *> FalseOps = GetNewOps(False);
4663
4664	IRB.SetInsertPoint(&GEPI);
4665	GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4666
4667	Type *Ty = GEPI.getSourceElementType();
4668	Value *NTrue = IRB.CreateGEP(Ty, Ptr: TrueOps [`0`], IdxList: ArrayRef(TrueOps).drop_front(),
4669	Name: True->getName() + ".sroa.gep", NW);
4670
4671	Value *NFalse =
4672	IRB.CreateGEP(Ty, Ptr: FalseOps [`0`], IdxList: ArrayRef(FalseOps).drop_front(),
4673	Name: False->getName() + ".sroa.gep", NW);
4674
4675	Value *NSel = MDFrom
4676	? IRB.CreateSelect(C: Cond, True: NTrue, False: NFalse,
4677	Name: Sel->getName() + ".sroa.sel", MDFrom)
4678	: IRB.CreateSelectWithUnknownProfile(
4679	C: Cond, True: NTrue, False: NFalse, DEBUG_TYPE,
4680	Name: Sel->getName() + ".sroa.sel");
4681	Visited.erase(Ptr: &GEPI);
4682	GEPI.replaceAllUsesWith(V: NSel);
4683	GEPI.eraseFromParent();
4684	Instruction *NSelI = cast<Instruction>(Val: NSel);
4685	Visited.insert(Ptr: NSelI);
4686	enqueueUsers(I&: *NSelI);
4687
4688	LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4689	dbgs() << " " << *NFalse << "\n";
4690	dbgs() << " " << *NSel << "\n";);
4691
4692	return true;
4693	}
4694
4695	// Unfold gep (phi ptr1, ptr2), idx
4696	// => phi ((gep ptr1, idx), (gep ptr2, idx))
4697	// and gep ptr, (phi idx1, idx2)
4698	// => phi ((gep ptr, idx1), (gep ptr, idx2))
4699	bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4700	// To prevent infinitely expanding recursive phis, bail if the GEP pointer
4701	// operand (looking through the phi if it is the phi we want to unfold) is
4702	// an instruction besides a static alloca.
4703	PHINode *Phi = dyn_cast<PHINode>(Val: GEPI.getPointerOperand());
4704	auto IsInvalidPointerOperand = [](Value *V) {
4705	if (!isa<Instruction>(Val: V))
4706	return false;
4707	if (auto *AI = dyn_cast<AllocaInst>(Val: V))
4708	return !AI->isStaticAlloca();
4709	return true;
4710	};
4711	if (Phi) {
4712	if (any_of(Range: Phi->operands(), P: IsInvalidPointerOperand))
4713	return false;
4714	} else {
4715	if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4716	return false;
4717	}
4718	// Check whether the GEP has exactly one phi operand (including the pointer
4719	// operand) and all indices will become constant after the transform.
4720	for (Value *Op : GEPI.indices()) {
4721	if (auto *SI = dyn_cast<PHINode>(Val: Op)) {
4722	if (Phi)
4723	return false;
4724
4725	Phi = SI;
4726	if (!all_of(Range: Phi->incoming_values(),
4727	P: [](Value V) { return* isa<ConstantInt>(Val: V); }))
4728	return false;
4729	continue;
4730	}
4731
4732	if (!isa<ConstantInt>(Val: Op))
4733	return false;
4734	}
4735
4736	if (!Phi)
4737	return false;
4738
4739	LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4740	dbgs() << " original: " << *Phi << "\n";
4741	dbgs() << " " << GEPI << "\n";);
4742
4743	auto GetNewOps = [&](Value *PhiOp) {
4744	SmallVector<Value *> NewOps;
4745	for (Value *Op : GEPI.operands())
4746	if (Op == Phi)
4747	NewOps.push_back(Elt: PhiOp);
4748	else
4749	NewOps.push_back(Elt: Op);
4750	return NewOps;
4751	};
4752
4753	IRB.SetInsertPoint(Phi);
4754	PHINode *NewPhi = IRB.CreatePHI(Ty: GEPI.getType(), NumReservedValues: Phi->getNumIncomingValues(),
4755	Name: Phi->getName() + ".sroa.phi");
4756
4757	Type *SourceTy = GEPI.getSourceElementType();
4758	// We only handle arguments, constants, and static allocas here, so we can
4759	// insert GEPs at the end of the entry block.
4760	IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4761	for (unsigned I = `0`, E = Phi->getNumIncomingValues(); I != E; ++I) {
4762	Value *Op = Phi->getIncomingValue(i: I);
4763	BasicBlock *BB = Phi->getIncomingBlock(i: I);
4764	Value *NewGEP;
4765	if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= `0`) {
4766	NewGEP = NewPhi->getIncomingValue(i: NI);
4767	} else {
4768	SmallVector<Value *> NewOps = GetNewOps(Op);
4769	NewGEP =
4770	IRB.CreateGEP(Ty: SourceTy, Ptr: NewOps [`0`], IdxList: ArrayRef(NewOps).drop_front(),
4771	Name: Phi->getName() + ".sroa.gep", NW: GEPI.getNoWrapFlags());
4772	}
4773	NewPhi->addIncoming(V: NewGEP, BB);
4774	}
4775
4776	Visited.erase(Ptr: &GEPI);
4777	GEPI.replaceAllUsesWith(V: NewPhi);
4778	GEPI.eraseFromParent();
4779	Visited.insert(Ptr: NewPhi);
4780	enqueueUsers(I&: *NewPhi);
4781
4782	LLVM_DEBUG(dbgs() << " to: ";
4783	for (Value *In
4784	: NewPhi->incoming_values()) dbgs()
4785	<< "\n " << *In;
4786	dbgs() << "\n " << *NewPhi << `'\n'`);
4787
4788	return true;
4789	}
4790
4791	bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4792	if (unfoldGEPSelect(GEPI))
4793	return true;
4794
4795	if (unfoldGEPPhi(GEPI))
4796	return true;
4797
4798	enqueueUsers(I&: GEPI);
4799	return false;
4800	}
4801
4802	bool visitPHINode(PHINode &PN) {
4803	enqueueUsers(I&: PN);
4804	return false;
4805	}
4806
4807	bool visitSelectInst(SelectInst &SI) {
4808	enqueueUsers(I&: SI);
4809	return false;
4810	}
4811	};
4812
4813	} // end anonymous namespace
4814
4815	/// Strip aggregate type wrapping.
4816	///
4817	/// This removes no-op aggregate types wrapping an underlying type. It will
4818	/// strip as many layers of types as it can without changing either the type
4819	/// size or the allocated size.
4820	static Type stripAggregateTypeWrapping(const* DataLayout &DL, Type *Ty) {
4821	if (Ty->isSingleValueType())
4822	return Ty;
4823
4824	uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4825	uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4826
4827	Type *InnerTy;
4828	if (ArrayType *ArrTy = dyn_cast<ArrayType>(Val: Ty)) {
4829	InnerTy = ArrTy->getElementType();
4830	} else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
4831	const StructLayout *SL = DL.getStructLayout(Ty: STy);
4832	unsigned Index = SL->getElementContainingOffset(FixedOffset: `0`);
4833	InnerTy = STy->getElementType(N: Index);
4834	} else {
4835	return Ty;
4836	}
4837
4838	if (AllocSize > DL.getTypeAllocSize(Ty: InnerTy).getFixedValue() \|\|
4839	TypeSize > DL.getTypeSizeInBits(Ty: InnerTy).getFixedValue())
4840	return Ty;
4841
4842	return stripAggregateTypeWrapping(DL, Ty: InnerTy);
4843	}
4844
4845	/// Try to find a partition of the aggregate type passed in for a given
4846	/// offset and size.
4847	///
4848	/// This recurses through the aggregate type and tries to compute a subtype
4849	/// based on the offset and size. When the offset and size span a sub-section
4850	/// of an array, it will even compute a new array type for that sub-section,
4851	/// and the same for structs.
4852	///
4853	/// Note that this routine is very strict and tries to find a partition of the
4854	/// type which produces the exact* right offset and size. It is not forgiving*
4855	/// when the size or offset cause either end of type-based partition to be off.
4856	/// Also, this is a best-effort routine. It is reasonable to give up and not
4857	/// return a type if necessary.
4858	static Type getTypePartition(const* DataLayout &DL, Type *Ty, uint64_t Offset,
4859	uint64_t Size) {
4860	if (Offset == `0` && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4861	return stripAggregateTypeWrapping(DL, Ty);
4862	if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() \|\|
4863	(DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4864	return nullptr;
4865
4866	if (isa<ArrayType>(Val: Ty) \|\| isa<VectorType>(Val: Ty)) {
4867	Type *ElementTy;
4868	uint64_t TyNumElements;
4869	if (auto *AT = dyn_cast<ArrayType>(Val: Ty)) {
4870	ElementTy = AT->getElementType();
4871	TyNumElements = AT->getNumElements();
4872	} else {
4873	// FIXME: This isn't right for vectors with non-byte-sized or
4874	// non-power-of-two sized elements.
4875	auto *VT = cast<FixedVectorType>(Val: Ty);
4876	ElementTy = VT->getElementType();
4877	TyNumElements = VT->getNumElements();
4878	}
4879	uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4880	uint64_t NumSkippedElements = Offset / ElementSize;
4881	if (NumSkippedElements >= TyNumElements)
4882	return nullptr;
4883	Offset -= NumSkippedElements * ElementSize;
4884
4885	// First check if we need to recurse.
4886	if (Offset > `0` \|\| Size < ElementSize) {
4887	// Bail if the partition ends in a different array element.
4888	if ((Offset + Size) > ElementSize)
4889	return nullptr;
4890	// Recurse through the element type trying to peel off offset bytes.
4891	return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4892	}
4893	assert(Offset == `0`);
4894
4895	if (Size == ElementSize)
4896	return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4897	assert(Size > ElementSize);
4898	uint64_t NumElements = Size / ElementSize;
4899	if (NumElements * ElementSize != Size)
4900	return nullptr;
4901	return ArrayType::get(ElementType: ElementTy, NumElements);
4902	}
4903
4904	StructType *STy = dyn_cast<StructType>(Val: Ty);
4905	if (!STy)
4906	return nullptr;
4907
4908	const StructLayout *SL = DL.getStructLayout(Ty: STy);
4909
4910	if (SL->getSizeInBits().isScalable())
4911	return nullptr;
4912
4913	if (Offset >= SL->getSizeInBytes())
4914	return nullptr;
4915	uint64_t EndOffset = Offset + Size;
4916	if (EndOffset > SL->getSizeInBytes())
4917	return nullptr;
4918
4919	unsigned Index = SL->getElementContainingOffset(FixedOffset: Offset);
4920	Offset -= SL->getElementOffset(Idx: Index);
4921
4922	Type *ElementTy = STy->getElementType(N: Index);
4923	uint64_t ElementSize = DL.getTypeAllocSize(Ty: ElementTy).getFixedValue();
4924	if (Offset >= ElementSize)
4925	return nullptr; // The offset points into alignment padding.
4926
4927	// See if any partition must be contained by the element.
4928	if (Offset > `0` \|\| Size < ElementSize) {
4929	if ((Offset + Size) > ElementSize)
4930	return nullptr;
4931	return getTypePartition(DL, Ty: ElementTy, Offset, Size);
4932	}
4933	assert(Offset == `0`);
4934
4935	if (Size == ElementSize)
4936	return stripAggregateTypeWrapping(DL, Ty: ElementTy);
4937
4938	StructType::element_iterator EI = STy->element_begin() + Index,
4939	EE = STy->element_end();
4940	if (EndOffset < SL->getSizeInBytes()) {
4941	unsigned EndIndex = SL->getElementContainingOffset(FixedOffset: EndOffset);
4942	if (Index == EndIndex)
4943	return nullptr; // Within a single element and its padding.
4944
4945	// Don't try to form "natural" types if the elements don't line up with the
4946	// expected size.
4947	// FIXME: We could potentially recurse down through the last element in the
4948	// sub-struct to find a natural end point.
4949	if (SL->getElementOffset(Idx: EndIndex) != EndOffset)
4950	return nullptr;
4951
4952	assert(Index < EndIndex);
4953	EE = STy->element_begin() + EndIndex;
4954	}
4955
4956	// Try to build up a sub-structure.
4957	StructType *SubTy =
4958	StructType::get(Context&: STy->getContext(), Elements: ArrayRef(EI, EE), isPacked: STy->isPacked());
4959	const StructLayout *SubSL = DL.getStructLayout(Ty: SubTy);
4960	if (Size != SubSL->getSizeInBytes())
4961	return nullptr; // The sub-struct doesn't have quite the size needed.
4962
4963	return SubTy;
4964	}
4965
4966	/// Try to find a slice in the map that partially overlaps with S, i.e. some but
4967	/// not all of its range is contained within the range of S, and which forms a
4968	/// load/store pair with S (i.e. the two slices are used to perform a
4969	/// memmove-like copy). SliceMap is expected to contain the slices for a single
4970	/// Partition.
4971	static Slice *findOverlappingCopySlice(Slice &S,
4972	InstructionSliceMap &SliceMap) {
4973	// Single byte slices can't partially overlap anything
4974	if (S.endOffset() - S.beginOffset() == `1`)
4975	return nullptr;
4976	// The source/destination of this slice needs to be a memory instruction
4977	// whose slice overlaps this one.
4978	Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
4979	Instruction J = nullptr*;
4980	if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
4981	if (!LI->hasOneUser())
4982	return nullptr;
4983	J = cast<Instruction>(Val: *LI->user_begin());
4984	} else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4985	J = dyn_cast<Instruction>(Val: SI->getValueOperand());
4986	if (!J \|\| !J->hasOneUser())
4987	return nullptr;
4988	} else {
4989	return nullptr;
4990	}
4991	// Check if there's a slice that corresponds to J that overlaps this slice
4992	for (Slice *JS : SliceMap [J]) {
4993	if (S.beginOffset() > JS->beginOffset() &&
4994	S.beginOffset() < JS->endOffset())
4995	return JS;
4996	if (JS->beginOffset() > S.beginOffset() &&
4997	JS->beginOffset() < S.endOffset())
4998	return JS;
4999	}
5000	return nullptr;
5001	}
5002
5003	/// Pre-split loads and stores to simplify rewriting.
5004	///
5005	/// We want to break up the splittable load+store pairs as much as
5006	/// possible. This is important to do as a preprocessing step, as once we
5007	/// start rewriting the accesses to partitions of the alloca we lose the
5008	/// necessary information to correctly split apart paired loads and stores
5009	/// which both point into this alloca. The case to consider is something like
5010	/// the following:
5011	///
5012	/// %a = alloca [12 x i8]
5013	/// %gep1 = getelementptr i8, ptr %a, i32 0
5014	/// %gep2 = getelementptr i8, ptr %a, i32 4
5015	/// %gep3 = getelementptr i8, ptr %a, i32 8
5016	/// store float 0.0, ptr %gep1
5017	/// store float 1.0, ptr %gep2
5018	/// %v = load i64, ptr %gep1
5019	/// store i64 %v, ptr %gep2
5020	/// %f1 = load float, ptr %gep2
5021	/// %f2 = load float, ptr %gep3
5022	///
5023	/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
5024	/// promote everything so we recover the 2 SSA values that should have been
5025	/// there all along.
5026	///
5027	/// \returns true if any changes are made.
5028	bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
5029	LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
5030
5031	// Track the loads and stores which are candidates for pre-splitting here, in
5032	// the order they first appear during the partition scan. These give stable
5033	// iteration order and a basis for tracking which loads and stores we
5034	// actually split.
5035	SmallVector<LoadInst *, `4`> Loads;
5036	SmallVector<StoreInst *, `4`> Stores;
5037
5038	// We need to accumulate the splits required of each load or store where we
5039	// can find them via a direct lookup. This is important to cross-check loads
5040	// and stores against each other. We also track the slice so that we can kill
5041	// all the slices that end up split.
5042	struct SplitOffsets {
5043	Slice *S;
5044	std::vector<uint64_t> Splits;
5045	};
5046	SmallDenseMap<Instruction *, SplitOffsets, `8`> SplitOffsetsMap;
5047
5048	// Track loads out of this alloca which cannot, for any reason, be pre-split.
5049	// This is important as we also cannot pre-split stores of those loads!
5050	// FIXME: This is all pretty gross. It means that we can be more aggressive
5051	// in pre-splitting when the load feeding the store happens to come from
5052	// a separate alloca. Put another way, the effectiveness of SROA would be
5053	// decreased by a frontend which just concatenated all of its local allocas
5054	// into one big flat alloca. But defeating such patterns is exactly the job
5055	// SROA is tasked with! Sadly, to not have this discrepancy we would have
5056	// change store pre-splitting to actually force pre-splitting of the load
5057	// that feeds it and all stores. That makes pre-splitting much harder, but
5058	// maybe it would make it more principled?
5059	SmallPtrSet<LoadInst *, `8`> UnsplittableLoads;
5060
5061	LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
5062	for (auto &P : AS.partitions()) {
5063	InstructionSliceMap SliceMap;
5064	for (Slice &S : P) {
5065	SliceMap [cast<Instruction>(Val: S.getUse()->getUser())].insert(Ptr: &S);
5066	}
5067	for (Slice &S : P) {
5068	Instruction *I = cast<Instruction>(Val: S.getUse()->getUser());
5069	bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
5070	Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
5071	if (!S.isSplittable() \|\| (!ExtendsPastPartitionEnd && !CopyOverlap)) {
5072	// If this is a load we have to track that it can't participate in any
5073	// pre-splitting. If this is a store of a load we have to track that
5074	// that load also can't participate in any pre-splitting.
5075	if (auto *LI = dyn_cast<LoadInst>(Val: I))
5076	UnsplittableLoads.insert(Ptr: LI);
5077	else if (auto *SI = dyn_cast<StoreInst>(Val: I))
5078	if (auto *LI = dyn_cast<LoadInst>(Val: SI->getValueOperand()))
5079	UnsplittableLoads.insert(Ptr: LI);
5080	continue;
5081	}
5082	assert(P.endOffset() > S.beginOffset() &&
5083	"Empty or backwards partition!");
5084
5085	// Determine if this is a pre-splittable slice.
5086	if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5087	assert(!LI->isVolatile() && "Cannot split volatile loads!");
5088
5089	// The load must be used exclusively to store into other pointers for
5090	// us to be able to arbitrarily pre-split it. The stores must also be
5091	// simple to avoid changing semantics.
5092	auto IsLoadSimplyStored = [](LoadInst *LI) {
5093	for (User *LU : LI->users()) {
5094	auto *SI = dyn_cast<StoreInst>(Val: LU);
5095	if (!SI \|\| !SI->isSimple())
5096	return false;
5097	}
5098	return true;
5099	};
5100	if (!IsLoadSimplyStored (LI)) {
5101	UnsplittableLoads.insert(Ptr: LI);
5102	continue;
5103	}
5104
5105	Loads.push_back(Elt: LI);
5106	} else if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5107	if (S.getUse() != &SI->getOperandUse(i: SI->getPointerOperandIndex()))
5108	// Skip stores of* pointers. FIXME: This shouldn't even be possible!*
5109	continue;
5110	auto *StoredLoad = dyn_cast<LoadInst>(Val: SI->getValueOperand());
5111	if (!StoredLoad \|\| !StoredLoad->isSimple())
5112	continue;
5113	assert(!SI->isVolatile() && "Cannot split volatile stores!");
5114
5115	Stores.push_back(Elt: SI);
5116	} else {
5117	// Other uses cannot be pre-split.
5118	continue;
5119	}
5120
5121	// Record the initial split.
5122	LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
5123	auto &Offsets = SplitOffsetsMap [I];
5124	assert(Offsets.Splits.empty() &&
5125	"Should not have splits the first time we see an instruction!");
5126	Offsets.S = &S;
5127	if (CopyOverlap) {
5128	// S is being moved to CopyOverlap, which overlaps with S, so we split S
5129	// into three parts:
5130	// Initially outside the overlap and copied into the overlap.*
5131	// Either initially inside the overlap and remains inside it after*
5132	// the copy, or initially outside and remains outside, depending on
5133	// if the overlap or non-overlap area is larger, and which will be
5134	// empty if they are equal.
5135	// Initially inside the overlap and copied outside the overlap.*
5136	// This should result in the first and last parts being promoted to
5137	// scalars. If the middle part is an overlapping copy then we repeat
5138	// this process until it isn't.
5139	uint64_t OverlapStart =
5140	std::max(a: S.beginOffset(), b: CopyOverlap->beginOffset());
5141	uint64_t OverlapEnd = std::min(a: S.endOffset(), b: CopyOverlap->endOffset());
5142	uint64_t OverlapSize = OverlapEnd - OverlapStart;
5143	uint64_t SliceSize = S.endOffset() - S.beginOffset();
5144	uint64_t NonOverlapSize = SliceSize - OverlapSize;
5145	if (OverlapSize < NonOverlapSize) {
5146	// When the overlap area is smaller the middle part is initially and
5147	// remains outside the overlap, so splitting once is enough. The size
5148	// of the first and last part both equal the overlap size, so the
5149	// split points are that amount from the start and end of the slice.
5150	Offsets.Splits.push_back(x: OverlapSize);
5151	Offsets.Splits.push_back(x: SliceSize - OverlapSize);
5152	} else if (OverlapSize > NonOverlapSize) {
5153	// When the overlap area is larger the middle part is initially and
5154	// remains inside the overlap, so we repeatedly split it.
5155	for (uint64_t Split = NonOverlapSize; Split <= SliceSize / `2`;
5156	Split += NonOverlapSize) {
5157	Offsets.Splits.push_back(x: Split);
5158	if (Split != SliceSize - Split) {
5159	Offsets.Splits.push_back(x: SliceSize - Split);
5160	}
5161	}
5162	// Sort the splits as they need to be in ascending order.
5163	llvm::sort(C&: Offsets.Splits);
5164	} else {
5165	// Here the overlap and non-overlap size are the same, so the middle
5166	// part is empty and the slice is split exactly in the center.
5167	Offsets.Splits.push_back(x: OverlapSize);
5168	}
5169	} else {
5170	Offsets.Splits.push_back(x: P.endOffset() - S.beginOffset());
5171	}
5172	}
5173	}
5174
5175	// We may have split loads where some of their stores are split stores. For
5176	// such loads and stores, we can only pre-split them if their splits exactly
5177	// match relative to their starting offset. We have to verify this prior to
5178	// any rewriting.
5179	llvm::erase_if(C&: Stores, P: [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
5180	// Lookup the load we are storing in our map of split
5181	// offsets.
5182	auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5183	// If it was completely unsplittable, then we're done,
5184	// and this store can't be pre-split.
5185	if (UnsplittableLoads.count(Ptr: LI))
5186	return true;
5187
5188	auto LoadOffsetsI = SplitOffsetsMap.find(Val: LI);
5189	if (LoadOffsetsI == SplitOffsetsMap.end())
5190	return false; // Unrelated loads are definitely safe.
5191	auto &LoadOffsets = LoadOffsetsI ->second;
5192
5193	// Now lookup the store's offsets.
5194	auto &StoreOffsets = SplitOffsetsMap [SI];
5195
5196	// If the relative offsets of each split in the load and
5197	// store match exactly, then we can split them and we
5198	// don't need to remove them here.
5199	if (LoadOffsets.Splits == StoreOffsets.Splits)
5200	return false;
5201
5202	LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
5203	<< " " << *LI << "\n"
5204	<< " " << *SI << "\n");
5205
5206	// We've found a store and load that we need to split
5207	// with mismatched relative splits. Just give up on them
5208	// and remove both instructions from our list of
5209	// candidates.
5210	UnsplittableLoads.insert(Ptr: LI);
5211	return true;
5212	});
5213	// Now we have to go back* through all the stores, because a later store may*
5214	// have caused an earlier store's load to become unsplittable and if it is
5215	// unsplittable for the later store, then we can't rely on it being split in
5216	// the earlier store either.
5217	llvm::erase_if(C&: Stores, P: [&UnsplittableLoads](StoreInst *SI) {
5218	auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5219	return UnsplittableLoads.count(Ptr: LI);
5220	});
5221	// Once we've established all the loads that can't be split for some reason,
5222	// filter any that made it into our list out.
5223	llvm::erase_if(C&: Loads, P: [&UnsplittableLoads](LoadInst *LI) {
5224	return UnsplittableLoads.count(Ptr: LI);
5225	});
5226
5227	// If no loads or stores are left, there is no pre-splitting to be done for
5228	// this alloca.
5229	if (Loads.empty() && Stores.empty())
5230	return false;
5231
5232	// From here on, we can't fail and will be building new accesses, so rig up
5233	// an IR builder.
5234	IRBuilderTy IRB(&AI);
5235
5236	// Collect the new slices which we will merge into the alloca slices.
5237	SmallVector<Slice, `4`> NewSlices;
5238
5239	// Track any allocas we end up splitting loads and stores for so we iterate
5240	// on them.
5241	SmallPtrSet<AllocaInst *, `4`> ResplitPromotableAllocas;
5242
5243	// At this point, we have collected all of the loads and stores we can
5244	// pre-split, and the specific splits needed for them. We actually do the
5245	// splitting in a specific order in order to handle when one of the loads in
5246	// the value operand to one of the stores.
5247	//
5248	// First, we rewrite all of the split loads, and just accumulate each split
5249	// load in a parallel structure. We also build the slices for them and append
5250	// them to the alloca slices.
5251	SmallDenseMap<LoadInst , std::vector<LoadInst >, `1`> SplitLoadsMap;
5252	std::vector<LoadInst *> SplitLoads;
5253	const DataLayout &DL = AI.getDataLayout();
5254	for (LoadInst *LI : Loads) {
5255	SplitLoads.clear();
5256
5257	auto &Offsets = SplitOffsetsMap [LI];
5258	unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5259	assert(LI->getType()->getIntegerBitWidth() % `8` == `0` &&
5260	"Load must have type size equal to store size");
5261	assert(LI->getType()->getIntegerBitWidth() / `8` >= SliceSize &&
5262	"Load must be >= slice size");
5263
5264	uint64_t BaseOffset = Offsets.S->beginOffset();
5265	assert(BaseOffset + SliceSize > BaseOffset &&
5266	"Cannot represent alloca access size using 64-bit integers!");
5267
5268	Instruction *BasePtr = cast<Instruction>(Val: LI->getPointerOperand());
5269	IRB.SetInsertPoint(LI);
5270
5271	LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5272
5273	uint64_t PartOffset = `0`, PartSize = Offsets.Splits.front();
5274	int Idx = `0`, Size = Offsets.Splits.size();
5275	for (;;) {
5276	auto PartTy = Type::getIntNTy(C&: LI->getContext(), N: PartSize `8`);
5277	auto AS = LI->getPointerAddressSpace();
5278	auto *PartPtrTy = LI->getPointerOperandType();
5279	LoadInst *PLoad = IRB.CreateAlignedLoad(
5280	Ty: PartTy,
5281	Ptr: getAdjustedPtr(IRB, DL, Ptr: BasePtr,
5282	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5283	PointerTy: PartPtrTy, NamePrefix: BasePtr->getName() + "."),
5284	Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5285	/IsVolatile/ isVolatile: false, Name: LI->getName());
5286	PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5287	LLVMContext::MD_access_group});
5288
5289	// Append this load onto the list of split loads so we can find it later
5290	// to rewrite the stores.
5291	SplitLoads.push_back(x: PLoad);
5292
5293	// Now build a new slice for the alloca.
5294	NewSlices.push_back(
5295	Elt: Slice (BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5296	&PLoad->getOperandUse(i: PLoad->getPointerOperandIndex()),
5297	/IsSplittable/ false));
5298	LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5299	<< ", " << NewSlices.back().endOffset()
5300	<< "): " << *PLoad << "\n");
5301
5302	// See if we've handled all the splits.
5303	if (Idx >= Size)
5304	break;
5305
5306	// Setup the next partition.
5307	PartOffset = Offsets.Splits [Idx];
5308	++Idx;
5309	PartSize = (Idx < Size ? Offsets.Splits [Idx] : SliceSize) - PartOffset;
5310	}
5311
5312	// Now that we have the split loads, do the slow walk over all uses of the
5313	// load and rewrite them as split stores, or save the split loads to use
5314	// below if the store is going to be split there anyways.
5315	bool DeferredStores = false;
5316	for (User *LU : LI->users()) {
5317	StoreInst *SI = cast<StoreInst>(Val: LU);
5318	if (!Stores.empty() && SplitOffsetsMap.count(Val: SI)) {
5319	DeferredStores = true;
5320	LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5321	<< "\n");
5322	continue;
5323	}
5324
5325	Value *StoreBasePtr = SI->getPointerOperand();
5326	IRB.SetInsertPoint(SI);
5327	AAMDNodes AATags = SI->getAAMetadata();
5328
5329	LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5330
5331	for (int Idx = `0`, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5332	LoadInst *PLoad = SplitLoads [Idx];
5333	uint64_t PartOffset = Idx == `0` ? `0` : Offsets.Splits [Idx - `1`];
5334	auto *PartPtrTy = SI->getPointerOperandType();
5335
5336	auto AS = SI->getPointerAddressSpace();
5337	StoreInst *PStore = IRB.CreateAlignedStore(
5338	Val: PLoad,
5339	Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5340	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5341	PointerTy: PartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5342	Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5343	/IsVolatile/ isVolatile: false);
5344	PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5345	LLVMContext::MD_access_group,
5346	LLVMContext::MD_DIAssignID});
5347
5348	if (AATags)
5349	PStore->setAAMetadata(
5350	AATags.adjustForAccess(Offset: PartOffset, AccessTy: PLoad->getType(), DL));
5351	LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5352	}
5353
5354	// We want to immediately iterate on any allocas impacted by splitting
5355	// this store, and we have to track any promotable alloca (indicated by
5356	// a direct store) as needing to be resplit because it is no longer
5357	// promotable.
5358	if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: StoreBasePtr)) {
5359	ResplitPromotableAllocas.insert(Ptr: OtherAI);
5360	Worklist.insert(X: OtherAI);
5361	} else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5362	Val: StoreBasePtr->stripInBoundsOffsets())) {
5363	Worklist.insert(X: OtherAI);
5364	}
5365
5366	// Mark the original store as dead.
5367	DeadInsts.push_back(Elt: SI);
5368	}
5369
5370	// Save the split loads if there are deferred stores among the users.
5371	if (DeferredStores)
5372	SplitLoadsMap.insert(KV: std::make_pair(x&: LI, y: std::move(SplitLoads)));
5373
5374	// Mark the original load as dead and kill the original slice.
5375	DeadInsts.push_back(Elt: LI);
5376	Offsets.S->kill();
5377	}
5378
5379	// Second, we rewrite all of the split stores. At this point, we know that
5380	// all loads from this alloca have been split already. For stores of such
5381	// loads, we can simply look up the pre-existing split loads. For stores of
5382	// other loads, we split those loads first and then write split stores of
5383	// them.
5384	for (StoreInst *SI : Stores) {
5385	auto *LI = cast<LoadInst>(Val: SI->getValueOperand());
5386	IntegerType *Ty = cast<IntegerType>(Val: LI->getType());
5387	assert(Ty->getBitWidth() % `8` == `0`);
5388	uint64_t StoreSize = Ty->getBitWidth() / `8`;
5389	assert(StoreSize > `0` && "Cannot have a zero-sized integer store!");
5390
5391	auto &Offsets = SplitOffsetsMap [SI];
5392	assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5393	"Slice size should always match load size exactly!");
5394	uint64_t BaseOffset = Offsets.S->beginOffset();
5395	assert(BaseOffset + StoreSize > BaseOffset &&
5396	"Cannot represent alloca access size using 64-bit integers!");
5397
5398	Value *LoadBasePtr = LI->getPointerOperand();
5399	Instruction *StoreBasePtr = cast<Instruction>(Val: SI->getPointerOperand());
5400
5401	LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5402
5403	// Check whether we have an already split load.
5404	auto SplitLoadsMapI = SplitLoadsMap.find(Val: LI);
5405	std::vector<LoadInst > SplitLoads = nullptr;
5406	if (SplitLoadsMapI != SplitLoadsMap.end()) {
5407	SplitLoads = &SplitLoadsMapI ->second;
5408	assert(SplitLoads->size() == Offsets.Splits.size() + `1` &&
5409	"Too few split loads for the number of splits in the store!");
5410	} else {
5411	LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5412	}
5413
5414	uint64_t PartOffset = `0`, PartSize = Offsets.Splits.front();
5415	int Idx = `0`, Size = Offsets.Splits.size();
5416	for (;;) {
5417	auto PartTy = Type::getIntNTy(C&: Ty->getContext(), N: PartSize `8`);
5418	auto *LoadPartPtrTy = LI->getPointerOperandType();
5419	auto *StorePartPtrTy = SI->getPointerOperandType();
5420
5421	// Either lookup a split load or create one.
5422	LoadInst *PLoad;
5423	if (SplitLoads) {
5424	PLoad = (*SplitLoads)[Idx];
5425	} else {
5426	IRB.SetInsertPoint(LI);
5427	auto AS = LI->getPointerAddressSpace();
5428	PLoad = IRB.CreateAlignedLoad(
5429	Ty: PartTy,
5430	Ptr: getAdjustedPtr(IRB, DL, Ptr: LoadBasePtr,
5431	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5432	PointerTy: LoadPartPtrTy, NamePrefix: LoadBasePtr->getName() + "."),
5433	Align: getAdjustedAlignment(I: LI, Offset: PartOffset),
5434	/IsVolatile/ isVolatile: false, Name: LI->getName());
5435	PLoad->copyMetadata(SrcInst: *LI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5436	LLVMContext::MD_access_group});
5437	}
5438
5439	// And store this partition.
5440	IRB.SetInsertPoint(SI);
5441	auto AS = SI->getPointerAddressSpace();
5442	StoreInst *PStore = IRB.CreateAlignedStore(
5443	Val: PLoad,
5444	Ptr: getAdjustedPtr(IRB, DL, Ptr: StoreBasePtr,
5445	Offset: APInt (DL.getIndexSizeInBits(AS), PartOffset),
5446	PointerTy: StorePartPtrTy, NamePrefix: StoreBasePtr->getName() + "."),
5447	Align: getAdjustedAlignment(I: SI, Offset: PartOffset),
5448	/IsVolatile/ isVolatile: false);
5449	PStore->copyMetadata(SrcInst: *SI, WL: {LLVMContext::MD_mem_parallel_loop_access,
5450	LLVMContext::MD_access_group});
5451
5452	// Now build a new slice for the alloca.
5453	NewSlices.push_back(
5454	Elt: Slice (BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5455	&PStore->getOperandUse(i: PStore->getPointerOperandIndex()),
5456	/IsSplittable/ false));
5457	LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5458	<< ", " << NewSlices.back().endOffset()
5459	<< "): " << *PStore << "\n");
5460	if (!SplitLoads) {
5461	LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5462	}
5463
5464	// See if we've finished all the splits.
5465	if (Idx >= Size)
5466	break;
5467
5468	// Setup the next partition.
5469	PartOffset = Offsets.Splits [Idx];
5470	++Idx;
5471	PartSize = (Idx < Size ? Offsets.Splits [Idx] : StoreSize) - PartOffset;
5472	}
5473
5474	// We want to immediately iterate on any allocas impacted by splitting
5475	// this load, which is only relevant if it isn't a load of this alloca and
5476	// thus we didn't already split the loads above. We also have to keep track
5477	// of any promotable allocas we split loads on as they can no longer be
5478	// promoted.
5479	if (!SplitLoads) {
5480	if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(Val: LoadBasePtr)) {
5481	assert(OtherAI != &AI && "We can't re-split our own alloca!");
5482	ResplitPromotableAllocas.insert(Ptr: OtherAI);
5483	Worklist.insert(X: OtherAI);
5484	} else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5485	Val: LoadBasePtr->stripInBoundsOffsets())) {
5486	assert(OtherAI != &AI && "We can't re-split our own alloca!");
5487	Worklist.insert(X: OtherAI);
5488	}
5489	}
5490
5491	// Mark the original store as dead now that we've split it up and kill its
5492	// slice. Note that we leave the original load in place unless this store
5493	// was its only use. It may in turn be split up if it is an alloca load
5494	// for some other alloca, but it may be a normal load. This may introduce
5495	// redundant loads, but where those can be merged the rest of the optimizer
5496	// should handle the merging, and this uncovers SSA splits which is more
5497	// important. In practice, the original loads will almost always be fully
5498	// split and removed eventually, and the splits will be merged by any
5499	// trivial CSE, including instcombine.
5500	if (LI->hasOneUse()) {
5501	assert(*LI->user_begin() == SI && "Single use isn't this store!");
5502	DeadInsts.push_back(Elt: LI);
5503	}
5504	DeadInsts.push_back(Elt: SI);
5505	Offsets.S->kill();
5506	}
5507
5508	// Remove the killed slices that have ben pre-split.
5509	llvm::erase_if(C&: AS, P: [](const Slice &S) { return S.isDead(); });
5510
5511	// Insert our new slices. This will sort and merge them into the sorted
5512	// sequence.
5513	AS.insert(NewSlices);
5514
5515	LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5516	#ifndef NDEBUG
5517	for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5518	LLVM_DEBUG(AS.print(dbgs(), I, " "));
5519	#endif
5520
5521	// Finally, don't try to promote any allocas that new require re-splitting.
5522	// They have already been added to the worklist above.
5523	PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5524
5525	return true;
5526	}
5527
5528	/// Try to canonicalize a homogeneous struct partition to a vector type.
5529	///
5530	/// We can do this if all the elements of the struct are the same and the
5531	/// corresponding vector has the same byte-level layout. This can sometimes
5532	/// eliminate allocas because structs cannot get promoted to LLVM values, but
5533	/// vectors can.
5534	///
5535	/// We only apply this transformation when all users of the partition are memory
5536	/// intrinsics. Otherwise, if there is a load or store of some other type to the
5537	/// partition, SROA would select that type.
5538	///
5539	/// Applying this transformation too early may hinder memcpyopt, which may
5540	/// generate better code when eliminating allocas. For example, see
5541	/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5542	/// this before memcpyopt can initialize previously uninitialized memory when
5543	/// the alloca gets promoted to an SSA value. For another example, see
5544	/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5545	/// this before memcpyopt can result in promoting an alloca so that we load a
5546	/// temporary value instead of copying the temporary value into memory, whereas
5547	/// memcpyopt eliminates the temporary altogether.
5548	///
5549	/// As such, we only apply this transformation after memcpyopt has run. We gate
5550	/// this transformation by the "AggregateToVector" pass option.
5551	static FixedVectorType tryCanonicalizeStructToVector(StructType STy,
5552	Partition &P,
5553	const DataLayout &DL) {
5554	unsigned NumElts = STy->getNumElements();
5555
5556	Type *EltTy = STy->getElementType(N: `0`);
5557	if (!llvm::all_equal(Range: STy->elements()))
5558	return nullptr;
5559
5560	bool IsIntegralPointerTy =
5561	EltTy->isPointerTy() && !DL.isNonIntegralPointerType(Ty: EltTy);
5562	if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5563	!IsIntegralPointerTy)
5564	return nullptr;
5565
5566	// Ensure the struct is tightly packed so that the bit-layout is the same as
5567	// the corresponding vector. For example, this prevents a miscompile for
5568	// { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5569	// tightly packed elements and trailing padding.
5570	if (DL.getTypeSizeInBits(Ty: EltTy) != DL.getTypeAllocSizeInBits(Ty: EltTy))
5571	return nullptr;
5572
5573	auto *VTy = FixedVectorType::get(ElementType: EltTy, NumElts);
5574	TypeSize StructSize = DL.getStructLayout(Ty: STy)->getSizeInBytes();
5575	TypeSize VectorSize = DL.getTypeStoreSize(Ty: VTy);
5576	// After ruling out per-element padding, make sure a vector load/store
5577	// covers the same number of bytes as the struct layout.
5578	if (StructSize != VectorSize)
5579	return nullptr;
5580
5581	auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5582	if (S.isDead())
5583	return true;
5584	auto *U = S.getUse();
5585	return !U \|\| isa<LifetimeIntrinsic, MemIntrinsic>(Val: U->getUser());
5586	};
5587
5588	for (const Slice &S : P)
5589	if (!IsIgnorableOrMemIntrinsicSlice (S))
5590	return nullptr;
5591
5592	for (const Slice *S : P.splitSliceTails())
5593	if (!IsIgnorableOrMemIntrinsicSlice (*S))
5594	return nullptr;
5595
5596	return VTy;
5597	}
5598
5599	/// Select a partition type for an alloca partition.
5600	///
5601	/// Try to compute a friendly type for this partition of the alloca. This
5602	/// won't always succeed, in which case we fall back to a legal integer type
5603	/// or an i8 array of an appropriate size.
5604	///
5605	/// \returns A tuple with the following elements:
5606	/// - PartitionType: The computed type for this partition.
5607	/// - IsIntegerWideningViable: True if integer widening promotion is used.
5608	/// - VectorType: The vector type if vector promotion is used, otherwise
5609	/// nullptr.
5610	static std::tuple<Type , bool, VectorType >
5611	selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI,
5612	LLVMContext &C, bool AggregateToVector) {
5613	auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5614	VectorType SelectedVecTy, bool* SelectedIntWidening) {
5615	LLVM_DEBUG({
5616	dbgs() << "selectPartitionType path=" << Path
5617	<< " func=" << AI.getFunction()->getName() << " alloca=";
5618	if (AI.hasName())
5619	dbgs() << AI.getName();
5620	else
5621	dbgs() << "<unnamed>";
5622	dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5623	<< ") size=" << P.size();
5624	if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5625	dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5626	if (SelectedTy)
5627	dbgs() << " chosen=" << *SelectedTy;
5628	if (SelectedVecTy)
5629	dbgs() << " vec=" << *SelectedVecTy;
5630	dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5631	});
5632	};
5633	// First check if the partition is viable for vector promotion.
5634	//
5635	// We prefer vector promotion over integer widening promotion when:
5636	// - The vector element type is a floating-point type.
5637	// - All the loads/stores to the alloca are vector loads/stores to the
5638	// entire alloca or load/store a single element of the vector.
5639	//
5640	// Otherwise when there is an integer vector with mixed type loads/stores we
5641	// prefer integer widening promotion because it's more likely the user is
5642	// doing bitwise arithmetic and we generate better code.
5643	VectorType *VecTy =
5644	isVectorPromotionViable(P, DL, VScale: AI.getFunction()->getVScaleValue());
5645	// If the vector element type is a floating-point type, we prefer vector
5646	// promotion. If the vector has one element, let the below code select
5647	// whether we promote with the vector or scalar.
5648	if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5649	VecTy->getElementCount().getFixedValue() > `1`) {
5650	LogSelection ("direct-fp-vecty", VecTy, VecTy, false);
5651	return {VecTy, false, VecTy};
5652	}
5653
5654	// Check if there is a common type that all slices of the partition use that
5655	// spans the partition.
5656	auto [CommonUseTy, LargestIntTy] =
5657	findCommonType(B: P.begin(), E: P.end(), EndOffset: P.endOffset());
5658	if (CommonUseTy) {
5659	TypeSize CommonUseSize = DL.getTypeAllocSize(Ty: CommonUseTy);
5660	if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5661	// We prefer vector promotion here because if vector promotion is viable
5662	// and there is a common type used, then it implies the second listed
5663	// condition for preferring vector promotion is true.
5664	if (VecTy) {
5665	LogSelection ("common-type-vecty", VecTy, VecTy, false);
5666	return {VecTy, false, VecTy};
5667	}
5668	bool IntWiden = isIntegerWideningViable(P, AllocaTy: CommonUseTy, DL);
5669	LogSelection ("common-type", CommonUseTy, nullptr, IntWiden);
5670	return {CommonUseTy, IntWiden, nullptr};
5671	}
5672	}
5673
5674	// Can we find an appropriate subtype in the original allocated
5675	// type?
5676	if (Type *TypePartitionTy = getTypePartition(DL, Ty: AI.getAllocatedType(),
5677	Offset: P.beginOffset(), Size: P.size())) {
5678	// If the partition is an integer array that can be spanned by a legal
5679	// integer type, prefer to represent it as a legal integer type because
5680	// it's more likely to be promotable.
5681	if (TypePartitionTy->isArrayTy() &&
5682	TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5683	DL.isLegalInteger(Width: P.size() * `8`))
5684	TypePartitionTy = Type::getIntNTy(C, N: P.size() * `8`);
5685	// There was no common type used, so we prefer integer widening promotion.
5686	if (isIntegerWideningViable(P, AllocaTy: TypePartitionTy, DL)) {
5687	LogSelection ("type-partition-int-widen", TypePartitionTy, nullptr, true);
5688	return {TypePartitionTy, true, nullptr};
5689	}
5690	if (VecTy) {
5691	LogSelection ("type-partition-vecty", VecTy, VecTy, false);
5692	return {VecTy, false, VecTy};
5693	}
5694	// If we couldn't promote with TypePartitionTy, try with the largest
5695	// integer type used.
5696	if (LargestIntTy &&
5697	DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size() &&
5698	isIntegerWideningViable(P, AllocaTy: LargestIntTy, DL)) {
5699	LogSelection ("largest-int-int-widen", LargestIntTy, nullptr, true);
5700	return {LargestIntTy, true, nullptr};
5701	}
5702
5703	// Try homogeneous struct to vector canonicalization when requested. Running
5704	// this too early can hide memcpy chains from MemCpyOpt.
5705	if (AggregateToVector) {
5706	if (auto *STy = dyn_cast<StructType>(Val: TypePartitionTy)) {
5707	if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5708	LogSelection ("struct-fallback-vecty", VTy, nullptr, false);
5709	return {VTy, false, nullptr};
5710	}
5711	}
5712	}
5713
5714	// Fallback to TypePartitionTy and we probably won't promote.
5715	LogSelection ("type-partition-fallback", TypePartitionTy, nullptr, false);
5716	return {TypePartitionTy, false, nullptr};
5717	}
5718
5719	// Select the largest integer type used if it spans the partition.
5720	if (LargestIntTy &&
5721	DL.getTypeAllocSize(Ty: LargestIntTy).getFixedValue() >= P.size()) {
5722	LogSelection ("largest-int-fallback", LargestIntTy, nullptr, false);
5723	return {LargestIntTy, false, nullptr};
5724	}
5725
5726	// Select a legal integer type if it spans the partition.
5727	if (DL.isLegalInteger(Width: P.size() * `8`)) {
5728	Type IntTy = Type::getIntNTy(C, N: P.size() `8`);
5729	LogSelection ("legal-int-fallback", IntTy, nullptr, false);
5730	return {IntTy, false, nullptr};
5731	}
5732
5733	// Fallback to an i8 array.
5734	Type *ArrayTy = ArrayType::get(ElementType: Type::getInt8Ty(C), NumElements: P.size());
5735	LogSelection ("byte-array-fallback", ArrayTy, nullptr, false);
5736	return {ArrayTy, false, nullptr};
5737	}
5738
5739	/// Rewrite an alloca partition's users.
5740	///
5741	/// This routine drives both of the rewriting goals of the SROA pass. It tries
5742	/// to rewrite uses of an alloca partition to be conducive for SSA value
5743	/// promotion. If the partition needs a new, more refined alloca, this will
5744	/// build that new alloca, preserving as much type information as possible, and
5745	/// rewrite the uses of the old alloca to point at the new one and have the
5746	/// appropriate new offsets. It also evaluates how successful the rewrite was
5747	/// at enabling promotion and if it was successful queues the alloca to be
5748	/// promoted.
5749	std::pair<AllocaInst *, uint64_t>
5750	SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5751	const DataLayout &DL = AI.getDataLayout();
5752	// Select the type for the new alloca that spans the partition.
5753	auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5754	selectPartitionType(P, DL, AI, C&: *C, AggregateToVector);
5755
5756	// Check for the case where we're going to rewrite to a new alloca of the
5757	// exact same type as the original, and with the same access offsets. In that
5758	// case, re-use the existing alloca, but still run through the rewriter to
5759	// perform phi and select speculation.
5760	// P.beginOffset() can be non-zero even with the same type in a case with
5761	// out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5762	AllocaInst *NewAI;
5763	if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == `0`) {
5764	NewAI = &AI;
5765	// FIXME: We should be able to bail at this point with "nothing changed".
5766	// FIXME: We might want to defer PHI speculation until after here.
5767	// FIXME: return nullptr;
5768	} else {
5769	// Make sure the alignment is compatible with P.beginOffset().
5770	const Align Alignment = commonAlignment(A: AI.getAlign(), Offset: P.beginOffset());
5771	NewAI =
5772	new AllocaInst (PartitionTy, AI.getAddressSpace(), nullptr, Alignment,
5773	AI.getName() + ".sroa." + Twine (P.begin() - AS.begin()),
5774	AI.getIterator());
5775	tryEnforceAlignment(V: NewAI, PrefAlign: DL.getPrefTypeAlign(Ty: PartitionTy), DL);
5776	// Copy the old AI debug location over to the new one.
5777	NewAI->setDebugLoc(AI.getDebugLoc());
5778	++NumNewAllocas;
5779	}
5780
5781	LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5782	<< "," << P.endOffset() << ") to: " << *NewAI << "\n");
5783
5784	// Track the high watermark on the worklist as it is only relevant for
5785	// promoted allocas. We will reset it to this point if the alloca is not in
5786	// fact scheduled for promotion.
5787	unsigned PPWOldSize = PostPromotionWorklist.size();
5788	unsigned NumUses = `0`;
5789	SmallSetVector<PHINode *, `8`> PHIUsers;
5790	SmallSetVector<SelectInst *, `8`> SelectUsers;
5791
5792	AllocaSliceRewriter Rewriter(
5793	DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5794	IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5795	bool Promotable = true;
5796	// Check whether we can have tree-structured merge.
5797	if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5798	NumUses += DeletedValues ->size() + `1`;
5799	for (Value V : DeletedValues)
5800	DeadInsts.push_back(Elt: V);
5801	} else {
5802	for (Slice *S : P.splitSliceTails()) {
5803	Promotable &= Rewriter.visit(I: S);
5804	++NumUses;
5805	}
5806	for (Slice &S : P) {
5807	Promotable &= Rewriter.visit(I: &S);
5808	++NumUses;
5809	}
5810	}
5811
5812	NumAllocaPartitionUses += NumUses;
5813	MaxUsesPerAllocaPartition.updateMax(V: NumUses);
5814
5815	// Now that we've processed all the slices in the new partition, check if any
5816	// PHIs or Selects would block promotion.
5817	SmallVector<PHINode *, `8`> NewSpeculatablePHIs;
5818	SmallVector<PHINode *, `2`> NewPHIsWithStoreToRewrite;
5819	for (PHINode *PHI : PHIUsers) {
5820	if (isSafePHIToSpeculate(PN&: *PHI)) {
5821	NewSpeculatablePHIs.push_back(Elt: PHI);
5822	continue;
5823	}
5824	if (getPHIStoreToRewrite(PN&: *PHI, PreserveCFG, DT&: DTU->getDomTree())) {
5825	NewPHIsWithStoreToRewrite.push_back(Elt: PHI);
5826	continue;
5827	}
5828
5829	Promotable = false;
5830	SelectUsers.clear();
5831	break;
5832	}
5833
5834	SmallVector<std::pair<SelectInst *, RewriteableMemOps>, `2`>
5835	NewSelectsToRewrite;
5836	NewSelectsToRewrite.reserve(N: SelectUsers.size());
5837	for (SelectInst *Sel : SelectUsers) {
5838	std::optional<RewriteableMemOps> Ops =
5839	isSafeSelectToSpeculateOrPredicate(SI&: *Sel, PreserveCFG);
5840	if (!Ops) {
5841	Promotable = false;
5842	break;
5843	}
5844	NewSelectsToRewrite.emplace_back(Args: std::make_pair(x&: Sel, y&: *Ops));
5845	}
5846
5847	if (Promotable) {
5848	for (Use *U : AS.getDeadUsesIfPromotable()) {
5849	auto *OldInst = dyn_cast<Instruction>(Val: U->get());
5850	Value::dropDroppableUse(U&: *U);
5851	if (OldInst)
5852	if (isInstructionTriviallyDead(I: OldInst))
5853	DeadInsts.push_back(Elt: OldInst);
5854	}
5855	if (NewSpeculatablePHIs.empty() && NewPHIsWithStoreToRewrite.empty() &&
5856	SelectUsers.empty()) {
5857	// Promote the alloca.
5858	PromotableAllocas.insert(X: NewAI);
5859	} else {
5860	// If we have either PHIs or Selects to rewrite, add them to those
5861	// worklists and re-queue the new alloca so that we promote it on the next
5862	// iteration.
5863	SpeculatablePHIs.insert_range(R&: NewSpeculatablePHIs);
5864	PHIsWithStoreToRewrite.insert_range(R&: NewPHIsWithStoreToRewrite);
5865	SelectsToRewrite.reserve(NumEntries: SelectsToRewrite.size() +
5866	NewSelectsToRewrite.size());
5867	for (auto &&KV : llvm::make_range(
5868	x: std::make_move_iterator(i: NewSelectsToRewrite.begin()),
5869	y: std::make_move_iterator(i: NewSelectsToRewrite.end())))
5870	SelectsToRewrite.insert(KV: std::move(KV));
5871	Worklist.insert(X: NewAI);
5872	}
5873	} else {
5874	// Drop any post-promotion work items if promotion didn't happen.
5875	while (PostPromotionWorklist.size() > PPWOldSize)
5876	PostPromotionWorklist.pop_back();
5877
5878	// We couldn't promote and we didn't create a new partition, nothing
5879	// happened.
5880	if (NewAI == &AI)
5881	return {nullptr, `0`};
5882
5883	// If we can't promote the alloca, iterate on it to check for new
5884	// refinements exposed by splitting the current alloca. Don't iterate on an
5885	// alloca which didn't actually change and didn't get promoted.
5886	Worklist.insert(X: NewAI);
5887	}
5888
5889	return {NewAI, DL.getTypeSizeInBits(Ty: PartitionTy).getFixedValue()};
5890	}
5891
5892	// There isn't a shared interface to get the "address" parts out of a
5893	// dbg.declare and dbg.assign, so provide some wrappers.
5894	bool isKillAddress(const DbgVariableRecord *DVR) {
5895	if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5896	return DVR->isKillAddress();
5897	return DVR->isKillLocation();
5898	}
5899
5900	const DIExpression getAddressExpression(const* DbgVariableRecord *DVR) {
5901	if (DVR->getType() == DbgVariableRecord::LocationType::Assign)
5902	return DVR->getAddressExpression();
5903	return DVR->getExpression();
5904	}
5905
5906	/// Create or replace an existing fragment in a DIExpression with \p Frag.
5907	/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5908	/// operation, add \p BitExtractOffset to the offset part.
5909	///
5910	/// Returns the new expression, or nullptr if this fails (see details below).
5911	///
5912	/// This function is similar to DIExpression::createFragmentExpression except
5913	/// for 3 important distinctions:
5914	/// 1. The new fragment isn't relative to an existing fragment.
5915	/// 2. It assumes the computed location is a memory location. This means we
5916	/// don't need to perform checks that creating the fragment preserves the
5917	/// expression semantics.
5918	/// 3. Existing extract_bits are modified independently of fragment changes
5919	/// using \p BitExtractOffset. A change to the fragment offset or size
5920	/// may affect a bit extract. But a bit extract offset can change
5921	/// independently of the fragment dimensions.
5922	///
5923	/// Returns the new expression, or nullptr if one couldn't be created.
5924	/// Ideally this is only used to signal that a bit-extract has become
5925	/// zero-sized (and thus the new debug record has no size and can be
5926	/// dropped), however, it fails for other reasons too - see the FIXME below.
5927	///
5928	/// FIXME: To keep the change that introduces this function NFC it bails
5929	/// in some situations unecessarily, e.g. when fragment and bit extract
5930	/// sizes differ.
5931	static DIExpression createOrReplaceFragment(const* DIExpression *Expr,
5932	DIExpression::FragmentInfo Frag,
5933	int64_t BitExtractOffset) {
5934	SmallVector<uint64_t, `8`> Ops;
5935	bool HasFragment = false;
5936	bool HasBitExtract = false;
5937
5938	for (auto &Op : Expr->expr_ops()) {
5939	if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5940	HasFragment = true;
5941	continue;
5942	}
5943	if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val: Op)) {
5944	HasBitExtract = true;
5945	int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5946	int64_t ExtractSizeInBits = Extract.getSizeInBits();
5947
5948	// DIExpression::createFragmentExpression doesn't know how to handle
5949	// a fragment that is smaller than the extract. Copy the behaviour
5950	// (bail) to avoid non-NFC changes.
5951	// FIXME: Don't do this.
5952	if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5953	return nullptr;
5954
5955	assert(BitExtractOffset <= `0`);
5956	int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5957
5958	// DIExpression::createFragmentExpression doesn't know what to do
5959	// if the new extract starts "outside" the existing one. Copy the
5960	// behaviour (bail) to avoid non-NFC changes.
5961	// FIXME: Don't do this.
5962	if (AdjustedOffset < `0`)
5963	return nullptr;
5964
5965	Ops.push_back(Elt: Op.getOp());
5966	Ops.push_back(Elt: std::max<int64_t>(a: `0`, b: AdjustedOffset));
5967	Ops.push_back(Elt: ExtractSizeInBits);
5968	continue;
5969	}
5970	Op.appendToVector(V&: Ops);
5971	}
5972
5973	// Unsupported by createFragmentExpression, so don't support it here yet to
5974	// preserve NFC-ness.
5975	if (HasFragment && HasBitExtract)
5976	return nullptr;
5977
5978	if (!HasBitExtract) {
5979	Ops.push_back(Elt: dwarf::DW_OP_LLVM_fragment);
5980	Ops.push_back(Elt: Frag.OffsetInBits);
5981	Ops.push_back(Elt: Frag.SizeInBits);
5982	}
5983	return DIExpression::get(Context&: Expr->getContext(), Elements: Ops);
5984	}
5985
5986	/// Insert a new DbgRecord.
5987	/// \p Orig Original to copy record type, debug loc and variable from, and
5988	/// additionally value and value expression for dbg_assign records.
5989	/// \p NewAddr Location's new base address.
5990	/// \p NewAddrExpr New expression to apply to address.
5991	/// \p BeforeInst Insert position.
5992	/// \p NewFragment New fragment (absolute, non-relative).
5993	/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
5994	static void
5995	insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord Orig, AllocaInst NewAddr,
5996	DIExpression NewAddrExpr, Instruction BeforeInst,
5997	std::optional<DIExpression::FragmentInfo> NewFragment,
5998	int64_t BitExtractAdjustment) {
5999	(void)DIB;
6000
6001	// A dbg_assign puts fragment info in the value expression only. The address
6002	// expression has already been built: NewAddrExpr. A dbg_declare puts the
6003	// new fragment info into NewAddrExpr (as it only has one expression).
6004	DIExpression *NewFragmentExpr =
6005	Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
6006	if (NewFragment)
6007	NewFragmentExpr = createOrReplaceFragment(Expr: NewFragmentExpr, Frag: *NewFragment,
6008	BitExtractOffset: BitExtractAdjustment);
6009	if (!NewFragmentExpr)
6010	return;
6011
6012	if (Orig->isDbgDeclare()) {
6013	DbgVariableRecord *DVR = DbgVariableRecord::createDVRDeclare(
6014	Address: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
6015	BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
6016	Here: BeforeInst->getIterator());
6017	return;
6018	}
6019
6020	if (Orig->isDbgValue()) {
6021	DbgVariableRecord *DVR = DbgVariableRecord::createDbgVariableRecord(
6022	Location: NewAddr, DV: Orig->getVariable(), Expr: NewFragmentExpr, DI: Orig->getDebugLoc());
6023	// Drop debug information if the expression doesn't start with a
6024	// DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
6025	// describes the address of alloca rather than the value inside the alloca.
6026	if (!NewFragmentExpr->startsWithDeref())
6027	DVR->setKillAddress();
6028	BeforeInst->getParent()->insertDbgRecordBefore(DR: DVR,
6029	Here: BeforeInst->getIterator());
6030	return;
6031	}
6032
6033	// Apply a DIAssignID to the store if it doesn't already have it.
6034	if (!NewAddr->hasMetadata(KindID: LLVMContext::MD_DIAssignID)) {
6035	NewAddr->setMetadata(KindID: LLVMContext::MD_DIAssignID,
6036	Node: DIAssignID::getDistinct(Context&: NewAddr->getContext()));
6037	}
6038
6039	DbgVariableRecord *NewAssign = DbgVariableRecord::createLinkedDVRAssign(
6040	LinkedInstr: NewAddr, Val: Orig->getValue(), Variable: Orig->getVariable(), Expression: NewFragmentExpr, Address: NewAddr,
6041	AddressExpression: NewAddrExpr, DI: Orig->getDebugLoc());
6042	LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
6043	(void)NewAssign;
6044	}
6045
6046	/// Walks the slices of an alloca and form partitions based on them,
6047	/// rewriting each of their uses.
6048	bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
6049	if (AS.begin() == AS.end())
6050	return false;
6051
6052	unsigned NumPartitions = `0`;
6053	bool Changed = false;
6054	const DataLayout &DL = AI.getDataLayout();
6055
6056	// First try to pre-split loads and stores.
6057	Changed \|= presplitLoadsAndStores(AI, AS);
6058
6059	// Now that we have identified any pre-splitting opportunities,
6060	// mark loads and stores unsplittable except for the following case.
6061	// We leave a slice splittable if all other slices are disjoint or fully
6062	// included in the slice, such as whole-alloca loads and stores.
6063	// If we fail to split these during pre-splitting, we want to force them
6064	// to be rewritten into a partition.
6065	bool IsSorted = true;
6066
6067	uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
6068	// We can split at the begin and end offsets of each slice, but only if those
6069	// offsets don't lie inside another slice. Because slices are ordered by
6070	// increasing begin offset, and then decreasing end offset, we can consider
6071	// the slices as being split up into sets with the same begin offset where we
6072	// can ignore every slice except the first (the begin offset will already be
6073	// handled as the begin offset of the set, and the end offset we know is not a
6074	// splittable offset as it's inside the first slice of the set).
6075	SparseBitVector<> SplittableOffset;
6076	uint64_t CurBegin = `0`, CurEnd = `0`;
6077	for (Slice &S : AS) {
6078	// Check if we have a new set of slices
6079	if (S.beginOffset() > CurBegin \|\| S.endOffset() > CurEnd) {
6080	// If the start isn't inside the previous set it's splittable
6081	if (S.beginOffset() >= CurEnd) {
6082	SplittableOffset.set(S.beginOffset());
6083	}
6084	// If the previous end is inside this slice then remove it
6085	if (CurEnd > S.beginOffset() && CurEnd < S.endOffset()) {
6086	SplittableOffset.reset(Idx: CurEnd);
6087	}
6088	CurBegin = S.beginOffset();
6089	// If the end offset isn't inside the previous set it's splittable. We
6090	// also don't update the end offset in that case, as the next set may also
6091	// be inside the previous set.
6092	if (S.endOffset() > CurEnd) {
6093	CurEnd = S.endOffset();
6094	SplittableOffset.set(CurEnd);
6095	}
6096	}
6097	}
6098
6099	for (Slice &S : AS) {
6100	if (!S.isSplittable())
6101	continue;
6102
6103	if ((S.beginOffset() > AllocaSize \|\|
6104	SplittableOffset.test(Idx: S.beginOffset())) &&
6105	(S.endOffset() > AllocaSize \|\| SplittableOffset.test(Idx: S.endOffset())))
6106	continue;
6107
6108	if (isa<LoadInst>(Val: S.getUse()->getUser()) \|\|
6109	isa<StoreInst>(Val: S.getUse()->getUser())) {
6110	S.makeUnsplittable();
6111	IsSorted = false;
6112	}
6113	}
6114
6115	if (!IsSorted)
6116	llvm::stable_sort(Range&: AS);
6117
6118	/// Describes the allocas introduced by rewritePartition in order to migrate
6119	/// the debug info.
6120	struct Fragment {
6121	AllocaInst *Alloca;
6122	uint64_t Offset;
6123	uint64_t Size;
6124	Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
6125	: Alloca(AI), Offset(O), Size(S) {}
6126	};
6127	SmallVector<Fragment, `4`> Fragments;
6128
6129	// Rewrite each partition.
6130	for (auto &P : AS.partitions()) {
6131	auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
6132	if (NewAI) {
6133	Changed = true;
6134	if (NewAI != &AI) {
6135	uint64_t SizeOfByte = `8`;
6136	// Don't include any padding.
6137	uint64_t Size = std::min(a: ActiveBits, b: P.size() * SizeOfByte);
6138	Fragments.push_back(
6139	Elt: Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
6140	}
6141	}
6142	++NumPartitions;
6143	}
6144
6145	NumAllocaPartitions += NumPartitions;
6146	MaxPartitionsPerAlloca.updateMax(V: NumPartitions);
6147
6148	// Migrate debug information from the old alloca to the new alloca(s)
6149	// and the individual partitions.
6150	auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
6151	// Can't overlap with undef memory.
6152	if (isKillAddress(DVR: DbgVariable))
6153	return;
6154
6155	const Value *DbgPtr = DbgVariable->getAddress();
6156	DIExpression::FragmentInfo VarFrag =
6157	DbgVariable->getFragmentOrEntireVariable();
6158	// Get the address expression constant offset if one exists and the ops
6159	// that come after it.
6160	int64_t CurrentExprOffsetInBytes = `0`;
6161	SmallVector<uint64_t> PostOffsetOps;
6162	if (!getAddressExpression(DVR: DbgVariable)
6163	->extractLeadingOffset(OffsetInBytes&: CurrentExprOffsetInBytes, RemainingOps&: PostOffsetOps))
6164	return; // Couldn't interpret this DIExpression - drop the var.
6165
6166	// Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
6167	int64_t ExtractOffsetInBits = `0`;
6168	for (auto Op : getAddressExpression(DVR: DbgVariable)->expr_ops()) {
6169	if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Val&: Op)) {
6170	ExtractOffsetInBits = Extract.getOffsetInBits();
6171	break;
6172	}
6173	}
6174
6175	DIBuilder DIB(AI.getModule(), /AllowUnresolved/* false);
6176	for (auto Fragment : Fragments) {
6177	int64_t OffsetFromLocationInBits;
6178	std::optional<DIExpression::FragmentInfo> NewDbgFragment;
6179	// Find the variable fragment that the new alloca slice covers.
6180	// Drop debug info for this variable fragment if we can't compute an
6181	// intersect between it and the alloca slice.
6182	if (!DIExpression::calculateFragmentIntersect(
6183	DL, SliceStart: &AI, SliceOffsetInBits: Fragment.Offset, SliceSizeInBits: Fragment.Size, DbgPtr,
6184	DbgPtrOffsetInBits: CurrentExprOffsetInBytes * `8`, DbgExtractOffsetInBits: ExtractOffsetInBits, VarFrag,
6185	Result&: NewDbgFragment, OffsetFromLocationInBits))
6186	continue; // Do not migrate this fragment to this slice.
6187
6188	// Zero sized fragment indicates there's no intersect between the variable
6189	// fragment and the alloca slice. Skip this slice for this variable
6190	// fragment.
6191	if (NewDbgFragment && !NewDbgFragment ->SizeInBits)
6192	continue; // Do not migrate this fragment to this slice.
6193
6194	// No fragment indicates DbgVariable's variable or fragment exactly
6195	// overlaps the slice; copy its fragment (or nullopt if there isn't one).
6196	if (!NewDbgFragment)
6197	NewDbgFragment = DbgVariable->getFragment();
6198
6199	// Reduce the new expression offset by the bit-extract offset since
6200	// we'll be keeping that.
6201	int64_t OffestFromNewAllocaInBits =
6202	OffsetFromLocationInBits - ExtractOffsetInBits;
6203	// We need to adjust an existing bit extract if the offset expression
6204	// can't eat the slack (i.e., if the new offset would be negative).
6205	int64_t BitExtractOffset =
6206	std::min<int64_t>(a: `0`, b: OffestFromNewAllocaInBits);
6207	// The magnitude of a negative value indicates the number of bits into
6208	// the existing variable fragment that the memory region begins. The new
6209	// variable fragment already excludes those bits - the new DbgPtr offset
6210	// only needs to be applied if it's positive.
6211	OffestFromNewAllocaInBits =
6212	std::max(a: int64_t(`0`), b: OffestFromNewAllocaInBits);
6213
6214	// Rebuild the expression:
6215	// {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
6216	// Add NewDbgFragment later, because dbg.assigns don't want it in the
6217	// address expression but the value expression instead.
6218	DIExpression *NewExpr = DIExpression::get(Context&: AI.getContext(), Elements: PostOffsetOps);
6219	if (OffestFromNewAllocaInBits > `0`) {
6220	int64_t OffsetInBytes = (OffestFromNewAllocaInBits + `7`) / `8`;
6221	NewExpr = DIExpression::prepend(Expr: NewExpr, /flags=/Flags: `0`, Offset: OffsetInBytes);
6222	}
6223
6224	// Remove any existing intrinsics on the new alloca describing
6225	// the variable fragment.
6226	auto RemoveOne = [DbgVariable](auto *OldDII) {
6227	auto SameVariableFragment = [](const auto LHS, const* auto *RHS) {
6228	return LHS->getVariable() == RHS->getVariable() &&
6229	LHS->getDebugLoc()->getInlinedAt() ==
6230	RHS->getDebugLoc()->getInlinedAt();
6231	};
6232	if (SameVariableFragment(OldDII, DbgVariable))
6233	OldDII->eraseFromParent();
6234	};
6235	for_each(Range: findDVRDeclares(V: Fragment.Alloca), F: RemoveOne);
6236	for_each(Range: findDVRValues(V: Fragment.Alloca), F: RemoveOne);
6237	insertNewDbgInst(DIB, Orig: DbgVariable, NewAddr: Fragment.Alloca, NewAddrExpr: NewExpr, BeforeInst: &AI,
6238	NewFragment: NewDbgFragment, BitExtractAdjustment: BitExtractOffset);
6239	}
6240	};
6241
6242	// Migrate debug information from the old alloca to the new alloca(s)
6243	// and the individual partitions.
6244	for_each(Range: findDVRDeclares(V: &AI), F: MigrateOne);
6245	for_each(Range: findDVRValues(V: &AI), F: MigrateOne);
6246	for_each(Range: at::getDVRAssignmentMarkers(Inst: &AI), F: MigrateOne);
6247
6248	return Changed;
6249	}
6250
6251	/// Clobber a use with poison, deleting the used value if it becomes dead.
6252	void SROA::clobberUse(Use &U) {
6253	Value *OldV = U;
6254	// Replace the use with an poison value.
6255	U = PoisonValue::get(T: OldV->getType());
6256
6257	// Check for this making an instruction dead. We have to garbage collect
6258	// all the dead instructions to ensure the uses of any alloca end up being
6259	// minimal.
6260	if (Instruction *OldI = dyn_cast<Instruction>(Val: OldV))
6261	if (isInstructionTriviallyDead(I: OldI)) {
6262	DeadInsts.push_back(Elt: OldI);
6263	}
6264	}
6265
6266	/// A basic LoadAndStorePromoter that does not remove store nodes.
6267	class BasicLoadAndStorePromoter : public LoadAndStorePromoter {
6268	public:
6269	BasicLoadAndStorePromoter(ArrayRef<const Instruction *> Insts, SSAUpdater &S,
6270	Type *ZeroType)
6271	: LoadAndStorePromoter (Insts, S), ZeroType(ZeroType) {}
6272	bool shouldDelete(Instruction I) const* override {
6273	return !isa<StoreInst>(Val: I) && !isa<AllocaInst>(Val: I);
6274	}
6275
6276	Value getValueToUseForAlloca(Instruction I) const override {
6277	return UndefValue::get(T: ZeroType);
6278	}
6279
6280	private:
6281	Type *ZeroType;
6282	};
6283
6284	bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6285	// Look through each "partition", looking for slices with the same start/end
6286	// that do not overlap with any before them. The slices are sorted by
6287	// increasing beginOffset. We don't use AS.partitions(), as it will use a more
6288	// sophisticated algorithm that takes splittable slices into account.
6289	LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6290	bool AllSameAndValid = true;
6291	Type PartitionType = nullptr*;
6292	SmallVector<Instruction *> Insts;
6293	uint64_t BeginOffset = `0`;
6294	uint64_t EndOffset = `0`;
6295
6296	auto Flush = [&]() {
6297	if (AllSameAndValid && !Insts.empty()) {
6298	LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6299	<< EndOffset << ")\n");
6300	SmallVector<PHINode *, `4`> NewPHIs;
6301	SSAUpdater SSA(&NewPHIs);
6302	Insts.push_back(Elt: &AI);
6303	BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6304	Promoter.run(Insts);
6305	}
6306	AllSameAndValid = true;
6307	PartitionType = nullptr;
6308	Insts.clear();
6309	};
6310
6311	for (Slice &S : AS) {
6312	auto *User = cast<Instruction>(Val: S.getUse()->getUser());
6313	if (isAssumeLikeIntrinsic(I: User)) {
6314	LLVM_DEBUG({
6315	dbgs() << "Ignoring slice: ";
6316	AS.print(dbgs(), &S);
6317	});
6318	continue;
6319	}
6320	if (S.beginOffset() >= EndOffset) {
6321	Flush ();
6322	BeginOffset = S.beginOffset();
6323	EndOffset = S.endOffset();
6324	} else if (S.beginOffset() != BeginOffset \|\| S.endOffset() != EndOffset) {
6325	if (AllSameAndValid) {
6326	LLVM_DEBUG({
6327	dbgs() << "Slice does not match range [" << BeginOffset << ", "
6328	<< EndOffset << ")";
6329	AS.print(dbgs(), &S);
6330	});
6331	AllSameAndValid = false;
6332	}
6333	EndOffset = std::max(a: EndOffset, b: S.endOffset());
6334	continue;
6335	}
6336
6337	if (auto *LI = dyn_cast<LoadInst>(Val: User)) {
6338	Type *UserTy = LI->getType();
6339	// LoadAndStorePromoter requires all the types to be the same.
6340	if (!LI->isSimple() \|\| (PartitionType && UserTy != PartitionType))
6341	AllSameAndValid = false;
6342	PartitionType = UserTy;
6343	Insts.push_back(Elt: User);
6344	} else if (auto *SI = dyn_cast<StoreInst>(Val: User)) {
6345	Type *UserTy = SI->getValueOperand()->getType();
6346	if (!SI->isSimple() \|\| (PartitionType && UserTy != PartitionType))
6347	AllSameAndValid = false;
6348	PartitionType = UserTy;
6349	Insts.push_back(Elt: User);
6350	} else {
6351	AllSameAndValid = false;
6352	}
6353	}
6354
6355	Flush ();
6356	return true;
6357	}
6358
6359	/// Analyze an alloca for SROA.
6360	///
6361	/// This analyzes the alloca to ensure we can reason about it, builds
6362	/// the slices of the alloca, and then hands it off to be split and
6363	/// rewritten as needed.
6364	std::pair<bool /Changed/, bool /CFGChanged/>
6365	SROA::runOnAlloca(AllocaInst &AI) {
6366	bool Changed = false;
6367	bool CFGChanged = false;
6368
6369	LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6370	++NumAllocasAnalyzed;
6371
6372	// Special case dead allocas, as they're trivial.
6373	if (AI.use_empty()) {
6374	AI.eraseFromParent();
6375	Changed = true;
6376	return {Changed, CFGChanged};
6377	}
6378	const DataLayout &DL = AI.getDataLayout();
6379
6380	// Skip alloca forms that this analysis can't handle.
6381	std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6382	if (AI.isArrayAllocation() \|\| !Size \|\| Size ->isScalable() \|\| Size ->isZero())
6383	return {Changed, CFGChanged};
6384
6385	// First, split any FCA loads and stores touching this alloca to promote
6386	// better splitting and promotion opportunities.
6387	IRBuilderTy IRB(&AI);
6388	AggLoadStoreRewriter AggRewriter(DL, IRB);
6389	Changed \|= AggRewriter.rewrite(I&: AI);
6390
6391	// Build the slices using a recursive instruction-visiting builder.
6392	AllocaSlices AS(DL, AI);
6393	LLVM_DEBUG(AS.print(dbgs()));
6394	if (AS.isEscaped())
6395	return {Changed, CFGChanged};
6396
6397	if (AS.isEscapedReadOnly()) {
6398	Changed \|= propagateStoredValuesToLoads(AI, AS);
6399	return {Changed, CFGChanged};
6400	}
6401
6402	// Delete all the dead users of this alloca before splitting and rewriting it.
6403	for (Instruction *DeadUser : AS.getDeadUsers()) {
6404	// Free up everything used by this instruction.
6405	for (Use &DeadOp : DeadUser->operands())
6406	clobberUse(U&: DeadOp);
6407
6408	// Now replace the uses of this instruction.
6409	DeadUser->replaceAllUsesWith(V: PoisonValue::get(T: DeadUser->getType()));
6410
6411	// And mark it for deletion.
6412	DeadInsts.push_back(Elt: DeadUser);
6413	Changed = true;
6414	}
6415	for (Use *DeadOp : AS.getDeadOperands()) {
6416	clobberUse(U&: *DeadOp);
6417	Changed = true;
6418	}
6419
6420	// No slices to split. Leave the dead alloca for a later pass to clean up.
6421	if (AS.begin() == AS.end())
6422	return {Changed, CFGChanged};
6423
6424	Changed \|= splitAlloca(AI, AS);
6425
6426	LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6427	while (!SpeculatablePHIs.empty())
6428	speculatePHINodeLoads(IRB, PN&: *SpeculatablePHIs.pop_back_val());
6429
6430	LLVM_DEBUG(dbgs() << " Rewriting stores through PHIs\n");
6431	auto RemainingPHIsWithStoreToRewrite = PHIsWithStoreToRewrite.takeVector();
6432	while (!RemainingPHIsWithStoreToRewrite.empty()) {
6433	PHINode *PN = RemainingPHIsWithStoreToRewrite.pop_back_val();
6434	auto *SI = cast<StoreInst>(Val: PN->user_back());
6435	CFGChanged \|= rewritePHINodeStore(PN&: PN, SI&: SI, DTU&: *DTU, Worklist);
6436	}
6437
6438	LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6439	auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6440	while (!RemainingSelectsToRewrite.empty()) {
6441	const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6442	CFGChanged \|=
6443	rewriteSelectInstMemOps(SI&: K, Ops: V, IRB, DTU: PreserveCFG ? nullptr* : DTU);
6444	}
6445
6446	return {Changed, CFGChanged};
6447	}
6448
6449	/// Delete the dead instructions accumulated in this run.
6450	///
6451	/// Recursively deletes the dead instructions we've accumulated. This is done
6452	/// at the very end to maximize locality of the recursive delete and to
6453	/// minimize the problems of invalidated instruction pointers as such pointers
6454	/// are used heavily in the intermediate stages of the algorithm.
6455	///
6456	/// We also record the alloca instructions deleted here so that they aren't
6457	/// subsequently handed to mem2reg to promote.
6458	bool SROA::deleteDeadInstructions(
6459	SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6460	bool Changed = false;
6461	while (!DeadInsts.empty()) {
6462	Instruction *I = dyn_cast_or_null<Instruction>(Val: DeadInsts.pop_back_val());
6463	if (!I)
6464	continue;
6465	LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6466
6467	// If the instruction is an alloca, find the possible dbg.declare connected
6468	// to it, and remove it too. We must do this before calling RAUW or we will
6469	// not be able to find it.
6470	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val: I)) {
6471	DeletedAllocas.insert(Ptr: AI);
6472	for (DbgVariableRecord *OldDII : findDVRDeclares(V: AI))
6473	OldDII->eraseFromParent();
6474	}
6475
6476	at::deleteAssignmentMarkers(Inst: I);
6477	I->replaceAllUsesWith(V: UndefValue::get(T: I->getType()));
6478
6479	for (Use &Operand : I->operands())
6480	if (Instruction *U = dyn_cast<Instruction>(Val&: Operand)) {
6481	// Zero out the operand and see if it becomes trivially dead.
6482	Operand = nullptr;
6483	if (isInstructionTriviallyDead(I: U))
6484	DeadInsts.push_back(Elt: U);
6485	}
6486
6487	++NumDeleted;
6488	I->eraseFromParent();
6489	Changed = true;
6490	}
6491	return Changed;
6492	}
6493	/// Promote the allocas, using the best available technique.
6494	///
6495	/// This attempts to promote whatever allocas have been identified as viable in
6496	/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6497	/// This function returns whether any promotion occurred.
6498	bool SROA::promoteAllocas() {
6499	if (PromotableAllocas.empty())
6500	return false;
6501
6502	if (C->getLibraryOptions().get<ScalarOptions>().sroa_skip_mem2reg) {
6503	LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6504	} else {
6505	LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6506	NumPromoted += PromotableAllocas.size();
6507	PromoteMemToReg(Allocas: PromotableAllocas.getArrayRef(), DT&: DTU->getDomTree(), AC);
6508	}
6509
6510	PromotableAllocas.clear();
6511	return true;
6512	}
6513
6514	std::pair<bool /Changed/, bool /CFGChanged/> SROA::runSROA(Function &F) {
6515	LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6516
6517	const DataLayout &DL = F.getDataLayout();
6518	BasicBlock &EntryBB = F.getEntryBlock();
6519	for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(x: EntryBB.end());
6520	I != E; ++I) {
6521	if (AllocaInst *AI = dyn_cast<AllocaInst>(Val&: I)) {
6522	std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6523	if (Size && Size ->isScalable() && isAllocaPromotable(AI))
6524	PromotableAllocas.insert(X: AI);
6525	else
6526	Worklist.insert(X: AI);
6527	}
6528	}
6529
6530	bool Changed = false;
6531	bool CFGChanged = false;
6532	// A set of deleted alloca instruction pointers which should be removed from
6533	// the list of promotable allocas.
6534	SmallPtrSet<AllocaInst *, `4`> DeletedAllocas;
6535
6536	do {
6537	while (!Worklist.empty()) {
6538	auto [IterationChanged, IterationCFGChanged] =
6539	runOnAlloca(AI&: *Worklist.pop_back_val());
6540	Changed \|= IterationChanged;
6541	CFGChanged \|= IterationCFGChanged;
6542
6543	Changed \|= deleteDeadInstructions(DeletedAllocas);
6544
6545	// Remove the deleted allocas from various lists so that we don't try to
6546	// continue processing them.
6547	if (!DeletedAllocas.empty()) {
6548	Worklist.set_subtract(DeletedAllocas);
6549	PostPromotionWorklist.set_subtract(DeletedAllocas);
6550	PromotableAllocas.set_subtract(DeletedAllocas);
6551	DeletedAllocas.clear();
6552	}
6553	}
6554
6555	Changed \|= promoteAllocas();
6556
6557	Worklist = PostPromotionWorklist;
6558	PostPromotionWorklist.clear();
6559	} while (!Worklist.empty());
6560
6561	assert((!CFGChanged \|\| Changed) && "Can not only modify the CFG.");
6562	assert((!CFGChanged \|\| !PreserveCFG) &&
6563	"Should not have modified the CFG when told to preserve it.");
6564
6565	if (Changed && isAssignmentTrackingEnabled(M: *F.getParent())) {
6566	for (auto &BB : F) {
6567	RemoveRedundantDbgInstrs(BB: &BB);
6568	}
6569	}
6570
6571	return {Changed, CFGChanged};
6572	}
6573
6574	PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) {
6575	DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
6576	AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
6577	DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6578	auto [Changed, CFGChanged] =
6579	SROA (&F.getContext(), &DTU, &AC, Options).runSROA(F);
6580	if (!Changed)
6581	return PreservedAnalyses::all();
6582	PreservedAnalyses PA;
6583	if (!CFGChanged)
6584	PA.preserveSet<CFGAnalyses>();
6585	PA.preserve<DominatorTreeAnalysis>();
6586	return PA;
6587	}
6588
6589	void SROAPass::printPipeline(
6590	raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6591	static_cast<PassInfoMixin<SROAPass> >(this*)->printPipeline(
6592	OS, MapClassName2PassName);
6593	OS << `'<'`
6594	<< (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6595	: "modify-cfg");
6596	if (Options.AggregateToVector)
6597	OS << ";aggregate-to-vector";
6598	OS << `'>'`;
6599	}
6600
6601	SROAPass::SROAPass(SROAOptions Options) : Options (Options) {}
6602
6603	namespace {
6604
6605	/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6606	class SROALegacyPass : public FunctionPass {
6607	SROAOptions Options;
6608
6609	public:
6610	static char ID;
6611
6612	SROALegacyPass(SROAOptions Options = SROAOptions::PreserveCFG)
6613	: FunctionPass (ID), Options (Options) {
6614	initializeSROALegacyPassPass(*PassRegistry::getPassRegistry());
6615	}
6616
6617	bool runOnFunction(Function &F) override {
6618	if (skipFunction(F))
6619	return false;
6620
6621	DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6622	AssumptionCache &AC =
6623	getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6624	DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6625	auto [Changed, _] = SROA (&F.getContext(), &DTU, &AC, Options).runSROA(F);
6626	return Changed;
6627	}
6628
6629	void getAnalysisUsage(AnalysisUsage &AU) const override {
6630	AU.addRequired<AssumptionCacheTracker>();
6631	AU.addRequired<DominatorTreeWrapperPass>();
6632	AU.addPreserved<GlobalsAAWrapperPass>();
6633	AU.addPreserved<DominatorTreeWrapperPass>();
6634	}
6635
6636	StringRef getPassName() const override { return "SROA"; }
6637	};
6638
6639	} // end anonymous namespace
6640
6641	char SROALegacyPass::ID = `0`;
6642
6643	FunctionPass llvm::createSROAPass(bool* PreserveCFG, bool AggregateToVector) {
6644	return new SROALegacyPass (SROAOptions (PreserveCFG ? SROAOptions::PreserveCFG
6645	: SROAOptions::ModifyCFG,
6646	AggregateToVector));
6647	}
6648
6649	INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6650	"Scalar Replacement Of Aggregates", false, false)
6651	INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
6652	INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
6653	INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
6654	false, false)
6655

Browse the source code of llvm_projects/llvm/lib/Transforms/Scalar/SROA.cpp