1//===- LoadStoreOpt.cpp ----------- Generic memory optimizations -*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the LoadStoreOpt optimization pass.
10//===----------------------------------------------------------------------===//
11
12#include "llvm/CodeGen/GlobalISel/LoadStoreOpt.h"
13#include "llvm/ADT/STLExtras.h"
14#include "llvm/ADT/SmallPtrSet.h"
15#include "llvm/ADT/Statistic.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/MemoryLocation.h"
18#include "llvm/Analysis/OptimizationRemarkEmitter.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
21#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/LowLevelTypeUtils.h"
24#include "llvm/CodeGen/MachineBasicBlock.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
28#include "llvm/CodeGen/MachineInstr.h"
29#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
30#include "llvm/CodeGen/MachinePassManager.h"
31#include "llvm/CodeGen/MachineRegisterInfo.h"
32#include "llvm/CodeGen/Register.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetOpcodes.h"
35#include "llvm/IR/Analysis.h"
36#include "llvm/InitializePasses.h"
37#include "llvm/Support/AtomicOrdering.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Support/Debug.h"
40#include "llvm/Support/ErrorHandling.h"
41#include <algorithm>
42
43#define DEBUG_TYPE "load-store-opt"
44
45using namespace llvm;
46using namespace llvm::GISelAddressing;
47using namespace ore;
48using namespace MIPatternMatch;
49
50STATISTIC(NumStoresMerged, "Number of stores merged");
51
52const unsigned MaxStoreSizeToForm = 128;
53
54namespace {
55
56class LoadStoreOptImpl {
57 MachineRegisterInfo *MRI = nullptr;
58 const TargetLowering *TLI = nullptr;
59 MachineFunction *MF = nullptr;
60 AliasAnalysis *AA = nullptr;
61 const LegalizerInfo *LI = nullptr;
62
63 MachineIRBuilder Builder;
64
65 /// Initialize the field members using \p MF.
66 void init(MachineFunction &MF, function_ref<AliasAnalysis *()> GetAA);
67
68 class StoreMergeCandidate {
69 public:
70 // The base pointer used as the base for all stores in this candidate.
71 Register BasePtr;
72 // Our algorithm is very simple at the moment. We assume that in instruction
73 // order stores are writing to incremeneting consecutive addresses. So when
74 // we walk the block in reverse order, the next eligible store must write to
75 // an offset one store width lower than CurrentLowestOffset.
76 int64_t CurrentLowestOffset;
77 SmallVector<GStore *> Stores;
78 // A vector of MachineInstr/unsigned pairs to denote potential aliases that
79 // need to be checked before the candidate is considered safe to merge. The
80 // unsigned value is an index into the Stores vector. The indexed store is
81 // the highest-indexed store that has already been checked to not have an
82 // alias with the instruction. We record this so we don't have to repeat
83 // alias checks that have been already done, only those with stores added
84 // after the potential alias is recorded.
85 SmallVector<std::pair<MachineInstr *, unsigned>> PotentialAliases;
86
87 LLVM_ABI void addPotentialAlias(MachineInstr &MI);
88
89 /// Reset this candidate back to an empty one.
90 void reset() {
91 Stores.clear();
92 PotentialAliases.clear();
93 CurrentLowestOffset = 0;
94 BasePtr = Register();
95 }
96 };
97
98 bool isLegalOrBeforeLegalizer(const LegalityQuery &Query,
99 MachineFunction &MF) const;
100 /// If the given store is valid to be a member of the candidate, add it and
101 /// return true. Otherwise, returns false.
102 bool addStoreToCandidate(GStore &MI, StoreMergeCandidate &C);
103 /// Returns true if the instruction \p MI would potentially alias with any
104 /// stores in the candidate \p C.
105 bool operationAliasesWithCandidate(MachineInstr &MI, StoreMergeCandidate &C);
106 /// Merges the stores in the given vector into a wide store.
107 /// \p returns true if at least some of the stores were merged.
108 /// This may decide not to merge stores if heuristics predict it will not be
109 /// worth it.
110 bool mergeStores(SmallVectorImpl<GStore *> &StoresToMerge);
111 /// Perform a merge of all the stores in \p Stores into a single store.
112 /// Erases the old stores from the block when finished.
113 /// \returns true if merging was done. It may fail to perform a merge if
114 /// there are issues with materializing legal wide values.
115 bool doSingleStoreMerge(SmallVectorImpl<GStore *> &Stores);
116 bool processMergeCandidate(StoreMergeCandidate &C);
117 bool mergeBlockStores(MachineBasicBlock &MBB);
118 bool mergeFunctionStores(MachineFunction &MF);
119
120 bool mergeTruncStore(GStore &StoreMI,
121 SmallPtrSetImpl<GStore *> &DeletedStores);
122 bool mergeTruncStoresBlock(MachineBasicBlock &MBB);
123
124 /// Initialize some target-specific data structures for the store merging
125 /// optimization. \p AddrSpace indicates which address space to use when
126 /// probing the legalizer info for legal stores.
127 void initializeStoreMergeTargetInfo(unsigned AddrSpace = 0);
128 /// A map between address space numbers and a bitvector of supported stores
129 /// sizes. Each bit in the bitvector represents whether a store size of
130 /// that bit's value is legal. E.g. if bit 64 is set, then 64 bit scalar
131 /// stores are legal.
132 DenseMap<unsigned, BitVector> LegalStoreSizes;
133 bool IsPreLegalizer = false;
134 /// Contains instructions to be erased at the end of a block scan.
135 SmallPtrSet<MachineInstr *, 16> InstsToErase;
136
137public:
138 bool runOnMachineFunction(MachineFunction &MF,
139 function_ref<AliasAnalysis *()> GetAA);
140};
141
142} // namespace
143
144char LoadStoreOptLegacy::ID = 0;
145INITIALIZE_PASS_BEGIN(LoadStoreOptLegacy, DEBUG_TYPE,
146 "Generic memory optimizations", false, false)
147INITIALIZE_PASS_END(LoadStoreOptLegacy, DEBUG_TYPE,
148 "Generic memory optimizations", false, false)
149
150LoadStoreOptLegacy::LoadStoreOptLegacy() : MachineFunctionPass(ID) {}
151
152void LoadStoreOptImpl::init(MachineFunction &MF,
153 function_ref<AliasAnalysis *()> GetAA) {
154 this->MF = &MF;
155 MRI = &MF.getRegInfo();
156 AA = GetAA();
157 TLI = MF.getSubtarget().getTargetLowering();
158 LI = MF.getSubtarget().getLegalizerInfo();
159 Builder.setMF(MF);
160 IsPreLegalizer = !MF.getProperties().hasLegalized();
161 InstsToErase.clear();
162}
163
164void LoadStoreOptLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
165 AU.addRequired<AAResultsWrapperPass>();
166 AU.setPreservesAll();
167 getSelectionDAGFallbackAnalysisUsage(AU);
168 MachineFunctionPass::getAnalysisUsage(AU);
169}
170
171BaseIndexOffset GISelAddressing::getPointerInfo(Register Ptr,
172 MachineRegisterInfo &MRI) {
173 BaseIndexOffset Info;
174 Register PtrAddRHS;
175 Register BaseReg;
176 if (!mi_match(R: Ptr, MRI, P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_Reg(R&: PtrAddRHS)))) {
177 Info.setBase(Ptr);
178 Info.setOffset(0);
179 return Info;
180 }
181 Info.setBase(BaseReg);
182 auto RHSCst = getIConstantVRegValWithLookThrough(VReg: PtrAddRHS, MRI);
183 if (RHSCst)
184 Info.setOffset(RHSCst->Value.getSExtValue());
185
186 // Just recognize a simple case for now. In future we'll need to match
187 // indexing patterns for base + index + constant.
188 Info.setIndex(PtrAddRHS);
189 return Info;
190}
191
192bool GISelAddressing::aliasIsKnownForLoadStore(const MachineInstr &MI1,
193 const MachineInstr &MI2,
194 bool &IsAlias,
195 MachineRegisterInfo &MRI) {
196 auto *LdSt1 = dyn_cast<GLoadStore>(Val: &MI1);
197 auto *LdSt2 = dyn_cast<GLoadStore>(Val: &MI2);
198 if (!LdSt1 || !LdSt2)
199 return false;
200
201 BaseIndexOffset BasePtr0 = getPointerInfo(Ptr: LdSt1->getPointerReg(), MRI);
202 BaseIndexOffset BasePtr1 = getPointerInfo(Ptr: LdSt2->getPointerReg(), MRI);
203
204 if (!BasePtr0.getBase().isValid() || !BasePtr1.getBase().isValid())
205 return false;
206
207 LocationSize Size1 = LdSt1->getMemSize();
208 LocationSize Size2 = LdSt2->getMemSize();
209
210 int64_t PtrDiff;
211 if (BasePtr0.getBase() == BasePtr1.getBase() && BasePtr0.hasValidOffset() &&
212 BasePtr1.hasValidOffset()) {
213 PtrDiff = BasePtr1.getOffset() - BasePtr0.getOffset();
214 // If the size of memory access is unknown, do not use it to do analysis.
215 // One example of unknown size memory access is to load/store scalable
216 // vector objects on the stack.
217 // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
218 // following situations arise:
219 if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) {
220 // [----BasePtr0----]
221 // [---BasePtr1--]
222 // ========PtrDiff========>
223 IsAlias = !((int64_t)Size1.getValue() <= PtrDiff);
224 return true;
225 }
226 if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) {
227 // [----BasePtr0----]
228 // [---BasePtr1--]
229 // =====(-PtrDiff)====>
230 IsAlias = !((PtrDiff + (int64_t)Size2.getValue()) <= 0);
231 return true;
232 }
233 return false;
234 }
235
236 // If both BasePtr0 and BasePtr1 are FrameIndexes, we will not be
237 // able to calculate their relative offset if at least one arises
238 // from an alloca. However, these allocas cannot overlap and we
239 // can infer there is no alias.
240 auto *Base0Def = getDefIgnoringCopies(Reg: BasePtr0.getBase(), MRI);
241 auto *Base1Def = getDefIgnoringCopies(Reg: BasePtr1.getBase(), MRI);
242 if (!Base0Def || !Base1Def)
243 return false; // Couldn't tell anything.
244
245
246 if (Base0Def->getOpcode() != Base1Def->getOpcode())
247 return false;
248
249 if (Base0Def->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
250 MachineFrameInfo &MFI = Base0Def->getMF()->getFrameInfo();
251 // If the bases have the same frame index but we couldn't find a
252 // constant offset, (indices are different) be conservative.
253 if (Base0Def != Base1Def &&
254 (!MFI.isFixedObjectIndex(ObjectIdx: Base0Def->getOperand(i: 1).getIndex()) ||
255 !MFI.isFixedObjectIndex(ObjectIdx: Base1Def->getOperand(i: 1).getIndex()))) {
256 IsAlias = false;
257 return true;
258 }
259 }
260
261 // This implementation is a lot more primitive than the SDAG one for now.
262 // FIXME: what about constant pools?
263 if (Base0Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
264 auto GV0 = Base0Def->getOperand(i: 1).getGlobal();
265 auto GV1 = Base1Def->getOperand(i: 1).getGlobal();
266 if (GV0 != GV1) {
267 IsAlias = false;
268 return true;
269 }
270 }
271
272 // Can't tell anything about aliasing.
273 return false;
274}
275
276bool GISelAddressing::instMayAlias(const MachineInstr &MI,
277 const MachineInstr &Other,
278 MachineRegisterInfo &MRI,
279 AliasAnalysis *AA) {
280 struct MemUseCharacteristics {
281 bool IsVolatile;
282 bool IsAtomic;
283 Register BasePtr;
284 int64_t Offset;
285 LocationSize NumBytes;
286 MachineMemOperand *MMO;
287 };
288
289 auto getCharacteristics =
290 [&](const MachineInstr *MI) -> MemUseCharacteristics {
291 if (const auto *LS = dyn_cast<GLoadStore>(Val: MI)) {
292 Register BaseReg;
293 int64_t Offset = 0;
294 // No pre/post-inc addressing modes are considered here, unlike in SDAG.
295 if (!mi_match(R: LS->getPointerReg(), MRI,
296 P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: Offset)))) {
297 BaseReg = LS->getPointerReg();
298 Offset = 0;
299 }
300
301 LocationSize Size = LS->getMMO().getSize();
302 return {.IsVolatile: LS->isVolatile(), .IsAtomic: LS->isAtomic(), .BasePtr: BaseReg,
303 .Offset: Offset /*base offset*/, .NumBytes: Size, .MMO: &LS->getMMO()};
304 }
305 // FIXME: support recognizing lifetime instructions.
306 // Default.
307 return {.IsVolatile: false /*isvolatile*/,
308 /*isAtomic*/ .IsAtomic: false,
309 .BasePtr: Register(),
310 .Offset: (int64_t)0 /*offset*/,
311 .NumBytes: LocationSize::beforeOrAfterPointer() /*size*/,
312 .MMO: (MachineMemOperand *)nullptr};
313 };
314 MemUseCharacteristics MUC0 = getCharacteristics(&MI),
315 MUC1 = getCharacteristics(&Other);
316
317 // If they are to the same address, then they must be aliases.
318 if (MUC0.BasePtr.isValid() && MUC0.BasePtr == MUC1.BasePtr &&
319 MUC0.Offset == MUC1.Offset)
320 return true;
321
322 // If they are both volatile then they cannot be reordered.
323 if (MUC0.IsVolatile && MUC1.IsVolatile)
324 return true;
325
326 // Be conservative about atomics for the moment
327 // TODO: This is way overconservative for unordered atomics (see D66309)
328 if (MUC0.IsAtomic && MUC1.IsAtomic)
329 return true;
330
331 // If one operation reads from invariant memory, and the other may store, they
332 // cannot alias.
333 if (MUC0.MMO && MUC1.MMO) {
334 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
335 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
336 return false;
337 }
338
339 // If NumBytes is scalable and offset is not 0, conservatively return may
340 // alias
341 if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) ||
342 (MUC1.NumBytes.isScalable() && MUC1.Offset != 0))
343 return true;
344
345 const bool BothNotScalable =
346 !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable();
347
348 // Try to prove that there is aliasing, or that there is no aliasing. Either
349 // way, we can return now. If nothing can be proved, proceed with more tests.
350 bool IsAlias;
351 if (BothNotScalable &&
352 GISelAddressing::aliasIsKnownForLoadStore(MI1: MI, MI2: Other, IsAlias, MRI))
353 return IsAlias;
354
355 // The following all rely on MMO0 and MMO1 being valid.
356 if (!MUC0.MMO || !MUC1.MMO)
357 return true;
358
359 // FIXME: port the alignment based alias analysis from SDAG's isAlias().
360 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
361 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
362 LocationSize Size0 = MUC0.NumBytes;
363 LocationSize Size1 = MUC1.NumBytes;
364 if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0.hasValue() &&
365 Size1.hasValue()) {
366 // Use alias analysis information.
367 int64_t MinOffset = std::min(a: SrcValOffset0, b: SrcValOffset1);
368 int64_t Overlap0 =
369 Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
370 int64_t Overlap1 =
371 Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
372 LocationSize Loc0 =
373 Size0.isScalable() ? Size0 : LocationSize::precise(Value: Overlap0);
374 LocationSize Loc1 =
375 Size1.isScalable() ? Size1 : LocationSize::precise(Value: Overlap1);
376
377 if (AA->isNoAlias(
378 LocA: MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
379 LocB: MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
380 return false;
381 }
382
383 // Otherwise we have to assume they alias.
384 return true;
385}
386
387/// Returns true if the instruction creates an unavoidable hazard that
388/// forces a boundary between store merge candidates.
389static bool isInstHardMergeHazard(MachineInstr &MI) {
390 return MI.hasUnmodeledSideEffects() || MI.hasOrderedMemoryRef();
391}
392
393bool LoadStoreOptImpl::mergeStores(SmallVectorImpl<GStore *> &StoresToMerge) {
394 // Try to merge all the stores in the vector, splitting into separate segments
395 // as necessary.
396 assert(StoresToMerge.size() > 1 && "Expected multiple stores to merge");
397 LLT OrigTy = MRI->getType(Reg: StoresToMerge[0]->getValueReg());
398 LLT PtrTy = MRI->getType(Reg: StoresToMerge[0]->getPointerReg());
399 unsigned AS = PtrTy.getAddressSpace();
400 // Ensure the legal store info is computed for this address space.
401 initializeStoreMergeTargetInfo(AddrSpace: AS);
402 const auto &LegalSizes = LegalStoreSizes[AS];
403
404 // FIXME: Support mismatching types (i16 + f16).
405 for (auto *StoreMI : StoresToMerge)
406 if (MRI->getType(Reg: StoreMI->getValueReg()) != OrigTy)
407 return false;
408
409 bool AnyMerged = false;
410 do {
411 unsigned NumPow2 = llvm::bit_floor(Value: StoresToMerge.size());
412 unsigned MaxSizeBits = NumPow2 * OrigTy.getSizeInBits().getFixedValue();
413 // Compute the biggest store we can generate to handle the number of stores.
414 unsigned MergeSizeBits;
415 for (MergeSizeBits = MaxSizeBits; MergeSizeBits > 1; MergeSizeBits /= 2) {
416 LLT StoreTy = LLT::scalar(SizeInBits: MergeSizeBits);
417 EVT StoreEVT =
418 getApproximateEVTForLLT(Ty: StoreTy, Ctx&: MF->getFunction().getContext());
419 if (LegalSizes.size() > MergeSizeBits && LegalSizes[MergeSizeBits] &&
420 TLI->canMergeStoresTo(AS, MemVT: StoreEVT, MF: *MF) &&
421 (TLI->isTypeLegal(VT: StoreEVT)))
422 break; // We can generate a MergeSize bits store.
423 }
424 if (MergeSizeBits <= OrigTy.getSizeInBits())
425 return AnyMerged; // No greater merge.
426
427 unsigned NumStoresToMerge = MergeSizeBits / OrigTy.getSizeInBits();
428 // Perform the actual merging.
429 SmallVector<GStore *, 8> SingleMergeStores(
430 StoresToMerge.begin(), StoresToMerge.begin() + NumStoresToMerge);
431 AnyMerged |= doSingleStoreMerge(Stores&: SingleMergeStores);
432 StoresToMerge.erase(CS: StoresToMerge.begin(),
433 CE: StoresToMerge.begin() + NumStoresToMerge);
434 } while (StoresToMerge.size() > 1);
435 return AnyMerged;
436}
437
438bool LoadStoreOptImpl::isLegalOrBeforeLegalizer(const LegalityQuery &Query,
439 MachineFunction &MF) const {
440 auto Action = LI->getAction(Query).Action;
441 // If the instruction is unsupported, it can't be legalized at all.
442 if (Action == LegalizeActions::Unsupported)
443 return false;
444 return IsPreLegalizer || Action == LegalizeAction::Legal;
445}
446
447bool LoadStoreOptImpl::doSingleStoreMerge(SmallVectorImpl<GStore *> &Stores) {
448 assert(Stores.size() > 1);
449 // We know that all the stores are consecutive and there are no aliasing
450 // operations in the range. However, the values that are being stored may be
451 // generated anywhere before each store. To ensure we have the values
452 // available, we materialize the wide value and new store at the place of the
453 // final store in the merge sequence.
454 GStore *FirstStore = Stores[0];
455 const unsigned NumStores = Stores.size();
456 LLT SmallTy = MRI->getType(Reg: FirstStore->getValueReg());
457 LLT WideValueTy =
458 LLT::integer(SizeInBits: NumStores * SmallTy.getSizeInBits().getFixedValue());
459
460 // For each store, compute pairwise merged debug locs.
461 DebugLoc MergedLoc = Stores.front()->getDebugLoc();
462 for (auto *Store : drop_begin(RangeOrContainer&: Stores))
463 MergedLoc = DebugLoc::getMergedLocation(LocA: MergedLoc, LocB: Store->getDebugLoc());
464
465 Builder.setInstr(*Stores.back());
466 Builder.setDebugLoc(MergedLoc);
467
468 // If all of the store values are constants, then create a wide constant
469 // directly. Otherwise, we need to generate some instructions to merge the
470 // existing values together into a wider type.
471 SmallVector<APInt, 8> ConstantVals;
472 for (auto *Store : Stores) {
473 auto MaybeCst =
474 getIConstantVRegValWithLookThrough(VReg: Store->getValueReg(), MRI: *MRI);
475 if (!MaybeCst) {
476 ConstantVals.clear();
477 break;
478 }
479 ConstantVals.emplace_back(Args&: MaybeCst->Value);
480 }
481
482 Register WideReg;
483 auto *WideMMO =
484 MF->getMachineMemOperand(MMO: &FirstStore->getMMO(), Offset: 0, Ty: WideValueTy);
485 if (ConstantVals.empty()) {
486 // Mimic the SDAG behaviour here and don't try to do anything for unknown
487 // values. In future, we should also support the cases of loads and
488 // extracted vector elements.
489 return false;
490 }
491
492 assert(ConstantVals.size() == NumStores);
493 // Check if our wide constant is legal.
494 if (!isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_CONSTANT, {WideValueTy}}, MF&: *MF))
495 return false;
496 APInt WideConst(WideValueTy.getSizeInBits(), 0);
497 for (unsigned Idx = 0; Idx < ConstantVals.size(); ++Idx) {
498 // Insert the smaller constant into the corresponding position in the
499 // wider one.
500 WideConst.insertBits(SubBits: ConstantVals[Idx], bitPosition: Idx * SmallTy.getSizeInBits());
501 }
502 WideReg = Builder.buildConstant(Res: WideValueTy, Val: WideConst).getReg(Idx: 0);
503 auto NewStore =
504 Builder.buildStore(Val: WideReg, Addr: FirstStore->getPointerReg(), MMO&: *WideMMO);
505 (void) NewStore;
506 LLVM_DEBUG(dbgs() << "Merged " << Stores.size()
507 << " stores into merged store: " << *NewStore);
508 LLVM_DEBUG(for (auto *MI : Stores) dbgs() << " " << *MI;);
509 NumStoresMerged += Stores.size();
510
511 MachineOptimizationRemarkEmitter MORE(*MF, nullptr);
512 MORE.emit(RemarkBuilder: [&]() {
513 MachineOptimizationRemark R(DEBUG_TYPE, "MergedStore",
514 FirstStore->getDebugLoc(),
515 FirstStore->getParent());
516 R << "Merged " << NV("NumMerged", Stores.size()) << " stores of "
517 << NV("OrigWidth", SmallTy.getSizeInBytes())
518 << " bytes into a single store of "
519 << NV("NewWidth", WideValueTy.getSizeInBytes()) << " bytes";
520 return R;
521 });
522
523 InstsToErase.insert_range(R&: Stores);
524 return true;
525}
526
527bool LoadStoreOptImpl::processMergeCandidate(StoreMergeCandidate &C) {
528 if (C.Stores.size() < 2) {
529 C.reset();
530 return false;
531 }
532
533 LLVM_DEBUG(dbgs() << "Checking store merge candidate with " << C.Stores.size()
534 << " stores, starting with " << *C.Stores[0]);
535 // We know that the stores in the candidate are adjacent.
536 // Now we need to check if any potential aliasing instructions recorded
537 // during the search alias with load/stores added to the candidate after.
538 // For example, if we have the candidate:
539 // C.Stores = [ST1, ST2, ST3, ST4]
540 // and after seeing ST2 we saw a load LD1, which did not alias with ST1 or
541 // ST2, then we would have recorded it into the PotentialAliases structure
542 // with the associated index value of "1". Then we see ST3 and ST4 and add
543 // them to the candidate group. We know that LD1 does not alias with ST1 or
544 // ST2, since we already did that check. However we don't yet know if it
545 // may alias ST3 and ST4, so we perform those checks now.
546 SmallVector<GStore *> StoresToMerge;
547
548 auto DoesStoreAliasWithPotential = [&](unsigned Idx, GStore &CheckStore) {
549 for (auto AliasInfo : reverse(C&: C.PotentialAliases)) {
550 MachineInstr *PotentialAliasOp = AliasInfo.first;
551 unsigned PreCheckedIdx = AliasInfo.second;
552 if (Idx < PreCheckedIdx) {
553 // Once our store index is lower than the index associated with the
554 // potential alias, we know that we've already checked for this alias
555 // and all of the earlier potential aliases too.
556 return false;
557 }
558 // Need to check this alias.
559 if (GISelAddressing::instMayAlias(MI: CheckStore, Other: *PotentialAliasOp, MRI&: *MRI,
560 AA)) {
561 LLVM_DEBUG(dbgs() << "Potential alias " << *PotentialAliasOp
562 << " detected\n");
563 return true;
564 }
565 }
566 return false;
567 };
568 // Start from the last store in the group, and check if it aliases with any
569 // of the potential aliasing operations in the list.
570 for (int StoreIdx = C.Stores.size() - 1; StoreIdx >= 0; --StoreIdx) {
571 auto *CheckStore = C.Stores[StoreIdx];
572 if (DoesStoreAliasWithPotential(StoreIdx, *CheckStore))
573 continue;
574 StoresToMerge.emplace_back(Args&: CheckStore);
575 }
576
577 LLVM_DEBUG(dbgs() << StoresToMerge.size()
578 << " stores remaining after alias checks. Merging...\n");
579
580 // Now we've checked for aliasing hazards, merge any stores left.
581 C.reset();
582 if (StoresToMerge.size() < 2)
583 return false;
584 return mergeStores(StoresToMerge);
585}
586
587bool LoadStoreOptImpl::operationAliasesWithCandidate(MachineInstr &MI,
588 StoreMergeCandidate &C) {
589 if (C.Stores.empty())
590 return false;
591 return llvm::any_of(Range&: C.Stores, P: [&](MachineInstr *OtherMI) {
592 return instMayAlias(MI, Other: *OtherMI, MRI&: *MRI, AA);
593 });
594}
595
596void LoadStoreOptImpl::StoreMergeCandidate::addPotentialAlias(
597 MachineInstr &MI) {
598 PotentialAliases.emplace_back(Args: std::make_pair(x: &MI, y: Stores.size() - 1));
599}
600
601bool LoadStoreOptImpl::addStoreToCandidate(GStore &StoreMI,
602 StoreMergeCandidate &C) {
603 // Check if the given store writes to an adjacent address, and other
604 // requirements.
605 LLT ValueTy = MRI->getType(Reg: StoreMI.getValueReg());
606 LLT PtrTy = MRI->getType(Reg: StoreMI.getPointerReg());
607
608 // Only handle scalars.
609 if (!ValueTy.isScalar())
610 return false;
611
612 // Don't allow truncating stores for now.
613 if (StoreMI.getMemSizeInBits() != ValueTy.getSizeInBits())
614 return false;
615
616 // Avoid adding volatile or ordered stores to the candidate. We already have a
617 // check for this in instMayAlias() but that only get's called later between
618 // potential aliasing hazards.
619 if (!StoreMI.isSimple())
620 return false;
621
622 Register StoreAddr = StoreMI.getPointerReg();
623 auto BIO = getPointerInfo(Ptr: StoreAddr, MRI&: *MRI);
624 Register StoreBase = BIO.getBase();
625 if (C.Stores.empty()) {
626 C.BasePtr = StoreBase;
627 if (!BIO.hasValidOffset()) {
628 C.CurrentLowestOffset = 0;
629 } else {
630 C.CurrentLowestOffset = BIO.getOffset();
631 }
632 // This is the first store of the candidate.
633 // If the offset can't possibly allow for a lower addressed store with the
634 // same base, don't bother adding it.
635 if (BIO.hasValidOffset() &&
636 BIO.getOffset() < static_cast<int64_t>(ValueTy.getSizeInBytes()))
637 return false;
638 C.Stores.emplace_back(Args: &StoreMI);
639 LLVM_DEBUG(dbgs() << "Starting a new merge candidate group with: "
640 << StoreMI);
641 return true;
642 }
643
644 // Check the store is the same size as the existing ones in the candidate.
645 if (MRI->getType(Reg: C.Stores[0]->getValueReg()).getSizeInBits() !=
646 ValueTy.getSizeInBits())
647 return false;
648
649 if (MRI->getType(Reg: C.Stores[0]->getPointerReg()).getAddressSpace() !=
650 PtrTy.getAddressSpace())
651 return false;
652
653 // There are other stores in the candidate. Check that the store address
654 // writes to the next lowest adjacent address.
655 if (C.BasePtr != StoreBase)
656 return false;
657 // If we don't have a valid offset, we can't guarantee to be an adjacent
658 // offset.
659 if (!BIO.hasValidOffset())
660 return false;
661 if ((C.CurrentLowestOffset -
662 static_cast<int64_t>(ValueTy.getSizeInBytes())) != BIO.getOffset())
663 return false;
664
665 // This writes to an adjacent address. Allow it.
666 C.Stores.emplace_back(Args: &StoreMI);
667 C.CurrentLowestOffset = C.CurrentLowestOffset - ValueTy.getSizeInBytes();
668 LLVM_DEBUG(dbgs() << "Candidate added store: " << StoreMI);
669 return true;
670}
671
672bool LoadStoreOptImpl::mergeBlockStores(MachineBasicBlock &MBB) {
673 bool Changed = false;
674 // Walk through the block bottom-up, looking for merging candidates.
675 StoreMergeCandidate Candidate;
676 for (MachineInstr &MI : llvm::reverse(C&: MBB)) {
677 if (InstsToErase.contains(Ptr: &MI))
678 continue;
679
680 if (auto *StoreMI = dyn_cast<GStore>(Val: &MI)) {
681 // We have a G_STORE. Add it to the candidate if it writes to an adjacent
682 // address.
683 if (!addStoreToCandidate(StoreMI&: *StoreMI, C&: Candidate)) {
684 // Store wasn't eligible to be added. May need to record it as a
685 // potential alias.
686 if (operationAliasesWithCandidate(MI&: *StoreMI, C&: Candidate)) {
687 Changed |= processMergeCandidate(C&: Candidate);
688 continue;
689 }
690 Candidate.addPotentialAlias(MI&: *StoreMI);
691 }
692 continue;
693 }
694
695 // If we don't have any stores yet, this instruction can't pose a problem.
696 if (Candidate.Stores.empty())
697 continue;
698
699 // We're dealing with some other kind of instruction.
700 if (isInstHardMergeHazard(MI)) {
701 Changed |= processMergeCandidate(C&: Candidate);
702 Candidate.Stores.clear();
703 continue;
704 }
705
706 if (!MI.mayLoadOrStore())
707 continue;
708
709 if (operationAliasesWithCandidate(MI, C&: Candidate)) {
710 // We have a potential alias, so process the current candidate if we can
711 // and then continue looking for a new candidate.
712 Changed |= processMergeCandidate(C&: Candidate);
713 continue;
714 }
715
716 // Record this instruction as a potential alias for future stores that are
717 // added to the candidate.
718 Candidate.addPotentialAlias(MI);
719 }
720
721 // Process any candidate left after finishing searching the entire block.
722 Changed |= processMergeCandidate(C&: Candidate);
723
724 // Erase instructions now that we're no longer iterating over the block.
725 for (auto *MI : InstsToErase)
726 MI->eraseFromParent();
727 InstsToErase.clear();
728 return Changed;
729}
730
731/// Check if the store \p Store is a truncstore that can be merged. That is,
732/// it's a store of a shifted value of \p SrcVal. If \p SrcVal is an empty
733/// Register then it does not need to match and SrcVal is set to the source
734/// value found.
735/// On match, returns the start byte offset of the \p SrcVal that is being
736/// stored.
737static std::optional<int64_t>
738getTruncStoreByteOffset(GStore &Store, Register &SrcVal,
739 MachineRegisterInfo &MRI) {
740 Register TruncVal;
741 if (!mi_match(R: Store.getValueReg(), MRI, P: m_GTrunc(Src: m_Reg(R&: TruncVal))))
742 return std::nullopt;
743
744 // The shift amount must be a constant multiple of the narrow type.
745 // It is translated to the offset address in the wide source value "y".
746 //
747 // x = G_LSHR y, ShiftAmtC
748 // s8 z = G_TRUNC x
749 // store z, ...
750 Register FoundSrcVal;
751 int64_t ShiftAmt;
752 if (!mi_match(R: TruncVal, MRI,
753 P: m_any_of(preds: m_GLShr(L: m_Reg(R&: FoundSrcVal), R: m_ICst(Cst&: ShiftAmt)),
754 preds: m_GAShr(L: m_Reg(R&: FoundSrcVal), R: m_ICst(Cst&: ShiftAmt))))) {
755 if (!SrcVal.isValid() || TruncVal == SrcVal) {
756 if (!SrcVal.isValid())
757 SrcVal = TruncVal;
758 return 0; // If it's the lowest index store.
759 }
760 return std::nullopt;
761 }
762
763 unsigned NarrowBits = Store.getMMO().getMemoryType().getScalarSizeInBits();
764 if (ShiftAmt % NarrowBits != 0)
765 return std::nullopt;
766 const unsigned Offset = ShiftAmt / NarrowBits;
767
768 if (SrcVal.isValid() && FoundSrcVal != SrcVal)
769 return std::nullopt;
770
771 if (!SrcVal.isValid())
772 SrcVal = FoundSrcVal;
773 else if (MRI.getType(Reg: SrcVal) != MRI.getType(Reg: FoundSrcVal))
774 return std::nullopt;
775 return Offset;
776}
777
778/// Match a pattern where a wide type scalar value is stored by several narrow
779/// stores. Fold it into a single store or a BSWAP and a store if the targets
780/// supports it.
781///
782/// Assuming little endian target:
783/// i8 *p = ...
784/// i32 val = ...
785/// p[0] = (val >> 0) & 0xFF;
786/// p[1] = (val >> 8) & 0xFF;
787/// p[2] = (val >> 16) & 0xFF;
788/// p[3] = (val >> 24) & 0xFF;
789/// =>
790/// *((i32)p) = val;
791///
792/// i8 *p = ...
793/// i32 val = ...
794/// p[0] = (val >> 24) & 0xFF;
795/// p[1] = (val >> 16) & 0xFF;
796/// p[2] = (val >> 8) & 0xFF;
797/// p[3] = (val >> 0) & 0xFF;
798/// =>
799/// *((i32)p) = BSWAP(val);
800bool LoadStoreOptImpl::mergeTruncStore(
801 GStore &StoreMI, SmallPtrSetImpl<GStore *> &DeletedStores) {
802 LLT MemTy = StoreMI.getMMO().getMemoryType();
803
804 // We only handle merging simple stores of 1-4 bytes.
805 if (!MemTy.isScalar())
806 return false;
807 switch (MemTy.getSizeInBits()) {
808 case 8:
809 case 16:
810 case 32:
811 break;
812 default:
813 return false;
814 }
815 if (!StoreMI.isSimple())
816 return false;
817
818 // We do a simple search for mergeable stores prior to this one.
819 // Any potential alias hazard along the way terminates the search.
820 SmallVector<GStore *> FoundStores;
821
822 // We're looking for:
823 // 1) a (store(trunc(...)))
824 // 2) of an LSHR/ASHR of a single wide value, by the appropriate shift to get
825 // the partial value stored.
826 // 3) where the offsets form either a little or big-endian sequence.
827
828 auto &LastStore = StoreMI;
829
830 // The single base pointer that all stores must use.
831 Register BaseReg;
832 int64_t LastOffset;
833 if (!mi_match(R: LastStore.getPointerReg(), MRI: *MRI,
834 P: m_GPtrAdd(L: m_Reg(R&: BaseReg), R: m_ICst(Cst&: LastOffset)))) {
835 BaseReg = LastStore.getPointerReg();
836 LastOffset = 0;
837 }
838
839 GStore *LowestIdxStore = &LastStore;
840 int64_t LowestIdxOffset = LastOffset;
841
842 Register WideSrcVal;
843 auto LowestShiftAmt = getTruncStoreByteOffset(Store&: LastStore, SrcVal&: WideSrcVal, MRI&: *MRI);
844 if (!LowestShiftAmt)
845 return false; // Didn't match a trunc.
846 assert(WideSrcVal.isValid());
847
848 LLT WideStoreTy = MRI->getType(Reg: WideSrcVal);
849 // The wide type might not be a multiple of the memory type, e.g. s48 and s32.
850 if (WideStoreTy.getSizeInBits() % MemTy.getSizeInBits() != 0)
851 return false;
852 const unsigned NumStoresRequired =
853 WideStoreTy.getSizeInBits() / MemTy.getSizeInBits();
854
855 SmallVector<int64_t, 8> OffsetMap(NumStoresRequired, INT64_MAX);
856 OffsetMap[*LowestShiftAmt] = LastOffset;
857 FoundStores.emplace_back(Args: &LastStore);
858
859 const int MaxInstsToCheck = 10;
860 int NumInstsChecked = 0;
861 for (auto II = ++LastStore.getReverseIterator();
862 II != LastStore.getParent()->rend() && NumInstsChecked < MaxInstsToCheck;
863 ++II) {
864 NumInstsChecked++;
865 GStore *NewStore;
866 if ((NewStore = dyn_cast<GStore>(Val: &*II))) {
867 if (NewStore->getMMO().getMemoryType() != MemTy || !NewStore->isSimple())
868 break;
869 } else if (II->isLoadFoldBarrier() || II->mayLoad()) {
870 break;
871 } else {
872 continue; // This is a safe instruction we can look past.
873 }
874
875 Register NewBaseReg;
876 int64_t MemOffset;
877 // Check we're storing to the same base + some offset.
878 if (!mi_match(R: NewStore->getPointerReg(), MRI: *MRI,
879 P: m_GPtrAdd(L: m_Reg(R&: NewBaseReg), R: m_ICst(Cst&: MemOffset)))) {
880 NewBaseReg = NewStore->getPointerReg();
881 MemOffset = 0;
882 }
883 if (BaseReg != NewBaseReg)
884 break;
885
886 auto ShiftByteOffset = getTruncStoreByteOffset(Store&: *NewStore, SrcVal&: WideSrcVal, MRI&: *MRI);
887 if (!ShiftByteOffset)
888 break;
889 if (MemOffset < LowestIdxOffset) {
890 LowestIdxOffset = MemOffset;
891 LowestIdxStore = NewStore;
892 }
893
894 // Map the offset in the store and the offset in the combined value, and
895 // early return if it has been set before.
896 if (*ShiftByteOffset < 0 || *ShiftByteOffset >= NumStoresRequired ||
897 OffsetMap[*ShiftByteOffset] != INT64_MAX)
898 break;
899 OffsetMap[*ShiftByteOffset] = MemOffset;
900
901 FoundStores.emplace_back(Args&: NewStore);
902 // Reset counter since we've found a matching inst.
903 NumInstsChecked = 0;
904 if (FoundStores.size() == NumStoresRequired)
905 break;
906 }
907
908 if (FoundStores.size() != NumStoresRequired) {
909 if (FoundStores.size() == 1)
910 return false;
911 // We didn't find enough stores to merge into the size of the original
912 // source value, but we may be able to generate a smaller store if we
913 // truncate the source value.
914 WideStoreTy =
915 LLT::integer(SizeInBits: FoundStores.size() * MemTy.getScalarSizeInBits());
916 }
917
918 unsigned NumStoresFound = FoundStores.size();
919
920 const auto &DL = LastStore.getMF()->getDataLayout();
921 auto &C = LastStore.getMF()->getFunction().getContext();
922 // Check that a store of the wide type is both allowed and fast on the target
923 unsigned Fast = 0;
924 bool Allowed = TLI->allowsMemoryAccess(
925 Context&: C, DL, Ty: WideStoreTy, MMO: LowestIdxStore->getMMO(), Fast: &Fast);
926 if (!Allowed || !Fast)
927 return false;
928
929 // Check if the pieces of the value are going to the expected places in memory
930 // to merge the stores.
931 unsigned NarrowBits = MemTy.getScalarSizeInBits();
932 auto checkOffsets = [&](bool MatchLittleEndian) {
933 if (MatchLittleEndian) {
934 for (unsigned i = 0; i != NumStoresFound; ++i)
935 if (OffsetMap[i] != i * (NarrowBits / 8) + LowestIdxOffset)
936 return false;
937 } else { // MatchBigEndian by reversing loop counter.
938 for (unsigned i = 0, j = NumStoresFound - 1; i != NumStoresFound;
939 ++i, --j)
940 if (OffsetMap[j] != i * (NarrowBits / 8) + LowestIdxOffset)
941 return false;
942 }
943 return true;
944 };
945
946 // Check if the offsets line up for the native data layout of this target.
947 bool NeedBswap = false;
948 bool NeedRotate = false;
949 if (!checkOffsets(DL.isLittleEndian())) {
950 // Special-case: check if byte offsets line up for the opposite endian.
951 if (NarrowBits == 8 && checkOffsets(DL.isBigEndian()))
952 NeedBswap = true;
953 else if (NumStoresFound == 2 && checkOffsets(DL.isBigEndian()))
954 NeedRotate = true;
955 else
956 return false;
957 }
958
959 if (NeedBswap &&
960 !isLegalOrBeforeLegalizer(Query: {TargetOpcode::G_BSWAP, {WideStoreTy}}, MF&: *MF))
961 return false;
962 if (NeedRotate &&
963 !isLegalOrBeforeLegalizer(
964 Query: {TargetOpcode::G_ROTR, {WideStoreTy, WideStoreTy}}, MF&: *MF))
965 return false;
966
967 Builder.setInstrAndDebugLoc(StoreMI);
968
969 if (WideStoreTy != MRI->getType(Reg: WideSrcVal))
970 WideSrcVal = Builder.buildTrunc(Res: WideStoreTy, Op: WideSrcVal).getReg(Idx: 0);
971
972 if (NeedBswap) {
973 WideSrcVal = Builder.buildBSwap(Dst: WideStoreTy, Src0: WideSrcVal).getReg(Idx: 0);
974 } else if (NeedRotate) {
975 assert(WideStoreTy.getSizeInBits() % 2 == 0 &&
976 "Unexpected type for rotate");
977 auto RotAmt =
978 Builder.buildConstant(Res: WideStoreTy, Val: WideStoreTy.getSizeInBits() / 2);
979 WideSrcVal =
980 Builder.buildRotateRight(Dst: WideStoreTy, Src: WideSrcVal, Amt: RotAmt).getReg(Idx: 0);
981 }
982
983 Builder.buildStore(Val: WideSrcVal, Addr: LowestIdxStore->getPointerReg(),
984 PtrInfo: LowestIdxStore->getMMO().getPointerInfo(),
985 Alignment: LowestIdxStore->getMMO().getAlign());
986
987 // Erase the old stores.
988 for (auto *ST : FoundStores) {
989 ST->eraseFromParent();
990 DeletedStores.insert(Ptr: ST);
991 }
992 return true;
993}
994
995bool LoadStoreOptImpl::mergeTruncStoresBlock(MachineBasicBlock &BB) {
996 bool Changed = false;
997 SmallVector<GStore *, 16> Stores;
998 SmallPtrSet<GStore *, 8> DeletedStores;
999 // Walk up the block so we can see the most eligible stores.
1000 for (MachineInstr &MI : llvm::reverse(C&: BB))
1001 if (auto *StoreMI = dyn_cast<GStore>(Val: &MI))
1002 Stores.emplace_back(Args&: StoreMI);
1003
1004 for (auto *StoreMI : Stores) {
1005 if (DeletedStores.count(Ptr: StoreMI))
1006 continue;
1007 if (mergeTruncStore(StoreMI&: *StoreMI, DeletedStores))
1008 Changed = true;
1009 }
1010 return Changed;
1011}
1012
1013bool LoadStoreOptImpl::mergeFunctionStores(MachineFunction &MF) {
1014 bool Changed = false;
1015 for (auto &BB : MF){
1016 Changed |= mergeBlockStores(MBB&: BB);
1017 Changed |= mergeTruncStoresBlock(BB);
1018 }
1019
1020 // Erase all dead instructions left over by the merging.
1021 if (Changed) {
1022 for (auto &BB : MF) {
1023 for (auto &I : make_early_inc_range(Range: reverse(C&: BB))) {
1024 if (isTriviallyDead(MI: I, MRI: *MRI))
1025 I.eraseFromParent();
1026 }
1027 }
1028 }
1029
1030 return Changed;
1031}
1032
1033void LoadStoreOptImpl::initializeStoreMergeTargetInfo(unsigned AddrSpace) {
1034 // Query the legalizer info to record what store types are legal.
1035 // We record this because we don't want to bother trying to merge stores into
1036 // illegal ones, which would just result in being split again.
1037
1038 if (LegalStoreSizes.count(Val: AddrSpace)) {
1039 assert(LegalStoreSizes[AddrSpace].any());
1040 return; // Already cached sizes for this address space.
1041 }
1042
1043 // Need to reserve at least MaxStoreSizeToForm + 1 bits.
1044 BitVector LegalSizes(MaxStoreSizeToForm * 2);
1045 const auto &LI = *MF->getSubtarget().getLegalizerInfo();
1046 const auto &DL = MF->getFunction().getDataLayout();
1047 Type *IRPtrTy = PointerType::get(C&: MF->getFunction().getContext(), AddressSpace: AddrSpace);
1048 LLT PtrTy = getLLTForType(Ty&: *IRPtrTy, DL);
1049 // We assume that we're not going to be generating any stores wider than
1050 // MaxStoreSizeToForm bits for now.
1051 for (unsigned Size = 2; Size <= MaxStoreSizeToForm; Size *= 2) {
1052 LLT Ty = LLT::scalar(SizeInBits: Size);
1053 SmallVector<LegalityQuery::MemDesc, 2> MemDescrs(
1054 {{Ty, Ty.getSizeInBits(), AtomicOrdering::NotAtomic,
1055 AtomicOrdering::NotAtomic}});
1056 SmallVector<LLT> StoreTys({Ty, PtrTy});
1057 LegalityQuery Q(TargetOpcode::G_STORE, StoreTys, MemDescrs);
1058 LegalizeActionStep ActionStep = LI.getAction(Query: Q);
1059 if (ActionStep.Action == LegalizeActions::Legal)
1060 LegalSizes.set(Size);
1061 }
1062 assert(LegalSizes.any() && "Expected some store sizes to be legal!");
1063 LegalStoreSizes[AddrSpace] = std::move(LegalSizes);
1064}
1065
1066bool LoadStoreOptImpl::runOnMachineFunction(
1067 MachineFunction &MF, function_ref<AliasAnalysis *()> GetAA) {
1068 // If the ISel pipeline failed, do not bother running that pass.
1069 if (MF.getProperties().hasFailedISel())
1070 return false;
1071
1072 LLVM_DEBUG(dbgs() << "Begin memory optimizations for: " << MF.getName()
1073 << '\n');
1074
1075 init(MF, GetAA);
1076 bool Changed = false;
1077 Changed |= mergeFunctionStores(MF);
1078
1079 LegalStoreSizes.clear();
1080 return Changed;
1081}
1082
1083bool LoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
1084 LoadStoreOptImpl Impl;
1085 return Impl.runOnMachineFunction(MF, GetAA: [&]() {
1086 return &getAnalysis<AAResultsWrapperPass>().getAAResults();
1087 });
1088}
1089
1090PreservedAnalyses LoadStoreOptPass::run(MachineFunction &MF,
1091 MachineFunctionAnalysisManager &MFAM) {
1092 MFPropsModifier<LoadStoreOptPass> _(*this, MF);
1093 LoadStoreOptImpl Impl;
1094 Impl.runOnMachineFunction(MF, GetAA: [&]() {
1095 FunctionAnalysisManager &FAM =
1096 MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
1097 .getManager();
1098 return &FAM.getResult<AAManager>(IR&: MF.getFunction());
1099 });
1100 return PreservedAnalyses::all();
1101}
1102