1//===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs load / store related peephole
10// optimizations. This pass should be run after register allocation.
11//
12// The pass runs after the PrologEpilogInserter where we emit the CFI
13// instructions. In order to preserve the correctness of the unwind information,
14// the pass should not change the order of any two instructions, one of which
15// has the FrameSetup/FrameDestroy flag or, alternatively, apply an add-hoc fix
16// to unwind information.
17//
18//===----------------------------------------------------------------------===//
19
20#include "AArch64InstrInfo.h"
21#include "AArch64MachineFunctionInfo.h"
22#include "AArch64Subtarget.h"
23#include "MCTargetDesc/AArch64AddressingModes.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineFunctionPass.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineOperand.h"
36#include "llvm/CodeGen/MachineRegisterInfo.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCDwarf.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/CommandLine.h"
43#include "llvm/Support/Debug.h"
44#include "llvm/Support/DebugCounter.h"
45#include "llvm/Support/ErrorHandling.h"
46#include <cassert>
47#include <cstdint>
48#include <functional>
49#include <iterator>
50#include <limits>
51#include <optional>
52
53using namespace llvm;
54
55#define DEBUG_TYPE "aarch64-ldst-opt"
56
57STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
58STATISTIC(NumPostFolded, "Number of post-index updates folded");
59STATISTIC(NumPreFolded, "Number of pre-index updates folded");
60STATISTIC(NumUnscaledPairCreated,
61 "Number of load/store from unscaled generated");
62STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
63STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
64STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation "
65 "not passed the alignment check");
66STATISTIC(NumConstOffsetFolded,
67 "Number of const offset of index address folded");
68STATISTIC(NumUMOVFoldedToFPRStore,
69 "Number of UMOV + GPR stores folded to FPR stores");
70
71DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
72 "Controls which pairs are considered for renaming");
73
74// The LdStLimit limits how far we search for load/store pairs.
75static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit",
76 cl::init(Val: 20), cl::Hidden);
77
78// The UpdateLimit limits how far we search for update instructions when we form
79// pre-/post-index instructions.
80static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(Val: 100),
81 cl::Hidden);
82
83// The LdStConstLimit limits how far we search for const offset instructions
84// when we form index address load/store instructions.
85static cl::opt<unsigned> LdStConstLimit("aarch64-load-store-const-scan-limit",
86 cl::init(Val: 10), cl::Hidden);
87
88// The UMOVFoldLimit limits how far back we scan from a GPR store to find a
89// UMOV that can be folded into a direct FPR store.
90static cl::opt<unsigned> UMOVFoldLimit("aarch64-umov-fold-scan-limit",
91 cl::init(Val: 16), cl::Hidden);
92
93// Enable register renaming to find additional store pairing opportunities.
94static cl::opt<bool> EnableRenaming("aarch64-load-store-renaming",
95 cl::init(Val: true), cl::Hidden);
96
97#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
98
99namespace {
100
101using LdStPairFlags = struct LdStPairFlags {
102 // If a matching instruction is found, MergeForward is set to true if the
103 // merge is to remove the first instruction and replace the second with
104 // a pair-wise insn, and false if the reverse is true.
105 bool MergeForward = false;
106
107 // SExtIdx gives the index of the result of the load pair that must be
108 // extended. The value of SExtIdx assumes that the paired load produces the
109 // value in this order: (I, returned iterator), i.e., -1 means no value has
110 // to be extended, 0 means I, and 1 means the returned iterator.
111 int SExtIdx = -1;
112
113 // If not none, RenameReg can be used to rename the result register of the
114 // first store in a pair. Currently this only works when merging stores
115 // forward.
116 std::optional<MCPhysReg> RenameReg;
117
118 LdStPairFlags() = default;
119
120 void setMergeForward(bool V = true) { MergeForward = V; }
121 bool getMergeForward() const { return MergeForward; }
122
123 void setSExtIdx(int V) { SExtIdx = V; }
124 int getSExtIdx() const { return SExtIdx; }
125
126 void setRenameReg(MCPhysReg R) { RenameReg = R; }
127 void clearRenameReg() { RenameReg = std::nullopt; }
128 std::optional<MCPhysReg> getRenameReg() const { return RenameReg; }
129};
130
131struct AArch64LoadStoreOpt {
132 AliasAnalysis *AA;
133 const AArch64InstrInfo *TII;
134 const TargetRegisterInfo *TRI;
135 const AArch64Subtarget *Subtarget;
136
137 // Track which register units have been modified and used.
138 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
139 LiveRegUnits DefinedInBB;
140
141 // Scan the instructions looking for a load/store that can be combined
142 // with the current instruction into a load/store pair.
143 // Return the matching instruction if one is found, else MBB->end().
144 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
145 LdStPairFlags &Flags,
146 unsigned Limit,
147 bool FindNarrowMerge);
148
149 // Scan the instructions looking for a store that writes to the address from
150 // which the current load instruction reads. Return true if one is found.
151 bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
152 MachineBasicBlock::iterator &StoreI);
153
154 // Merge the two instructions indicated into a wider narrow store instruction.
155 MachineBasicBlock::iterator
156 mergeNarrowZeroStores(MachineBasicBlock::iterator I,
157 MachineBasicBlock::iterator MergeMI,
158 const LdStPairFlags &Flags);
159
160 // Merge the two instructions indicated into a single pair-wise instruction.
161 MachineBasicBlock::iterator
162 mergePairedInsns(MachineBasicBlock::iterator I,
163 MachineBasicBlock::iterator Paired,
164 const LdStPairFlags &Flags);
165
166 // Promote the load that reads directly from the address stored to.
167 MachineBasicBlock::iterator
168 promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
169 MachineBasicBlock::iterator StoreI);
170
171 // Scan the instruction list to find a base register update that can
172 // be combined with the current instruction (a load or store) using
173 // pre or post indexed addressing with writeback. Scan forwards.
174 MachineBasicBlock::iterator
175 findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
176 int UnscaledOffset, unsigned Limit);
177
178 // Scan the instruction list to find a register assigned with a const
179 // value that can be combined with the current instruction (a load or store)
180 // using base addressing with writeback. Scan backwards.
181 MachineBasicBlock::iterator
182 findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit,
183 unsigned &Offset);
184
185 // Scan the instruction list to find a base register update that can
186 // be combined with the current instruction (a load or store) using
187 // pre or post indexed addressing with writeback. Scan backwards.
188 // `MergeEither` is set to true if the combined instruction may be placed
189 // either at the location of the load/store instruction or at the location of
190 // the update instruction.
191 MachineBasicBlock::iterator
192 findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit,
193 bool &MergeEither);
194
195 // Find an instruction that updates the base register of the ld/st
196 // instruction.
197 bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
198 unsigned BaseReg, int Offset);
199
200 bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI,
201 unsigned IndexReg, unsigned &Offset);
202
203 // Merge a pre- or post-index base register update into a ld/st instruction.
204 std::optional<MachineBasicBlock::iterator>
205 mergeUpdateInsn(MachineBasicBlock::iterator I,
206 MachineBasicBlock::iterator Update, bool IsForward,
207 bool IsPreIdx, bool MergeEither);
208
209 MachineBasicBlock::iterator
210 mergeConstOffsetInsn(MachineBasicBlock::iterator I,
211 MachineBasicBlock::iterator Update, unsigned Offset,
212 int Scale);
213
214 // Find and merge zero store instructions.
215 bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
216
217 // Find and pair ldr/str instructions.
218 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
219
220 // Find and promote load instructions which read directly from store.
221 bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
222
223 // Find and merge a base register updates before or after a ld/st instruction.
224 bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
225
226 // Find and merge an index ldr/st instruction into a base ld/st instruction.
227 bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale);
228
229 // Replace a UMOV (lane 0) + GPR store with a direct FPR sub-register store.
230 bool tryToReplaceUMOVStore(MachineBasicBlock::iterator &MBBI);
231
232 bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
233
234 bool runOnMachineFunction(MachineFunction &MF);
235};
236
237struct AArch64LoadStoreOptLegacy : public MachineFunctionPass {
238 static char ID;
239
240 AArch64LoadStoreOptLegacy() : MachineFunctionPass(ID) {}
241
242 bool runOnMachineFunction(MachineFunction &Fn) override;
243
244 void getAnalysisUsage(AnalysisUsage &AU) const override {
245 AU.addRequired<AAResultsWrapperPass>();
246 MachineFunctionPass::getAnalysisUsage(AU);
247 }
248
249 MachineFunctionProperties getRequiredProperties() const override {
250 return MachineFunctionProperties().setNoVRegs();
251 }
252
253 StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; }
254};
255
256char AArch64LoadStoreOptLegacy::ID = 0;
257
258} // end anonymous namespace
259
260INITIALIZE_PASS(AArch64LoadStoreOptLegacy, "aarch64-ldst-opt",
261 AARCH64_LOAD_STORE_OPT_NAME, false, false)
262
263static bool isNarrowStore(unsigned Opc) {
264 switch (Opc) {
265 default:
266 return false;
267 case AArch64::STRBBui:
268 case AArch64::STURBBi:
269 case AArch64::STRHHui:
270 case AArch64::STURHHi:
271 return true;
272 }
273}
274
275// These instruction set memory tag and either keep memory contents unchanged or
276// set it to zero, ignoring the address part of the source register.
277static bool isTagStore(const MachineInstr &MI) {
278 switch (MI.getOpcode()) {
279 default:
280 return false;
281 case AArch64::STGi:
282 case AArch64::STZGi:
283 case AArch64::ST2Gi:
284 case AArch64::STZ2Gi:
285 return true;
286 }
287}
288
289static unsigned getMatchingNonSExtOpcode(unsigned Opc,
290 bool *IsValidLdStrOpc = nullptr) {
291 if (IsValidLdStrOpc)
292 *IsValidLdStrOpc = true;
293 switch (Opc) {
294 default:
295 if (IsValidLdStrOpc)
296 *IsValidLdStrOpc = false;
297 return std::numeric_limits<unsigned>::max();
298 case AArch64::STRDui:
299 case AArch64::STURDi:
300 case AArch64::STRDpre:
301 case AArch64::STRQui:
302 case AArch64::STURQi:
303 case AArch64::STRQpre:
304 case AArch64::STRBBui:
305 case AArch64::STURBBi:
306 case AArch64::STRHHui:
307 case AArch64::STURHHi:
308 case AArch64::STRWui:
309 case AArch64::STRWpre:
310 case AArch64::STURWi:
311 case AArch64::STRXui:
312 case AArch64::STRXpre:
313 case AArch64::STURXi:
314 case AArch64::STR_ZXI:
315 case AArch64::LDRDui:
316 case AArch64::LDURDi:
317 case AArch64::LDRDpre:
318 case AArch64::LDRQui:
319 case AArch64::LDURQi:
320 case AArch64::LDRQpre:
321 case AArch64::LDRWui:
322 case AArch64::LDURWi:
323 case AArch64::LDRWpre:
324 case AArch64::LDRXui:
325 case AArch64::LDURXi:
326 case AArch64::LDRXpre:
327 case AArch64::STRSui:
328 case AArch64::STURSi:
329 case AArch64::STRSpre:
330 case AArch64::LDRSui:
331 case AArch64::LDURSi:
332 case AArch64::LDRSpre:
333 case AArch64::LDR_ZXI:
334 return Opc;
335 case AArch64::LDRSWui:
336 return AArch64::LDRWui;
337 case AArch64::LDURSWi:
338 return AArch64::LDURWi;
339 case AArch64::LDRSWpre:
340 return AArch64::LDRWpre;
341 }
342}
343
344static unsigned getMatchingWideOpcode(unsigned Opc) {
345 switch (Opc) {
346 default:
347 llvm_unreachable("Opcode has no wide equivalent!");
348 case AArch64::STRBBui:
349 return AArch64::STRHHui;
350 case AArch64::STRHHui:
351 return AArch64::STRWui;
352 case AArch64::STURBBi:
353 return AArch64::STURHHi;
354 case AArch64::STURHHi:
355 return AArch64::STURWi;
356 case AArch64::STURWi:
357 return AArch64::STURXi;
358 case AArch64::STRWui:
359 return AArch64::STRXui;
360 }
361}
362
363static unsigned getMatchingPairOpcode(unsigned Opc) {
364 switch (Opc) {
365 default:
366 llvm_unreachable("Opcode has no pairwise equivalent!");
367 case AArch64::STRSui:
368 case AArch64::STURSi:
369 return AArch64::STPSi;
370 case AArch64::STRSpre:
371 return AArch64::STPSpre;
372 case AArch64::STRDui:
373 case AArch64::STURDi:
374 return AArch64::STPDi;
375 case AArch64::STRDpre:
376 return AArch64::STPDpre;
377 case AArch64::STRQui:
378 case AArch64::STURQi:
379 case AArch64::STR_ZXI:
380 return AArch64::STPQi;
381 case AArch64::STRQpre:
382 return AArch64::STPQpre;
383 case AArch64::STRWui:
384 case AArch64::STURWi:
385 return AArch64::STPWi;
386 case AArch64::STRWpre:
387 return AArch64::STPWpre;
388 case AArch64::STRXui:
389 case AArch64::STURXi:
390 return AArch64::STPXi;
391 case AArch64::STRXpre:
392 return AArch64::STPXpre;
393 case AArch64::LDRSui:
394 case AArch64::LDURSi:
395 return AArch64::LDPSi;
396 case AArch64::LDRSpre:
397 return AArch64::LDPSpre;
398 case AArch64::LDRDui:
399 case AArch64::LDURDi:
400 return AArch64::LDPDi;
401 case AArch64::LDRDpre:
402 return AArch64::LDPDpre;
403 case AArch64::LDRQui:
404 case AArch64::LDURQi:
405 case AArch64::LDR_ZXI:
406 return AArch64::LDPQi;
407 case AArch64::LDRQpre:
408 return AArch64::LDPQpre;
409 case AArch64::LDRWui:
410 case AArch64::LDURWi:
411 return AArch64::LDPWi;
412 case AArch64::LDRWpre:
413 return AArch64::LDPWpre;
414 case AArch64::LDRXui:
415 case AArch64::LDURXi:
416 return AArch64::LDPXi;
417 case AArch64::LDRXpre:
418 return AArch64::LDPXpre;
419 case AArch64::LDRSWui:
420 case AArch64::LDURSWi:
421 return AArch64::LDPSWi;
422 case AArch64::LDRSWpre:
423 return AArch64::LDPSWpre;
424 }
425}
426
427static unsigned isMatchingStore(MachineInstr &LoadInst,
428 MachineInstr &StoreInst) {
429 unsigned LdOpc = LoadInst.getOpcode();
430 unsigned StOpc = StoreInst.getOpcode();
431 switch (LdOpc) {
432 default:
433 llvm_unreachable("Unsupported load instruction!");
434 case AArch64::LDRBBui:
435 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
436 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
437 case AArch64::LDURBBi:
438 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
439 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
440 case AArch64::LDRHHui:
441 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
442 StOpc == AArch64::STRXui;
443 case AArch64::LDURHHi:
444 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
445 StOpc == AArch64::STURXi;
446 case AArch64::LDRWui:
447 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
448 case AArch64::LDURWi:
449 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
450 case AArch64::LDRXui:
451 return StOpc == AArch64::STRXui;
452 case AArch64::LDURXi:
453 return StOpc == AArch64::STURXi;
454 }
455}
456
457static unsigned getPreIndexedOpcode(unsigned Opc) {
458 // FIXME: We don't currently support creating pre-indexed loads/stores when
459 // the load or store is the unscaled version. If we decide to perform such an
460 // optimization in the future the cases for the unscaled loads/stores will
461 // need to be added here.
462 switch (Opc) {
463 default:
464 llvm_unreachable("Opcode has no pre-indexed equivalent!");
465 case AArch64::STRBui:
466 return AArch64::STRBpre;
467 case AArch64::STRHui:
468 return AArch64::STRHpre;
469 case AArch64::STRSui:
470 return AArch64::STRSpre;
471 case AArch64::STRDui:
472 return AArch64::STRDpre;
473 case AArch64::STRQui:
474 return AArch64::STRQpre;
475 case AArch64::STRBBui:
476 return AArch64::STRBBpre;
477 case AArch64::STRHHui:
478 return AArch64::STRHHpre;
479 case AArch64::STRWui:
480 return AArch64::STRWpre;
481 case AArch64::STRXui:
482 return AArch64::STRXpre;
483 case AArch64::LDRBui:
484 return AArch64::LDRBpre;
485 case AArch64::LDRHui:
486 return AArch64::LDRHpre;
487 case AArch64::LDRSui:
488 return AArch64::LDRSpre;
489 case AArch64::LDRDui:
490 return AArch64::LDRDpre;
491 case AArch64::LDRQui:
492 return AArch64::LDRQpre;
493 case AArch64::LDRBBui:
494 return AArch64::LDRBBpre;
495 case AArch64::LDRHHui:
496 return AArch64::LDRHHpre;
497 case AArch64::LDRWui:
498 return AArch64::LDRWpre;
499 case AArch64::LDRXui:
500 return AArch64::LDRXpre;
501 case AArch64::LDRSWui:
502 return AArch64::LDRSWpre;
503 case AArch64::LDPSi:
504 return AArch64::LDPSpre;
505 case AArch64::LDPSWi:
506 return AArch64::LDPSWpre;
507 case AArch64::LDPDi:
508 return AArch64::LDPDpre;
509 case AArch64::LDPQi:
510 return AArch64::LDPQpre;
511 case AArch64::LDPWi:
512 return AArch64::LDPWpre;
513 case AArch64::LDPXi:
514 return AArch64::LDPXpre;
515 case AArch64::STPSi:
516 return AArch64::STPSpre;
517 case AArch64::STPDi:
518 return AArch64::STPDpre;
519 case AArch64::STPQi:
520 return AArch64::STPQpre;
521 case AArch64::STPWi:
522 return AArch64::STPWpre;
523 case AArch64::STPXi:
524 return AArch64::STPXpre;
525 case AArch64::STGi:
526 return AArch64::STGPreIndex;
527 case AArch64::STZGi:
528 return AArch64::STZGPreIndex;
529 case AArch64::ST2Gi:
530 return AArch64::ST2GPreIndex;
531 case AArch64::STZ2Gi:
532 return AArch64::STZ2GPreIndex;
533 case AArch64::STGPi:
534 return AArch64::STGPpre;
535 }
536}
537
538static unsigned getBaseAddressOpcode(unsigned Opc) {
539 // TODO: Add more index address stores.
540 switch (Opc) {
541 default:
542 llvm_unreachable("Opcode has no base address equivalent!");
543 case AArch64::LDRBroX:
544 return AArch64::LDRBui;
545 case AArch64::LDRBBroX:
546 return AArch64::LDRBBui;
547 case AArch64::LDRSBXroX:
548 return AArch64::LDRSBXui;
549 case AArch64::LDRSBWroX:
550 return AArch64::LDRSBWui;
551 case AArch64::LDRHroX:
552 return AArch64::LDRHui;
553 case AArch64::LDRHHroX:
554 return AArch64::LDRHHui;
555 case AArch64::LDRSHXroX:
556 return AArch64::LDRSHXui;
557 case AArch64::LDRSHWroX:
558 return AArch64::LDRSHWui;
559 case AArch64::LDRWroX:
560 return AArch64::LDRWui;
561 case AArch64::LDRSroX:
562 return AArch64::LDRSui;
563 case AArch64::LDRSWroX:
564 return AArch64::LDRSWui;
565 case AArch64::LDRDroX:
566 return AArch64::LDRDui;
567 case AArch64::LDRXroX:
568 return AArch64::LDRXui;
569 case AArch64::LDRQroX:
570 return AArch64::LDRQui;
571 }
572}
573
574static unsigned getPostIndexedOpcode(unsigned Opc) {
575 switch (Opc) {
576 default:
577 llvm_unreachable("Opcode has no post-indexed wise equivalent!");
578 case AArch64::STRBui:
579 return AArch64::STRBpost;
580 case AArch64::STRHui:
581 return AArch64::STRHpost;
582 case AArch64::STRSui:
583 case AArch64::STURSi:
584 return AArch64::STRSpost;
585 case AArch64::STRDui:
586 case AArch64::STURDi:
587 return AArch64::STRDpost;
588 case AArch64::STRQui:
589 case AArch64::STURQi:
590 return AArch64::STRQpost;
591 case AArch64::STRBBui:
592 return AArch64::STRBBpost;
593 case AArch64::STRHHui:
594 return AArch64::STRHHpost;
595 case AArch64::STRWui:
596 case AArch64::STURWi:
597 return AArch64::STRWpost;
598 case AArch64::STRXui:
599 case AArch64::STURXi:
600 return AArch64::STRXpost;
601 case AArch64::LDRBui:
602 return AArch64::LDRBpost;
603 case AArch64::LDRHui:
604 return AArch64::LDRHpost;
605 case AArch64::LDRSui:
606 case AArch64::LDURSi:
607 return AArch64::LDRSpost;
608 case AArch64::LDRDui:
609 case AArch64::LDURDi:
610 return AArch64::LDRDpost;
611 case AArch64::LDRQui:
612 case AArch64::LDURQi:
613 return AArch64::LDRQpost;
614 case AArch64::LDRBBui:
615 return AArch64::LDRBBpost;
616 case AArch64::LDRHHui:
617 return AArch64::LDRHHpost;
618 case AArch64::LDRWui:
619 case AArch64::LDURWi:
620 return AArch64::LDRWpost;
621 case AArch64::LDRXui:
622 case AArch64::LDURXi:
623 return AArch64::LDRXpost;
624 case AArch64::LDRSWui:
625 return AArch64::LDRSWpost;
626 case AArch64::LDPSi:
627 return AArch64::LDPSpost;
628 case AArch64::LDPSWi:
629 return AArch64::LDPSWpost;
630 case AArch64::LDPDi:
631 return AArch64::LDPDpost;
632 case AArch64::LDPQi:
633 return AArch64::LDPQpost;
634 case AArch64::LDPWi:
635 return AArch64::LDPWpost;
636 case AArch64::LDPXi:
637 return AArch64::LDPXpost;
638 case AArch64::STPSi:
639 return AArch64::STPSpost;
640 case AArch64::STPDi:
641 return AArch64::STPDpost;
642 case AArch64::STPQi:
643 return AArch64::STPQpost;
644 case AArch64::STPWi:
645 return AArch64::STPWpost;
646 case AArch64::STPXi:
647 return AArch64::STPXpost;
648 case AArch64::STGi:
649 return AArch64::STGPostIndex;
650 case AArch64::STZGi:
651 return AArch64::STZGPostIndex;
652 case AArch64::ST2Gi:
653 return AArch64::ST2GPostIndex;
654 case AArch64::STZ2Gi:
655 return AArch64::STZ2GPostIndex;
656 case AArch64::STGPi:
657 return AArch64::STGPpost;
658 }
659}
660
661static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI) {
662
663 unsigned OpcA = FirstMI.getOpcode();
664 unsigned OpcB = MI.getOpcode();
665
666 switch (OpcA) {
667 default:
668 return false;
669 case AArch64::STRSpre:
670 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
671 case AArch64::STRDpre:
672 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
673 case AArch64::STRQpre:
674 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
675 case AArch64::STRWpre:
676 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
677 case AArch64::STRXpre:
678 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
679 case AArch64::LDRSpre:
680 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
681 case AArch64::LDRDpre:
682 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
683 case AArch64::LDRQpre:
684 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
685 case AArch64::LDRWpre:
686 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
687 case AArch64::LDRXpre:
688 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
689 case AArch64::LDRSWpre:
690 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
691 }
692}
693
694// Returns the scale and offset range of pre/post indexed variants of MI.
695static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
696 int &MinOffset, int &MaxOffset) {
697 bool IsPaired = AArch64InstrInfo::isPairedLdSt(MI);
698 bool IsTagStore = isTagStore(MI);
699 // ST*G and all paired ldst have the same scale in pre/post-indexed variants
700 // as in the "unsigned offset" variant.
701 // All other pre/post indexed ldst instructions are unscaled.
702 Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
703
704 if (IsPaired) {
705 MinOffset = -64;
706 MaxOffset = 63;
707 } else {
708 MinOffset = -256;
709 MaxOffset = 255;
710 }
711}
712
713static MachineOperand &getLdStRegOp(MachineInstr &MI,
714 unsigned PairedRegOp = 0) {
715 assert(PairedRegOp < 2 && "Unexpected register operand idx.");
716 bool IsPreLdSt = AArch64InstrInfo::isPreLdSt(MI);
717 if (IsPreLdSt)
718 PairedRegOp += 1;
719 unsigned Idx =
720 AArch64InstrInfo::isPairedLdSt(MI) || IsPreLdSt ? PairedRegOp : 0;
721 return MI.getOperand(i: Idx);
722}
723
724static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst,
725 MachineInstr &StoreInst,
726 const AArch64InstrInfo *TII) {
727 assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
728 int LoadSize = TII->getMemScale(MI: LoadInst);
729 int StoreSize = TII->getMemScale(MI: StoreInst);
730 int UnscaledStOffset =
731 TII->hasUnscaledLdStOffset(MI&: StoreInst)
732 ? AArch64InstrInfo::getLdStOffsetOp(MI: StoreInst).getImm()
733 : AArch64InstrInfo::getLdStOffsetOp(MI: StoreInst).getImm() * StoreSize;
734 int UnscaledLdOffset =
735 TII->hasUnscaledLdStOffset(MI&: LoadInst)
736 ? AArch64InstrInfo::getLdStOffsetOp(MI: LoadInst).getImm()
737 : AArch64InstrInfo::getLdStOffsetOp(MI: LoadInst).getImm() * LoadSize;
738 return (UnscaledStOffset <= UnscaledLdOffset) &&
739 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
740}
741
742static bool isPromotableZeroStoreInst(MachineInstr &MI) {
743 unsigned Opc = MI.getOpcode();
744 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
745 isNarrowStore(Opc)) &&
746 getLdStRegOp(MI).getReg() == AArch64::WZR;
747}
748
749static bool isPromotableLoadFromStore(MachineInstr &MI) {
750 switch (MI.getOpcode()) {
751 default:
752 return false;
753 // Scaled instructions.
754 case AArch64::LDRBBui:
755 case AArch64::LDRHHui:
756 case AArch64::LDRWui:
757 case AArch64::LDRXui:
758 // Unscaled instructions.
759 case AArch64::LDURBBi:
760 case AArch64::LDURHHi:
761 case AArch64::LDURWi:
762 case AArch64::LDURXi:
763 return true;
764 }
765}
766
767static bool isMergeableLdStUpdate(MachineInstr &MI, AArch64FunctionInfo &AFI) {
768 unsigned Opc = MI.getOpcode();
769 switch (Opc) {
770 default:
771 return false;
772 // Scaled instructions.
773 case AArch64::STRBui:
774 case AArch64::STRHui:
775 case AArch64::STRSui:
776 case AArch64::STRDui:
777 case AArch64::STRQui:
778 case AArch64::STRXui:
779 case AArch64::STRWui:
780 case AArch64::STRHHui:
781 case AArch64::STRBBui:
782 case AArch64::LDRBui:
783 case AArch64::LDRHui:
784 case AArch64::LDRSui:
785 case AArch64::LDRDui:
786 case AArch64::LDRQui:
787 case AArch64::LDRXui:
788 case AArch64::LDRWui:
789 case AArch64::LDRHHui:
790 case AArch64::LDRBBui:
791 case AArch64::STGi:
792 case AArch64::STZGi:
793 case AArch64::ST2Gi:
794 case AArch64::STZ2Gi:
795 case AArch64::STGPi:
796 // Unscaled instructions.
797 case AArch64::STURSi:
798 case AArch64::STURDi:
799 case AArch64::STURQi:
800 case AArch64::STURWi:
801 case AArch64::STURXi:
802 case AArch64::LDURSi:
803 case AArch64::LDURDi:
804 case AArch64::LDURQi:
805 case AArch64::LDURWi:
806 case AArch64::LDURXi:
807 // Paired instructions.
808 case AArch64::LDPSi:
809 case AArch64::LDPSWi:
810 case AArch64::LDPDi:
811 case AArch64::LDPQi:
812 case AArch64::LDPWi:
813 case AArch64::LDPXi:
814 case AArch64::STPSi:
815 case AArch64::STPDi:
816 case AArch64::STPQi:
817 case AArch64::STPWi:
818 case AArch64::STPXi:
819 // Make sure this is a reg+imm (as opposed to an address reloc).
820 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
821 return false;
822
823 // When using stack tagging, simple sp+imm loads and stores are not
824 // tag-checked, but pre- and post-indexed versions of them are, so we can't
825 // replace the former with the latter. This transformation would be valid
826 // if the load/store accesses an untagged stack slot, but we don't have
827 // that information available after frame indices have been eliminated.
828 if (AFI.isMTETagged() &&
829 AArch64InstrInfo::getLdStBaseOp(MI).getReg() == AArch64::SP)
830 return false;
831
832 return true;
833 }
834}
835
836// Make sure this is a reg+reg Ld/St
837static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
838 unsigned Opc = MI.getOpcode();
839 switch (Opc) {
840 default:
841 return false;
842 // Scaled instructions.
843 // TODO: Add more index address stores.
844 case AArch64::LDRBroX:
845 case AArch64::LDRBBroX:
846 case AArch64::LDRSBXroX:
847 case AArch64::LDRSBWroX:
848 Scale = 1;
849 return true;
850 case AArch64::LDRHroX:
851 case AArch64::LDRHHroX:
852 case AArch64::LDRSHXroX:
853 case AArch64::LDRSHWroX:
854 Scale = 2;
855 return true;
856 case AArch64::LDRWroX:
857 case AArch64::LDRSroX:
858 case AArch64::LDRSWroX:
859 Scale = 4;
860 return true;
861 case AArch64::LDRDroX:
862 case AArch64::LDRXroX:
863 Scale = 8;
864 return true;
865 case AArch64::LDRQroX:
866 Scale = 16;
867 return true;
868 }
869}
870
871static bool isRewritableImplicitDef(const MachineInstr &MI,
872 const MachineOperand &MO) {
873 switch (MI.getOpcode()) {
874 default:
875 return MO.isRenamable();
876 case AArch64::ORRWrs:
877 case AArch64::ADDWri:
878 return true;
879 }
880}
881
882MachineBasicBlock::iterator
883AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
884 MachineBasicBlock::iterator MergeMI,
885 const LdStPairFlags &Flags) {
886 assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
887 "Expected promotable zero stores.");
888
889 MachineBasicBlock::iterator E = I->getParent()->end();
890 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
891 // If NextI is the second of the two instructions to be merged, we need
892 // to skip one further. Either way we merge will invalidate the iterator,
893 // and we don't need to scan the new instruction, as it's a pairwise
894 // instruction, which we're not considering for further action anyway.
895 if (NextI == MergeMI)
896 NextI = next_nodbg(It: NextI, End: E);
897
898 unsigned Opc = I->getOpcode();
899 unsigned MergeMIOpc = MergeMI->getOpcode();
900 bool IsScaled = !TII->hasUnscaledLdStOffset(Opc);
901 bool IsMergedMIScaled = !TII->hasUnscaledLdStOffset(Opc: MergeMIOpc);
902 int OffsetStride = IsScaled ? TII->getMemScale(MI: *I) : 1;
903 int MergeMIOffsetStride = IsMergedMIScaled ? TII->getMemScale(MI: *MergeMI) : 1;
904
905 bool MergeForward = Flags.getMergeForward();
906 // Insert our new paired instruction after whichever of the paired
907 // instructions MergeForward indicates.
908 MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
909 // Also based on MergeForward is from where we copy the base register operand
910 // so we get the flags compatible with the input code.
911 const MachineOperand &BaseRegOp =
912 MergeForward ? AArch64InstrInfo::getLdStBaseOp(MI: *MergeMI)
913 : AArch64InstrInfo::getLdStBaseOp(MI: *I);
914
915 // Which register is Rt and which is Rt2 depends on the offset order.
916 int64_t IOffsetInBytes =
917 AArch64InstrInfo::getLdStOffsetOp(MI: *I).getImm() * OffsetStride;
918 int64_t MIOffsetInBytes =
919 AArch64InstrInfo::getLdStOffsetOp(MI: *MergeMI).getImm() *
920 MergeMIOffsetStride;
921 // Select final offset based on the offset order.
922 int64_t OffsetImm;
923 if (IOffsetInBytes > MIOffsetInBytes)
924 OffsetImm = MIOffsetInBytes;
925 else
926 OffsetImm = IOffsetInBytes;
927
928 int NewOpcode = getMatchingWideOpcode(Opc);
929 // Adjust final offset on scaled stores because the new instruction
930 // has a different scale.
931 if (!TII->hasUnscaledLdStOffset(Opc: NewOpcode)) {
932 int NewOffsetStride = TII->getMemScale(Opc: NewOpcode);
933 assert(((OffsetImm % NewOffsetStride) == 0) &&
934 "Offset should be a multiple of the store memory scale");
935 OffsetImm = OffsetImm / NewOffsetStride;
936 }
937
938 // Construct the new instruction.
939 DebugLoc DL = I->getDebugLoc();
940 MachineBasicBlock *MBB = I->getParent();
941 MachineInstrBuilder MIB;
942 MIB = BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: NewOpcode))
943 .addReg(RegNo: isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
944 .add(MO: BaseRegOp)
945 .addImm(Val: OffsetImm)
946 .cloneMergedMemRefs(OtherMIs: {&*I, &*MergeMI})
947 .setMIFlags(I->mergeFlagsWith(Other: *MergeMI));
948 (void)MIB;
949
950 LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
951 LLVM_DEBUG(I->print(dbgs()));
952 LLVM_DEBUG(dbgs() << " ");
953 LLVM_DEBUG(MergeMI->print(dbgs()));
954 LLVM_DEBUG(dbgs() << " with instruction:\n ");
955 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
956 LLVM_DEBUG(dbgs() << "\n");
957
958 // Erase the old instructions.
959 I->eraseFromParent();
960 MergeMI->eraseFromParent();
961 return NextI;
962}
963
964// Apply Fn to all instructions between MI and the beginning of the block, until
965// a def for DefReg is reached. Returns true, iff Fn returns true for all
966// visited instructions. Stop after visiting Limit iterations.
967static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
968 const TargetRegisterInfo *TRI, unsigned Limit,
969 std::function<bool(MachineInstr &, bool)> &Fn) {
970 auto MBB = MI.getParent();
971 for (MachineInstr &I :
972 instructionsWithoutDebug(It: MI.getReverseIterator(), End: MBB->instr_rend())) {
973 if (!Limit)
974 return false;
975 --Limit;
976
977 bool isDef = any_of(Range: I.operands(), P: [DefReg, TRI](MachineOperand &MOP) {
978 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
979 TRI->regsOverlap(RegA: MOP.getReg(), RegB: DefReg);
980 });
981 if (!Fn(I, isDef))
982 return false;
983 if (isDef)
984 break;
985 }
986 return true;
987}
988
989static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units,
990 const TargetRegisterInfo *TRI) {
991
992 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
993 if (MOP.isReg() && MOP.isKill())
994 Units.removeReg(Reg: MOP.getReg());
995
996 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
997 if (MOP.isReg() && !MOP.isKill())
998 Units.addReg(Reg: MOP.getReg());
999}
1000
1001/// This function will add a new entry into the debugValueSubstitutions table
1002/// when two instruction have been merged into a new one represented by \p
1003/// MergedInstr.
1004static void addDebugSubstitutionsToTable(MachineFunction *MF,
1005 unsigned InstrNumToSet,
1006 MachineInstr &OriginalInstr,
1007 MachineInstr &MergedInstr) {
1008
1009 // Figure out the Operand Index of the destination register of the
1010 // OriginalInstr in the new MergedInstr.
1011 auto Reg = OriginalInstr.getOperand(i: 0).getReg();
1012 unsigned OperandNo = 0;
1013 bool RegFound = false;
1014 for (const auto Op : MergedInstr.operands()) {
1015 if (Op.getReg() == Reg) {
1016 RegFound = true;
1017 break;
1018 }
1019 OperandNo++;
1020 }
1021
1022 if (RegFound)
1023 MF->makeDebugValueSubstitution({OriginalInstr.peekDebugInstrNum(), 0},
1024 {InstrNumToSet, OperandNo});
1025}
1026
1027MachineBasicBlock::iterator
1028AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
1029 MachineBasicBlock::iterator Paired,
1030 const LdStPairFlags &Flags) {
1031 MachineBasicBlock::iterator E = I->getParent()->end();
1032 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
1033 // If NextI is the second of the two instructions to be merged, we need
1034 // to skip one further. Either way we merge will invalidate the iterator,
1035 // and we don't need to scan the new instruction, as it's a pairwise
1036 // instruction, which we're not considering for further action anyway.
1037 if (NextI == Paired)
1038 NextI = next_nodbg(It: NextI, End: E);
1039
1040 int SExtIdx = Flags.getSExtIdx();
1041 unsigned Opc =
1042 SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(Opc: I->getOpcode());
1043 bool IsUnscaled = TII->hasUnscaledLdStOffset(Opc);
1044 int OffsetStride = IsUnscaled ? TII->getMemScale(MI: *I) : 1;
1045
1046 bool MergeForward = Flags.getMergeForward();
1047
1048 std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
1049 if (RenameReg) {
1050 MCRegister RegToRename = getLdStRegOp(MI&: *I).getReg();
1051 DefinedInBB.addReg(Reg: *RenameReg);
1052
1053 // Return the sub/super register for RenameReg, matching the size of
1054 // OriginalReg.
1055 auto GetMatchingSubReg =
1056 [this, RenameReg](const TargetRegisterClass *C) -> MCPhysReg {
1057 for (MCPhysReg SubOrSuper :
1058 TRI->sub_and_superregs_inclusive(Reg: *RenameReg)) {
1059 if (C->contains(Reg: SubOrSuper))
1060 return SubOrSuper;
1061 }
1062 llvm_unreachable("Should have found matching sub or super register!");
1063 };
1064
1065 std::function<bool(MachineInstr &, bool)> UpdateMIs =
1066 [this, RegToRename, GetMatchingSubReg, MergeForward](MachineInstr &MI,
1067 bool IsDef) {
1068 if (IsDef) {
1069 bool SeenDef = false;
1070 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1071 MachineOperand &MOP = MI.getOperand(i: OpIdx);
1072 // Rename the first explicit definition and all implicit
1073 // definitions matching RegToRename.
1074 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1075 (!MergeForward || !SeenDef ||
1076 (MOP.isDef() && MOP.isImplicit())) &&
1077 TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename)) {
1078 assert((MOP.isImplicit() ||
1079 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1080 "Need renamable operands");
1081 Register MatchingReg;
1082 if (const TargetRegisterClass *RC =
1083 MI.getRegClassConstraint(OpIdx, TII, TRI))
1084 MatchingReg = GetMatchingSubReg(RC);
1085 else {
1086 if (!isRewritableImplicitDef(MI, MO: MOP))
1087 continue;
1088 MatchingReg = GetMatchingSubReg(
1089 TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1090 }
1091 MOP.setReg(MatchingReg);
1092 SeenDef = true;
1093 }
1094 }
1095 } else {
1096 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1097 MachineOperand &MOP = MI.getOperand(i: OpIdx);
1098 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1099 TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename)) {
1100 assert((MOP.isImplicit() ||
1101 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1102 "Need renamable operands");
1103 Register MatchingReg;
1104 if (const TargetRegisterClass *RC =
1105 MI.getRegClassConstraint(OpIdx, TII, TRI))
1106 MatchingReg = GetMatchingSubReg(RC);
1107 else
1108 MatchingReg = GetMatchingSubReg(
1109 TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1110 assert(MatchingReg != AArch64::NoRegister &&
1111 "Cannot find matching regs for renaming");
1112 MOP.setReg(MatchingReg);
1113 }
1114 }
1115 }
1116 LLVM_DEBUG(dbgs() << "Renamed " << MI);
1117 return true;
1118 };
1119 forAllMIsUntilDef(MI&: MergeForward ? *I : *Paired->getPrevNode(), DefReg: RegToRename,
1120 TRI, UINT32_MAX, Fn&: UpdateMIs);
1121
1122#if !defined(NDEBUG)
1123 // For forward merging store:
1124 // Make sure the register used for renaming is not used between the
1125 // paired instructions. That would trash the content before the new
1126 // paired instruction.
1127 MCPhysReg RegToCheck = *RenameReg;
1128 // For backward merging load:
1129 // Make sure the register being renamed is not used between the
1130 // paired instructions. That would trash the content after the new
1131 // paired instruction.
1132 if (!MergeForward)
1133 RegToCheck = RegToRename;
1134 for (auto &MI :
1135 iterator_range<MachineInstrBundleIterator<llvm::MachineInstr>>(
1136 MergeForward ? std::next(I) : I,
1137 MergeForward ? std::next(Paired) : Paired))
1138 assert(all_of(MI.operands(),
1139 [this, RegToCheck](const MachineOperand &MOP) {
1140 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1141 MOP.isUndef() ||
1142 !TRI->regsOverlap(MOP.getReg(), RegToCheck);
1143 }) &&
1144 "Rename register used between paired instruction, trashing the "
1145 "content");
1146#endif
1147 }
1148
1149 // Insert our new paired instruction after whichever of the paired
1150 // instructions MergeForward indicates.
1151 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
1152 // Also based on MergeForward is from where we copy the base register operand
1153 // so we get the flags compatible with the input code.
1154 const MachineOperand &BaseRegOp =
1155 MergeForward ? AArch64InstrInfo::getLdStBaseOp(MI: *Paired)
1156 : AArch64InstrInfo::getLdStBaseOp(MI: *I);
1157
1158 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: *I).getImm();
1159 int PairedOffset = AArch64InstrInfo::getLdStOffsetOp(MI: *Paired).getImm();
1160 bool PairedIsUnscaled = TII->hasUnscaledLdStOffset(Opc: Paired->getOpcode());
1161 if (IsUnscaled != PairedIsUnscaled) {
1162 // We're trying to pair instructions that differ in how they are scaled. If
1163 // I is scaled then scale the offset of Paired accordingly. Otherwise, do
1164 // the opposite (i.e., make Paired's offset unscaled).
1165 int MemSize = TII->getMemScale(MI: *Paired);
1166 if (PairedIsUnscaled) {
1167 // If the unscaled offset isn't a multiple of the MemSize, we can't
1168 // pair the operations together.
1169 assert(!(PairedOffset % TII->getMemScale(*Paired)) &&
1170 "Offset should be a multiple of the stride!");
1171 PairedOffset /= MemSize;
1172 } else {
1173 PairedOffset *= MemSize;
1174 }
1175 }
1176
1177 // Which register is Rt and which is Rt2 depends on the offset order.
1178 // However, for pre load/stores the Rt should be the one of the pre
1179 // load/store.
1180 MachineInstr *RtMI, *Rt2MI;
1181 if (Offset == PairedOffset + OffsetStride &&
1182 !AArch64InstrInfo::isPreLdSt(MI: *I)) {
1183 RtMI = &*Paired;
1184 Rt2MI = &*I;
1185 // Here we swapped the assumption made for SExtIdx.
1186 // I.e., we turn ldp I, Paired into ldp Paired, I.
1187 // Update the index accordingly.
1188 if (SExtIdx != -1)
1189 SExtIdx = (SExtIdx + 1) % 2;
1190 } else {
1191 RtMI = &*I;
1192 Rt2MI = &*Paired;
1193 }
1194 int OffsetImm = AArch64InstrInfo::getLdStOffsetOp(MI: *RtMI).getImm();
1195 // Scale the immediate offset, if necessary.
1196 if (TII->hasUnscaledLdStOffset(Opc: RtMI->getOpcode())) {
1197 assert(!(OffsetImm % TII->getMemScale(*RtMI)) &&
1198 "Unscaled offset cannot be scaled.");
1199 OffsetImm /= TII->getMemScale(MI: *RtMI);
1200 }
1201
1202 // Construct the new instruction.
1203 MachineInstrBuilder MIB;
1204 DebugLoc DL = I->getDebugLoc();
1205 MachineBasicBlock *MBB = I->getParent();
1206 MachineOperand RegOp0 = getLdStRegOp(MI&: *RtMI);
1207 MachineOperand RegOp1 = getLdStRegOp(MI&: *Rt2MI);
1208 MachineOperand &PairedRegOp = RtMI == &*Paired ? RegOp0 : RegOp1;
1209 // Kill flags may become invalid when moving stores for pairing.
1210 if (RegOp0.isUse()) {
1211 if (!MergeForward) {
1212 // Clear kill flags on store if moving upwards. Example:
1213 // STRWui kill %w0, ...
1214 // USE %w1
1215 // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards
1216 // We are about to move the store of w1, so its kill flag may become
1217 // invalid; not the case for w0.
1218 // Since w1 is used between the stores, the kill flag on w1 is cleared
1219 // after merging.
1220 // STPWi kill %w0, %w1, ...
1221 // USE %w1
1222 for (auto It = std::next(x: I); It != Paired && PairedRegOp.isKill(); ++It)
1223 if (It->readsRegister(Reg: PairedRegOp.getReg(), TRI))
1224 PairedRegOp.setIsKill(false);
1225 } else {
1226 // Clear kill flags of the first stores register. Example:
1227 // STRWui %w1, ...
1228 // USE kill %w1 ; need to clear kill flag when moving STRWui downwards
1229 // STRW %w0
1230 Register Reg = getLdStRegOp(MI&: *I).getReg();
1231 for (MachineInstr &MI :
1232 make_range(x: std::next(x: I->getIterator()), y: Paired->getIterator()))
1233 MI.clearRegisterKills(Reg, RegInfo: TRI);
1234 }
1235 }
1236
1237 unsigned int MatchPairOpcode = getMatchingPairOpcode(Opc);
1238 MIB = BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: MatchPairOpcode));
1239
1240 // Adds the pre-index operand for pre-indexed ld/st pairs.
1241 if (AArch64InstrInfo::isPreLdSt(MI: *RtMI))
1242 MIB.addReg(RegNo: BaseRegOp.getReg(), Flags: RegState::Define);
1243
1244 MIB.add(MO: RegOp0)
1245 .add(MO: RegOp1)
1246 .add(MO: BaseRegOp)
1247 .addImm(Val: OffsetImm)
1248 .cloneMergedMemRefs(OtherMIs: {&*I, &*Paired})
1249 .setMIFlags(I->mergeFlagsWith(Other: *Paired));
1250
1251 (void)MIB;
1252
1253 LLVM_DEBUG(
1254 dbgs() << "Creating pair load/store. Replacing instructions:\n ");
1255 LLVM_DEBUG(I->print(dbgs()));
1256 LLVM_DEBUG(dbgs() << " ");
1257 LLVM_DEBUG(Paired->print(dbgs()));
1258 LLVM_DEBUG(dbgs() << " with instruction:\n ");
1259 if (SExtIdx != -1) {
1260 // Generate the sign extension for the proper result of the ldp.
1261 // I.e., with X1, that would be:
1262 // %w1 = KILL %w1, implicit-def %x1
1263 // %x1 = SBFMXri killed %x1, 0, 31
1264 MachineOperand &DstMO = MIB->getOperand(i: SExtIdx);
1265 // Right now, DstMO has the extended register, since it comes from an
1266 // extended opcode.
1267 Register DstRegX = DstMO.getReg();
1268 // Get the W variant of that register.
1269 Register DstRegW = TRI->getSubReg(Reg: DstRegX, Idx: AArch64::sub_32);
1270 // Update the result of LDP to use the W instead of the X variant.
1271 DstMO.setReg(DstRegW);
1272 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1273 LLVM_DEBUG(dbgs() << "\n");
1274 // Make the machine verifier happy by providing a definition for
1275 // the X register.
1276 // Insert this definition right after the generated LDP, i.e., before
1277 // InsertionPoint.
1278 MachineInstrBuilder MIBKill =
1279 BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::KILL), DestReg: DstRegW)
1280 .addReg(RegNo: DstRegW)
1281 .addReg(RegNo: DstRegX, Flags: RegState::Define);
1282 MIBKill->getOperand(i: 2).setImplicit();
1283 // Create the sign extension.
1284 MachineInstrBuilder MIBSXTW =
1285 BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::SBFMXri), DestReg: DstRegX)
1286 .addReg(RegNo: DstRegX)
1287 .addImm(Val: 0)
1288 .addImm(Val: 31);
1289 (void)MIBSXTW;
1290
1291 // In the case of a sign-extend, where we have something like:
1292 // debugValueSubstitutions:[]
1293 // $w1 = LDRWui $x0, 1, debug-instr-number 1
1294 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1295 // $x0 = LDRSWui $x0, 0, debug-instr-number 2
1296 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1297
1298 // It will be converted to:
1299 // debugValueSubstitutions:[]
1300 // $w0, $w1 = LDPWi $x0, 0
1301 // $w0 = KILL $w0, implicit-def $x0
1302 // $x0 = SBFMXri $x0, 0, 31
1303 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1304 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1305
1306 // We want the final result to look like:
1307 // debugValueSubstitutions:
1308 // - { srcinst: 1, srcop: 0, dstinst: 4, dstop: 1, subreg: 0 }
1309 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1310 // $w0, $w1 = LDPWi $x0, 0, debug-instr-number 4
1311 // $w0 = KILL $w0, implicit-def $x0
1312 // $x0 = SBFMXri $x0, 0, 31, debug-instr-number 3
1313 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1314 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1315
1316 // $x0 is where the final value is stored, so the sign extend (SBFMXri)
1317 // instruction contains the final value we care about we give it a new
1318 // debug-instr-number 3. Whereas, $w1 contains the final value that we care
1319 // about, therefore the LDP instruction is also given a new
1320 // debug-instr-number 4. We have to add these substitutions to the
1321 // debugValueSubstitutions table. However, we also have to ensure that the
1322 // OpIndex that pointed to debug-instr-number 1 gets updated to 1, because
1323 // $w1 is the second operand of the LDP instruction.
1324
1325 if (I->peekDebugInstrNum()) {
1326 // If I is the instruction which got sign extended and has a
1327 // debug-instr-number, give the SBFMXri instruction a new
1328 // debug-instr-number, and update the debugValueSubstitutions table with
1329 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1330 // instruction a new debug-instr-number, and update the
1331 // debugValueSubstitutions table with the new debug-instr-number and
1332 // OpIndex pair.
1333 unsigned NewInstrNum;
1334 if (DstRegX == I->getOperand(i: 0).getReg()) {
1335 NewInstrNum = MIBSXTW->getDebugInstrNum();
1336 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *I,
1337 MergedInstr&: *MIBSXTW);
1338 } else {
1339 NewInstrNum = MIB->getDebugInstrNum();
1340 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *I, MergedInstr&: *MIB);
1341 }
1342 }
1343 if (Paired->peekDebugInstrNum()) {
1344 // If Paired is the instruction which got sign extended and has a
1345 // debug-instr-number, give the SBFMXri instruction a new
1346 // debug-instr-number, and update the debugValueSubstitutions table with
1347 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1348 // instruction a new debug-instr-number, and update the
1349 // debugValueSubstitutions table with the new debug-instr-number and
1350 // OpIndex pair.
1351 unsigned NewInstrNum;
1352 if (DstRegX == Paired->getOperand(i: 0).getReg()) {
1353 NewInstrNum = MIBSXTW->getDebugInstrNum();
1354 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *Paired,
1355 MergedInstr&: *MIBSXTW);
1356 } else {
1357 NewInstrNum = MIB->getDebugInstrNum();
1358 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *Paired,
1359 MergedInstr&: *MIB);
1360 }
1361 }
1362
1363 LLVM_DEBUG(dbgs() << " Extend operand:\n ");
1364 LLVM_DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs()));
1365 } else if (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI) {
1366 // We are combining SVE fill/spill to LDP/STP, so we need to use the Q
1367 // variant of the registers.
1368 MachineOperand &MOp0 = MIB->getOperand(i: 0);
1369 MachineOperand &MOp1 = MIB->getOperand(i: 1);
1370 assert(AArch64::ZPRRegClass.contains(MOp0.getReg()) &&
1371 AArch64::ZPRRegClass.contains(MOp1.getReg()) && "Invalid register.");
1372 MOp0.setReg(AArch64::Q0 + (MOp0.getReg() - AArch64::Z0));
1373 MOp1.setReg(AArch64::Q0 + (MOp1.getReg() - AArch64::Z0));
1374 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1375 } else {
1376
1377 // In the case that the merge doesn't result in a sign-extend, if we have
1378 // something like:
1379 // debugValueSubstitutions:[]
1380 // $x1 = LDRXui $x0, 1, debug-instr-number 1
1381 // DBG_INSTR_REF !13, dbg-instr-ref(1, 0), debug-location !11
1382 // $x0 = LDRXui killed $x0, 0, debug-instr-number 2
1383 // DBG_INSTR_REF !14, dbg-instr-ref(2, 0), debug-location !11
1384
1385 // It will be converted to:
1386 // debugValueSubstitutions: []
1387 // $x0, $x1 = LDPXi $x0, 0
1388 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1389 // DBG_INSTR_REF !13, dbg-instr-ref(2, 0), debug-location !14
1390
1391 // We want the final result to look like:
1392 // debugValueSubstitutions:
1393 // - { srcinst: 1, srcop: 0, dstinst: 3, dstop: 1, subreg: 0 }
1394 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1395 // $x0, $x1 = LDPXi $x0, 0, debug-instr-number 3
1396 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1397 // DBG_INSTR_REF !12, dbg-instr-ref(2, 0), debug-location !14
1398
1399 // Here all that needs to be done is, that the LDP instruction needs to be
1400 // updated with a new debug-instr-number, we then need to add entries into
1401 // the debugSubstitutions table to map the old instr-refs to the new ones.
1402
1403 // Assign new DebugInstrNum to the Paired instruction.
1404 if (I->peekDebugInstrNum()) {
1405 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1406 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewDebugInstrNum, OriginalInstr&: *I,
1407 MergedInstr&: *MIB);
1408 }
1409 if (Paired->peekDebugInstrNum()) {
1410 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1411 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewDebugInstrNum, OriginalInstr&: *Paired,
1412 MergedInstr&: *MIB);
1413 }
1414
1415 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1416 }
1417 LLVM_DEBUG(dbgs() << "\n");
1418
1419 if (MergeForward)
1420 for (const MachineOperand &MOP : phys_regs_and_masks(MI: *I))
1421 if (MOP.isReg() && MOP.isKill())
1422 DefinedInBB.addReg(Reg: MOP.getReg());
1423
1424 // Copy over any implicit-def operands. This is like MI.copyImplicitOps, but
1425 // only copies implicit defs and makes sure that each operand is only added
1426 // once in case of duplicates.
1427 auto CopyImplicitOps = [&](MachineBasicBlock::iterator MI1,
1428 MachineBasicBlock::iterator MI2) {
1429 SmallSetVector<Register, 4> Ops;
1430 for (const MachineOperand &MO :
1431 llvm::drop_begin(RangeOrContainer: MI1->operands(), N: MI1->getDesc().getNumOperands()))
1432 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1433 Ops.insert(X: MO.getReg());
1434 for (const MachineOperand &MO :
1435 llvm::drop_begin(RangeOrContainer: MI2->operands(), N: MI2->getDesc().getNumOperands()))
1436 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1437 Ops.insert(X: MO.getReg());
1438 for (auto Op : Ops)
1439 MIB.addDef(RegNo: Op, Flags: RegState::Implicit);
1440 };
1441 CopyImplicitOps(I, Paired);
1442
1443 // Erase the old instructions.
1444 I->eraseFromParent();
1445 Paired->eraseFromParent();
1446
1447 return NextI;
1448}
1449
1450MachineBasicBlock::iterator
1451AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
1452 MachineBasicBlock::iterator StoreI) {
1453 MachineBasicBlock::iterator NextI =
1454 next_nodbg(It: LoadI, End: LoadI->getParent()->end());
1455
1456 int LoadSize = TII->getMemScale(MI: *LoadI);
1457 int StoreSize = TII->getMemScale(MI: *StoreI);
1458 Register LdRt = getLdStRegOp(MI&: *LoadI).getReg();
1459 const MachineOperand &StMO = getLdStRegOp(MI&: *StoreI);
1460 Register StRt = getLdStRegOp(MI&: *StoreI).getReg();
1461 bool IsStoreXReg = TRI->getRegClass(i: AArch64::GPR64RegClassID)->contains(Reg: StRt);
1462
1463 assert((IsStoreXReg ||
1464 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1465 "Unexpected RegClass");
1466
1467 MachineInstr *BitExtMI;
1468 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1469 // Remove the load, if the destination register of the loads is the same
1470 // register for stored value.
1471 if (StRt == LdRt && LoadSize == 8) {
1472 for (MachineInstr &MI : make_range(x: StoreI->getIterator(),
1473 y: LoadI->getIterator())) {
1474 if (MI.killsRegister(Reg: StRt, TRI)) {
1475 MI.clearRegisterKills(Reg: StRt, RegInfo: TRI);
1476 break;
1477 }
1478 }
1479 LLVM_DEBUG(dbgs() << "Remove load instruction:\n ");
1480 LLVM_DEBUG(LoadI->print(dbgs()));
1481 LLVM_DEBUG(dbgs() << "\n");
1482 LoadI->eraseFromParent();
1483 return NextI;
1484 }
1485 // Replace the load with a mov if the load and store are in the same size.
1486 BitExtMI =
1487 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1488 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), DestReg: LdRt)
1489 .addReg(RegNo: IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1490 .add(MO: StMO)
1491 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
1492 .setMIFlags(LoadI->getFlags());
1493 } else {
1494 // FIXME: Currently we disable this transformation in big-endian targets as
1495 // performance and correctness are verified only in little-endian.
1496 if (!Subtarget->isLittleEndian())
1497 return NextI;
1498 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI&: *LoadI);
1499 assert(IsUnscaled == TII->hasUnscaledLdStOffset(*StoreI) &&
1500 "Unsupported ld/st match");
1501 assert(LoadSize <= StoreSize && "Invalid load size");
1502 int UnscaledLdOffset =
1503 IsUnscaled
1504 ? AArch64InstrInfo::getLdStOffsetOp(MI: *LoadI).getImm()
1505 : AArch64InstrInfo::getLdStOffsetOp(MI: *LoadI).getImm() * LoadSize;
1506 int UnscaledStOffset =
1507 IsUnscaled
1508 ? AArch64InstrInfo::getLdStOffsetOp(MI: *StoreI).getImm()
1509 : AArch64InstrInfo::getLdStOffsetOp(MI: *StoreI).getImm() * StoreSize;
1510 int Width = LoadSize * 8;
1511 Register DestReg =
1512 IsStoreXReg ? Register(TRI->getMatchingSuperReg(
1513 Reg: LdRt, SubIdx: AArch64::sub_32, RC: &AArch64::GPR64RegClass))
1514 : LdRt;
1515
1516 assert((UnscaledLdOffset >= UnscaledStOffset &&
1517 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1518 "Invalid offset");
1519
1520 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1521 int Imms = Immr + Width - 1;
1522 if (UnscaledLdOffset == UnscaledStOffset) {
1523 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N
1524 | ((Immr) << 6) // immr
1525 | ((Imms) << 0) // imms
1526 ;
1527
1528 BitExtMI =
1529 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1530 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1531 DestReg)
1532 .add(MO: StMO)
1533 .addImm(Val: AndMaskEncoded)
1534 .setMIFlags(LoadI->getFlags());
1535 } else if (IsStoreXReg && Imms == 31) {
1536 // Use the 32 bit variant of UBFM if it's the LSR alias of the
1537 // instruction.
1538 assert(Immr <= Imms && "Expected LSR alias of UBFM");
1539 BitExtMI = BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1540 MCID: TII->get(Opcode: AArch64::UBFMWri),
1541 DestReg: TRI->getSubReg(Reg: DestReg, Idx: AArch64::sub_32))
1542 .addReg(RegNo: TRI->getSubReg(Reg: StRt, Idx: AArch64::sub_32))
1543 .addImm(Val: Immr)
1544 .addImm(Val: Imms)
1545 .setMIFlags(LoadI->getFlags());
1546 } else {
1547 BitExtMI =
1548 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1549 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1550 DestReg)
1551 .add(MO: StMO)
1552 .addImm(Val: Immr)
1553 .addImm(Val: Imms)
1554 .setMIFlags(LoadI->getFlags());
1555 }
1556 }
1557
1558 // Clear kill flags between store and load.
1559 for (MachineInstr &MI : make_range(x: StoreI->getIterator(),
1560 y: BitExtMI->getIterator()))
1561 if (MI.killsRegister(Reg: StRt, TRI)) {
1562 MI.clearRegisterKills(Reg: StRt, RegInfo: TRI);
1563 break;
1564 }
1565
1566 LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
1567 LLVM_DEBUG(StoreI->print(dbgs()));
1568 LLVM_DEBUG(dbgs() << " ");
1569 LLVM_DEBUG(LoadI->print(dbgs()));
1570 LLVM_DEBUG(dbgs() << " with instructions:\n ");
1571 LLVM_DEBUG(StoreI->print(dbgs()));
1572 LLVM_DEBUG(dbgs() << " ");
1573 LLVM_DEBUG((BitExtMI)->print(dbgs()));
1574 LLVM_DEBUG(dbgs() << "\n");
1575
1576 // Erase the old instructions.
1577 LoadI->eraseFromParent();
1578 return NextI;
1579}
1580
1581static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
1582 // Convert the byte-offset used by unscaled into an "element" offset used
1583 // by the scaled pair load/store instructions.
1584 if (IsUnscaled) {
1585 // If the byte-offset isn't a multiple of the stride, there's no point
1586 // trying to match it.
1587 if (Offset % OffsetStride)
1588 return false;
1589 Offset /= OffsetStride;
1590 }
1591 return Offset <= 63 && Offset >= -64;
1592}
1593
1594// Do alignment, specialized to power of 2 and for signed ints,
1595// avoiding having to do a C-style cast from uint_64t to int when
1596// using alignTo from include/llvm/Support/MathExtras.h.
1597// FIXME: Move this function to include/MathExtras.h?
1598static int alignTo(int Num, int PowOf2) {
1599 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1600}
1601
1602static bool mayAlias(MachineInstr &MIa,
1603 SmallVectorImpl<MachineInstr *> &MemInsns,
1604 AliasAnalysis *AA) {
1605 for (MachineInstr *MIb : MemInsns) {
1606 if (MIa.mayAlias(AA, Other: *MIb, /*UseTBAA*/ false)) {
1607 LLVM_DEBUG(dbgs() << "Aliasing with: "; MIb->dump());
1608 return true;
1609 }
1610 }
1611
1612 LLVM_DEBUG(dbgs() << "No aliases found\n");
1613 return false;
1614}
1615
1616bool AArch64LoadStoreOpt::findMatchingStore(
1617 MachineBasicBlock::iterator I, unsigned Limit,
1618 MachineBasicBlock::iterator &StoreI) {
1619 MachineBasicBlock::iterator B = I->getParent()->begin();
1620 MachineBasicBlock::iterator MBBI = I;
1621 MachineInstr &LoadMI = *I;
1622 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: LoadMI).getReg();
1623
1624 // If the load is the first instruction in the block, there's obviously
1625 // not any matching store.
1626 if (MBBI == B)
1627 return false;
1628
1629 // Track which register units have been modified and used between the first
1630 // insn and the second insn.
1631 ModifiedRegUnits.clear();
1632 UsedRegUnits.clear();
1633
1634 unsigned Count = 0;
1635 do {
1636 MBBI = prev_nodbg(It: MBBI, Begin: B);
1637 MachineInstr &MI = *MBBI;
1638
1639 // Don't count transient instructions towards the search limit since there
1640 // may be different numbers of them if e.g. debug information is present.
1641 if (!MI.isTransient())
1642 ++Count;
1643
1644 // If the load instruction reads directly from the address to which the
1645 // store instruction writes and the stored value is not modified, we can
1646 // promote the load. Since we do not handle stores with pre-/post-index,
1647 // it's unnecessary to check if BaseReg is modified by the store itself.
1648 // Also we can't handle stores without an immediate offset operand,
1649 // while the operand might be the address for a global variable.
1650 if (MI.mayStore() && isMatchingStore(LoadInst&: LoadMI, StoreInst&: MI) &&
1651 BaseReg == AArch64InstrInfo::getLdStBaseOp(MI).getReg() &&
1652 AArch64InstrInfo::getLdStOffsetOp(MI).isImm() &&
1653 isLdOffsetInRangeOfSt(LoadInst&: LoadMI, StoreInst&: MI, TII) &&
1654 ModifiedRegUnits.available(Reg: getLdStRegOp(MI).getReg())) {
1655 StoreI = MBBI;
1656 return true;
1657 }
1658
1659 if (MI.isCall())
1660 return false;
1661
1662 // Update modified / uses register units.
1663 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1664
1665 // Otherwise, if the base register is modified, we have no match, so
1666 // return early.
1667 if (!ModifiedRegUnits.available(Reg: BaseReg))
1668 return false;
1669
1670 // If we encounter a store aliased with the load, return early.
1671 if (MI.mayStore() && LoadMI.mayAlias(AA, Other: MI, /*UseTBAA*/ false))
1672 return false;
1673 } while (MBBI != B && Count < Limit);
1674 return false;
1675}
1676
1677static bool needsWinCFI(const MachineFunction *MF) {
1678 return MF->getTarget().getMCAsmInfo().usesWindowsCFI() &&
1679 MF->getFunction().needsUnwindTableEntry();
1680}
1681
1682// Returns true if FirstMI and MI are candidates for merging or pairing.
1683// Otherwise, returns false.
1684static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
1685 LdStPairFlags &Flags,
1686 const AArch64InstrInfo *TII) {
1687 // If this is volatile or if pairing is suppressed, not a candidate.
1688 if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
1689 return false;
1690
1691 // We should have already checked FirstMI for pair suppression and volatility.
1692 assert(!FirstMI.hasOrderedMemoryRef() &&
1693 !TII->isLdStPairSuppressed(FirstMI) &&
1694 "FirstMI shouldn't get here if either of these checks are true.");
1695
1696 if (needsWinCFI(MF: MI.getMF()) && (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
1697 MI.getFlag(Flag: MachineInstr::FrameDestroy)))
1698 return false;
1699
1700 unsigned OpcA = FirstMI.getOpcode();
1701 unsigned OpcB = MI.getOpcode();
1702
1703 // Opcodes match: If the opcodes are pre ld/st there is nothing more to check.
1704 if (OpcA == OpcB)
1705 return !AArch64InstrInfo::isPreLdSt(MI: FirstMI);
1706
1707 // Bail out if one of the opcodes is SVE fill/spill, as we currently don't
1708 // allow pairing them with other instructions.
1709 if (OpcA == AArch64::LDR_ZXI || OpcA == AArch64::STR_ZXI ||
1710 OpcB == AArch64::LDR_ZXI || OpcB == AArch64::STR_ZXI)
1711 return false;
1712
1713 // Two pre ld/st of different opcodes cannot be merged either
1714 if (AArch64InstrInfo::isPreLdSt(MI: FirstMI) && AArch64InstrInfo::isPreLdSt(MI))
1715 return false;
1716
1717 // Try to match a sign-extended load/store with a zero-extended load/store.
1718 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1719 unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc: OpcA, IsValidLdStrOpc: &IsValidLdStrOpc);
1720 assert(IsValidLdStrOpc &&
1721 "Given Opc should be a Load or Store with an immediate");
1722 // OpcA will be the first instruction in the pair.
1723 if (NonSExtOpc == getMatchingNonSExtOpcode(Opc: OpcB, IsValidLdStrOpc: &PairIsValidLdStrOpc)) {
1724 Flags.setSExtIdx(NonSExtOpc == OpcA ? 1 : 0);
1725 return true;
1726 }
1727
1728 // If the second instruction isn't even a mergable/pairable load/store, bail
1729 // out.
1730 if (!PairIsValidLdStrOpc)
1731 return false;
1732
1733 // Narrow stores do not have a matching pair opcodes, so constrain their
1734 // merging to zero stores.
1735 if (isNarrowStore(Opc: OpcA) || isNarrowStore(Opc: OpcB))
1736 return getLdStRegOp(MI&: FirstMI).getReg() == AArch64::WZR &&
1737 getLdStRegOp(MI).getReg() == AArch64::WZR &&
1738 TII->getMemScale(MI: FirstMI) == TII->getMemScale(MI);
1739
1740 // The STR<S,D,Q,W,X>pre - STR<S,D,Q,W,X>ui and
1741 // LDR<S,D,Q,W,X,SW>pre-LDR<S,D,Q,W,X,SW>ui
1742 // are candidate pairs that can be merged.
1743 if (isPreLdStPairCandidate(FirstMI, MI))
1744 return true;
1745
1746 // Try to match an unscaled load/store with a scaled load/store.
1747 return TII->hasUnscaledLdStOffset(Opc: OpcA) != TII->hasUnscaledLdStOffset(Opc: OpcB) &&
1748 getMatchingPairOpcode(Opc: OpcA) == getMatchingPairOpcode(Opc: OpcB);
1749
1750 // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
1751}
1752
1753static bool canRenameMOP(const MachineInstr &MI, const MachineOperand &MOP,
1754 const TargetRegisterInfo *TRI) {
1755 if (MOP.isReg()) {
1756 auto *RegClass = TRI->getMinimalPhysRegClass(Reg: MOP.getReg());
1757 // Renaming registers with multiple disjunct sub-registers (e.g. the
1758 // result of a LD3) means that all sub-registers are renamed, potentially
1759 // impacting other instructions we did not check. Bail out.
1760 // Note that this relies on the structure of the AArch64 register file. In
1761 // particular, a subregister cannot be written without overwriting the
1762 // whole register.
1763 if (RegClass->HasDisjunctSubRegs && RegClass->CoveredBySubRegs &&
1764 (TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::dsub0) ||
1765 TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::qsub0) ||
1766 TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::zsub0))) {
1767 LLVM_DEBUG(
1768 dbgs()
1769 << " Cannot rename operands with multiple disjunct subregisters ("
1770 << MOP << ")\n");
1771 return false;
1772 }
1773
1774 // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
1775 // them must be known. For example, in ORRWrs the implicit-def
1776 // corresponds to the result register.
1777 if (MOP.isImplicit() && MOP.isDef()) {
1778 if (!isRewritableImplicitDef(MI, MO: MOP))
1779 return false;
1780 return TRI->isSuperOrSubRegisterEq(RegA: MI.getOperand(i: 0).getReg(),
1781 RegB: MOP.getReg());
1782 }
1783 }
1784 return MOP.isImplicit() ||
1785 (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
1786}
1787
1788static bool
1789canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
1790 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1791 const TargetRegisterInfo *TRI) {
1792 if (!FirstMI.mayStore())
1793 return false;
1794
1795 // Check if we can find an unused register which we can use to rename
1796 // the register used by the first load/store.
1797
1798 auto RegToRename = getLdStRegOp(MI&: FirstMI).getReg();
1799 // For now, we only rename if the store operand gets killed at the store.
1800 if (!getLdStRegOp(MI&: FirstMI).isKill() &&
1801 !any_of(Range: FirstMI.operands(),
1802 P: [TRI, RegToRename](const MachineOperand &MOP) {
1803 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1804 MOP.isImplicit() && MOP.isKill() &&
1805 TRI->regsOverlap(RegA: RegToRename, RegB: MOP.getReg());
1806 })) {
1807 LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI);
1808 return false;
1809 }
1810
1811 bool FoundDef = false;
1812
1813 // For each instruction between FirstMI and the previous def for RegToRename,
1814 // we
1815 // * check if we can rename RegToRename in this instruction
1816 // * collect the registers used and required register classes for RegToRename.
1817 std::function<bool(MachineInstr &, bool)> CheckMIs = [&](MachineInstr &MI,
1818 bool IsDef) {
1819 LLVM_DEBUG(dbgs() << "Checking " << MI);
1820 // Currently we do not try to rename across frame-setup instructions.
1821 if (MI.getFlag(Flag: MachineInstr::FrameSetup)) {
1822 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1823 << "currently\n");
1824 return false;
1825 }
1826
1827 UsedInBetween.accumulate(MI);
1828
1829 // For a definition, check that we can rename the definition and exit the
1830 // loop.
1831 FoundDef = IsDef;
1832
1833 // For defs, check if we can rename the first def of RegToRename.
1834 if (FoundDef) {
1835 // For some pseudo instructions, we might not generate code in the end
1836 // (e.g. KILL) and we would end up without a correct def for the rename
1837 // register.
1838 // TODO: This might be overly conservative and we could handle those cases
1839 // in multiple ways:
1840 // 1. Insert an extra copy, to materialize the def.
1841 // 2. Skip pseudo-defs until we find an non-pseudo def.
1842 if (MI.isPseudo()) {
1843 LLVM_DEBUG(dbgs() << " Cannot rename pseudo/bundle instruction\n");
1844 return false;
1845 }
1846
1847 for (auto &MOP : MI.operands()) {
1848 if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
1849 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1850 continue;
1851 if (!canRenameMOP(MI, MOP, TRI)) {
1852 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1853 return false;
1854 }
1855 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1856 }
1857 return true;
1858 } else {
1859 for (auto &MOP : MI.operands()) {
1860 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1861 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1862 continue;
1863
1864 if (!canRenameMOP(MI, MOP, TRI)) {
1865 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1866 return false;
1867 }
1868 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1869 }
1870 }
1871 return true;
1872 };
1873
1874 if (!forAllMIsUntilDef(MI&: FirstMI, DefReg: RegToRename, TRI, Limit: LdStLimit, Fn&: CheckMIs))
1875 return false;
1876
1877 if (!FoundDef) {
1878 LLVM_DEBUG(dbgs() << " Did not find definition for register in BB\n");
1879 return false;
1880 }
1881 return true;
1882}
1883
1884// We want to merge the second load into the first by rewriting the usages of
1885// the same reg between first (incl.) and second (excl.). We don't need to care
1886// about any insns before FirstLoad or after SecondLoad.
1887// 1. The second load writes new value into the same reg.
1888// - The renaming is impossible to impact later use of the reg.
1889// - The second load always trash the value written by the first load which
1890// means the reg must be killed before the second load.
1891// 2. The first load must be a def for the same reg so we don't need to look
1892// into anything before it.
1893static bool canRenameUntilSecondLoad(
1894 MachineInstr &FirstLoad, MachineInstr &SecondLoad,
1895 LiveRegUnits &UsedInBetween,
1896 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1897 const TargetRegisterInfo *TRI) {
1898 if (FirstLoad.isPseudo())
1899 return false;
1900
1901 UsedInBetween.accumulate(MI: FirstLoad);
1902 auto RegToRename = getLdStRegOp(MI&: FirstLoad).getReg();
1903 bool Success = std::all_of(
1904 first: FirstLoad.getIterator(), last: SecondLoad.getIterator(),
1905 pred: [&](MachineInstr &MI) {
1906 LLVM_DEBUG(dbgs() << "Checking " << MI);
1907 // Currently we do not try to rename across frame-setup instructions.
1908 if (MI.getFlag(Flag: MachineInstr::FrameSetup)) {
1909 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1910 << "currently\n");
1911 return false;
1912 }
1913
1914 for (auto &MOP : MI.operands()) {
1915 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1916 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1917 continue;
1918 if (!canRenameMOP(MI, MOP, TRI)) {
1919 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1920 return false;
1921 }
1922 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1923 }
1924
1925 return true;
1926 });
1927 return Success;
1928}
1929
1930// Check if we can find a physical register for renaming \p Reg. This register
1931// must:
1932// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
1933// defined registers up to the point where the renamed register will be used,
1934// * not used in \p UsedInBetween; UsedInBetween must contain all accessed
1935// registers in the range the rename register will be used,
1936// * is available in all used register classes (checked using RequiredClasses).
1937static std::optional<MCPhysReg> tryToFindRegisterToRename(
1938 const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB,
1939 LiveRegUnits &UsedInBetween,
1940 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1941 const TargetRegisterInfo *TRI) {
1942 const MachineRegisterInfo &RegInfo = MF.getRegInfo();
1943
1944 // Checks if any sub- or super-register of PR is callee saved.
1945 auto AnySubOrSuperRegCalleePreserved = [&MF, TRI](MCPhysReg PR) {
1946 return any_of(Range: TRI->sub_and_superregs_inclusive(Reg: PR),
1947 P: [&MF, TRI](MCPhysReg SubOrSuper) {
1948 return TRI->isCalleeSavedPhysReg(PhysReg: SubOrSuper, MF);
1949 });
1950 };
1951
1952 // Check if PR or one of its sub- or super-registers can be used for all
1953 // required register classes.
1954 auto CanBeUsedForAllClasses = [&RequiredClasses, TRI](MCPhysReg PR) {
1955 return all_of(Range&: RequiredClasses, P: [PR, TRI](const TargetRegisterClass *C) {
1956 return any_of(
1957 Range: TRI->sub_and_superregs_inclusive(Reg: PR),
1958 P: [C](MCPhysReg SubOrSuper) { return C->contains(Reg: SubOrSuper); });
1959 });
1960 };
1961
1962 auto *RegClass = TRI->getMinimalPhysRegClass(Reg);
1963 for (const MCPhysReg &PR : *RegClass) {
1964 if (DefinedInBB.available(Reg: PR) && UsedInBetween.available(Reg: PR) &&
1965 !RegInfo.isReserved(PhysReg: PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1966 CanBeUsedForAllClasses(PR)) {
1967 DefinedInBB.addReg(Reg: PR);
1968 LLVM_DEBUG(dbgs() << "Found rename register " << printReg(PR, TRI)
1969 << "\n");
1970 return {PR};
1971 }
1972 }
1973 LLVM_DEBUG(dbgs() << "No rename register found from "
1974 << TRI->getRegClassName(RegClass) << "\n");
1975 return std::nullopt;
1976}
1977
1978// For store pairs: returns a register from FirstMI to the beginning of the
1979// block that can be renamed.
1980// For load pairs: returns a register from FirstMI to MI that can be renamed.
1981static std::optional<MCPhysReg> findRenameRegForSameLdStRegPair(
1982 std::optional<bool> MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI,
1983 Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween,
1984 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1985 const TargetRegisterInfo *TRI) {
1986 std::optional<MCPhysReg> RenameReg;
1987 if (!DebugCounter::shouldExecute(Counter&: RegRenamingCounter))
1988 return RenameReg;
1989
1990 auto *RegClass = TRI->getMinimalPhysRegClass(Reg: getLdStRegOp(MI&: FirstMI).getReg());
1991 MachineFunction &MF = *FirstMI.getParent()->getParent();
1992 if (!RegClass || !MF.getRegInfo().tracksLiveness())
1993 return RenameReg;
1994
1995 const bool IsLoad = FirstMI.mayLoad();
1996
1997 if (!MaybeCanRename) {
1998 if (IsLoad)
1999 MaybeCanRename = {canRenameUntilSecondLoad(FirstLoad&: FirstMI, SecondLoad&: MI, UsedInBetween,
2000 RequiredClasses, TRI)};
2001 else
2002 MaybeCanRename = {
2003 canRenameUpToDef(FirstMI, UsedInBetween, RequiredClasses, TRI)};
2004 }
2005
2006 if (*MaybeCanRename) {
2007 RenameReg = tryToFindRegisterToRename(MF, Reg, DefinedInBB, UsedInBetween,
2008 RequiredClasses, TRI);
2009 }
2010 return RenameReg;
2011}
2012
2013/// Scan the instructions looking for a load/store that can be combined with the
2014/// current instruction into a wider equivalent or a load/store pair.
2015MachineBasicBlock::iterator
2016AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
2017 LdStPairFlags &Flags, unsigned Limit,
2018 bool FindNarrowMerge) {
2019 MachineBasicBlock::iterator E = I->getParent()->end();
2020 MachineBasicBlock::iterator MBBI = I;
2021 MachineInstr &FirstMI = *I;
2022 MBBI = next_nodbg(It: MBBI, End: E);
2023
2024 bool MayLoad = FirstMI.mayLoad();
2025 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI&: FirstMI);
2026 Register Reg = getLdStRegOp(MI&: FirstMI).getReg();
2027 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: FirstMI).getReg();
2028 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: FirstMI).getImm();
2029 int OffsetStride = IsUnscaled ? TII->getMemScale(MI: FirstMI) : 1;
2030 bool IsPromotableZeroStore = isPromotableZeroStoreInst(MI&: FirstMI);
2031
2032 std::optional<bool> MaybeCanRename;
2033 if (!EnableRenaming)
2034 MaybeCanRename = {false};
2035
2036 SmallPtrSet<const TargetRegisterClass *, 5> RequiredClasses;
2037 LiveRegUnits UsedInBetween;
2038 UsedInBetween.init(TRI: *TRI);
2039
2040 Flags.clearRenameReg();
2041
2042 // Track which register units have been modified and used between the first
2043 // insn (inclusive) and the second insn.
2044 ModifiedRegUnits.clear();
2045 UsedRegUnits.clear();
2046
2047 // Remember any instructions that read/write memory between FirstMI and MI.
2048 SmallVector<MachineInstr *, 4> MemInsns;
2049
2050 LLVM_DEBUG(dbgs() << "Find match for: "; FirstMI.dump());
2051 for (unsigned Count = 0; MBBI != E && Count < Limit;
2052 MBBI = next_nodbg(It: MBBI, End: E)) {
2053 MachineInstr &MI = *MBBI;
2054 LLVM_DEBUG(dbgs() << "Analysing 2nd insn: "; MI.dump());
2055
2056 UsedInBetween.accumulate(MI);
2057
2058 // Don't count transient instructions towards the search limit since there
2059 // may be different numbers of them if e.g. debug information is present.
2060 if (!MI.isTransient())
2061 ++Count;
2062
2063 Flags.setSExtIdx(-1);
2064 if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
2065 AArch64InstrInfo::getLdStOffsetOp(MI).isImm()) {
2066 assert(MI.mayLoadOrStore() && "Expected memory operation.");
2067 // If we've found another instruction with the same opcode, check to see
2068 // if the base and offset are compatible with our starting instruction.
2069 // These instructions all have scaled immediate operands, so we just
2070 // check for +1/-1. Make sure to check the new instruction offset is
2071 // actually an immediate and not a symbolic reference destined for
2072 // a relocation.
2073 Register MIBaseReg = AArch64InstrInfo::getLdStBaseOp(MI).getReg();
2074 int MIOffset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2075 bool MIIsUnscaled = TII->hasUnscaledLdStOffset(MI);
2076 if (IsUnscaled != MIIsUnscaled) {
2077 // We're trying to pair instructions that differ in how they are scaled.
2078 // If FirstMI is scaled then scale the offset of MI accordingly.
2079 // Otherwise, do the opposite (i.e., make MI's offset unscaled).
2080 int MemSize = TII->getMemScale(MI);
2081 if (MIIsUnscaled) {
2082 // If the unscaled offset isn't a multiple of the MemSize, we can't
2083 // pair the operations together: bail and keep looking.
2084 if (MIOffset % MemSize) {
2085 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2086 UsedRegUnits, TRI);
2087 MemInsns.push_back(Elt: &MI);
2088 continue;
2089 }
2090 MIOffset /= MemSize;
2091 } else {
2092 MIOffset *= MemSize;
2093 }
2094 }
2095
2096 bool IsPreLdSt = isPreLdStPairCandidate(FirstMI, MI);
2097
2098 if (BaseReg == MIBaseReg) {
2099 // If the offset of the second ld/st is not equal to the size of the
2100 // destination register it can’t be paired with a pre-index ld/st
2101 // pair. Additionally if the base reg is used or modified the operations
2102 // can't be paired: bail and keep looking.
2103 if (IsPreLdSt) {
2104 bool IsOutOfBounds = MIOffset != TII->getMemScale(MI);
2105 bool IsBaseRegUsed = !UsedRegUnits.available(
2106 Reg: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2107 bool IsBaseRegModified = !ModifiedRegUnits.available(
2108 Reg: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2109 // If the stored value and the address of the second instruction is
2110 // the same, it needs to be using the updated register and therefore
2111 // it must not be folded.
2112 bool IsMIRegTheSame =
2113 TRI->regsOverlap(RegA: getLdStRegOp(MI).getReg(),
2114 RegB: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2115 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
2116 IsMIRegTheSame) {
2117 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2118 UsedRegUnits, TRI);
2119 MemInsns.push_back(Elt: &MI);
2120 continue;
2121 }
2122 } else {
2123 if ((Offset != MIOffset + OffsetStride) &&
2124 (Offset + OffsetStride != MIOffset)) {
2125 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2126 UsedRegUnits, TRI);
2127 MemInsns.push_back(Elt: &MI);
2128 continue;
2129 }
2130 }
2131
2132 int MinOffset = Offset < MIOffset ? Offset : MIOffset;
2133 if (FindNarrowMerge) {
2134 // If the alignment requirements of the scaled wide load/store
2135 // instruction can't express the offset of the scaled narrow input,
2136 // bail and keep looking. For promotable zero stores, allow only when
2137 // the stored value is the same (i.e., WZR).
2138 if ((!IsUnscaled && alignTo(Num: MinOffset, PowOf2: 2) != MinOffset) ||
2139 (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) {
2140 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2141 UsedRegUnits, TRI);
2142 MemInsns.push_back(Elt: &MI);
2143 continue;
2144 }
2145 } else {
2146 // Pairwise instructions have a 7-bit signed offset field. Single
2147 // insns have a 12-bit unsigned offset field. If the resultant
2148 // immediate offset of merging these instructions is out of range for
2149 // a pairwise instruction, bail and keep looking.
2150 if (!inBoundsForPair(IsUnscaled, Offset: MinOffset, OffsetStride)) {
2151 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2152 UsedRegUnits, TRI);
2153 MemInsns.push_back(Elt: &MI);
2154 LLVM_DEBUG(dbgs() << "Offset doesn't fit in immediate, "
2155 << "keep looking.\n");
2156 continue;
2157 }
2158 // If the alignment requirements of the paired (scaled) instruction
2159 // can't express the offset of the unscaled input, bail and keep
2160 // looking.
2161 if (IsUnscaled && (alignTo(Num: MinOffset, PowOf2: OffsetStride) != MinOffset)) {
2162 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2163 UsedRegUnits, TRI);
2164 MemInsns.push_back(Elt: &MI);
2165 LLVM_DEBUG(dbgs()
2166 << "Offset doesn't fit due to alignment requirements, "
2167 << "keep looking.\n");
2168 continue;
2169 }
2170 }
2171
2172 // If the BaseReg has been modified, then we cannot do the optimization.
2173 // For example, in the following pattern
2174 // ldr x1 [x2]
2175 // ldr x2 [x3]
2176 // ldr x4 [x2, #8],
2177 // the first and third ldr cannot be converted to ldp x1, x4, [x2]
2178 if (!ModifiedRegUnits.available(Reg: BaseReg))
2179 return E;
2180
2181 const bool SameLoadReg = MayLoad && TRI->isSuperOrSubRegisterEq(
2182 RegA: Reg, RegB: getLdStRegOp(MI).getReg());
2183
2184 // If the Rt of the second instruction (destination register of the
2185 // load) was not modified or used between the two instructions and none
2186 // of the instructions between the second and first alias with the
2187 // second, we can combine the second into the first.
2188 bool RtNotModified =
2189 ModifiedRegUnits.available(Reg: getLdStRegOp(MI).getReg());
2190 bool RtNotUsed = !(MI.mayLoad() && !SameLoadReg &&
2191 !UsedRegUnits.available(Reg: getLdStRegOp(MI).getReg()));
2192
2193 LLVM_DEBUG(dbgs() << "Checking, can combine 2nd into 1st insn:\n"
2194 << "Reg '" << getLdStRegOp(MI) << "' not modified: "
2195 << (RtNotModified ? "true" : "false") << "\n"
2196 << "Reg '" << getLdStRegOp(MI) << "' not used: "
2197 << (RtNotUsed ? "true" : "false") << "\n");
2198
2199 if (RtNotModified && RtNotUsed && !mayAlias(MIa&: MI, MemInsns, AA)) {
2200 // For pairs loading into the same reg, try to find a renaming
2201 // opportunity to allow the renaming of Reg between FirstMI and MI
2202 // and combine MI into FirstMI; otherwise bail and keep looking.
2203 if (SameLoadReg) {
2204 std::optional<MCPhysReg> RenameReg =
2205 findRenameRegForSameLdStRegPair(MaybeCanRename, FirstMI, MI,
2206 Reg, DefinedInBB, UsedInBetween,
2207 RequiredClasses, TRI);
2208 if (!RenameReg) {
2209 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2210 UsedRegUnits, TRI);
2211 MemInsns.push_back(Elt: &MI);
2212 LLVM_DEBUG(dbgs() << "Can't find reg for renaming, "
2213 << "keep looking.\n");
2214 continue;
2215 }
2216 Flags.setRenameReg(*RenameReg);
2217 }
2218
2219 Flags.setMergeForward(false);
2220 if (!SameLoadReg)
2221 Flags.clearRenameReg();
2222 return MBBI;
2223 }
2224
2225 // Likewise, if the Rt of the first instruction is not modified or used
2226 // between the two instructions and none of the instructions between the
2227 // first and the second alias with the first, we can combine the first
2228 // into the second.
2229 RtNotModified = !(
2230 MayLoad && !UsedRegUnits.available(Reg: getLdStRegOp(MI&: FirstMI).getReg()));
2231
2232 LLVM_DEBUG(dbgs() << "Checking, can combine 1st into 2nd insn:\n"
2233 << "Reg '" << getLdStRegOp(FirstMI)
2234 << "' not modified: "
2235 << (RtNotModified ? "true" : "false") << "\n");
2236
2237 if (RtNotModified && !mayAlias(MIa&: FirstMI, MemInsns, AA)) {
2238 if (ModifiedRegUnits.available(Reg: getLdStRegOp(MI&: FirstMI).getReg())) {
2239 Flags.setMergeForward(true);
2240 Flags.clearRenameReg();
2241 return MBBI;
2242 }
2243
2244 std::optional<MCPhysReg> RenameReg = findRenameRegForSameLdStRegPair(
2245 MaybeCanRename, FirstMI, MI, Reg, DefinedInBB, UsedInBetween,
2246 RequiredClasses, TRI);
2247 if (RenameReg) {
2248 Flags.setMergeForward(true);
2249 Flags.setRenameReg(*RenameReg);
2250 return MBBI;
2251 }
2252 }
2253 LLVM_DEBUG(dbgs() << "Unable to combine these instructions due to "
2254 << "interference in between, keep looking.\n");
2255 }
2256 }
2257
2258 // If the instruction wasn't a matching load or store. Stop searching if we
2259 // encounter a call instruction that might modify memory.
2260 if (MI.isCall()) {
2261 LLVM_DEBUG(dbgs() << "Found a call, stop looking.\n");
2262 return E;
2263 }
2264
2265 // Update modified / uses register units.
2266 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2267
2268 // Otherwise, if the base register is modified, we have no match, so
2269 // return early.
2270 if (!ModifiedRegUnits.available(Reg: BaseReg)) {
2271 LLVM_DEBUG(dbgs() << "Base reg is modified, stop looking.\n");
2272 return E;
2273 }
2274
2275 // Update list of instructions that read/write memory.
2276 if (MI.mayLoadOrStore())
2277 MemInsns.push_back(Elt: &MI);
2278 }
2279 return E;
2280}
2281
2282static MachineBasicBlock::iterator
2283maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI) {
2284 assert((MI.getOpcode() == AArch64::SUBXri ||
2285 MI.getOpcode() == AArch64::ADDXri) &&
2286 "Expected a register update instruction");
2287 auto End = MI.getParent()->end();
2288 if (MaybeCFI == End ||
2289 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
2290 !(MI.getFlag(Flag: MachineInstr::FrameSetup) ||
2291 MI.getFlag(Flag: MachineInstr::FrameDestroy)) ||
2292 MI.getOperand(i: 0).getReg() != AArch64::SP)
2293 return End;
2294
2295 const MachineFunction &MF = *MI.getParent()->getParent();
2296 unsigned CFIIndex = MaybeCFI->getOperand(i: 0).getCFIIndex();
2297 const MCCFIInstruction &CFI = MF.getFrameInstructions()[CFIIndex];
2298 switch (CFI.getOperation()) {
2299 case MCCFIInstruction::OpDefCfa:
2300 case MCCFIInstruction::OpDefCfaOffset:
2301 return MaybeCFI;
2302 default:
2303 return End;
2304 }
2305}
2306
2307std::optional<MachineBasicBlock::iterator> AArch64LoadStoreOpt::mergeUpdateInsn(
2308 MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update,
2309 bool IsForward, bool IsPreIdx, bool MergeEither) {
2310 assert((Update->getOpcode() == AArch64::ADDXri ||
2311 Update->getOpcode() == AArch64::SUBXri) &&
2312 "Unexpected base register update instruction to merge!");
2313 MachineBasicBlock::iterator E = I->getParent()->end();
2314 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
2315
2316 // If updating the SP and the following instruction is CFA offset related CFI,
2317 // make sure the CFI follows the SP update either by merging at the location
2318 // of the update or by moving the CFI after the merged instruction. If unable
2319 // to do so, bail.
2320 MachineBasicBlock::iterator InsertPt = I;
2321 if (IsForward) {
2322 assert(IsPreIdx);
2323 if (auto CFI = maybeMoveCFI(MI&: *Update, MaybeCFI: next_nodbg(It: Update, End: E)); CFI != E) {
2324 if (MergeEither) {
2325 InsertPt = Update;
2326 } else {
2327 // Take care not to reorder CFIs.
2328 if (std::any_of(first: std::next(x: CFI), last: I, pred: [](const auto &Insn) {
2329 return Insn.getOpcode() == TargetOpcode::CFI_INSTRUCTION;
2330 }))
2331 return std::nullopt;
2332
2333 MachineBasicBlock *MBB = InsertPt->getParent();
2334 MBB->splice(Where: std::next(x: InsertPt), Other: MBB, From: CFI);
2335 }
2336 }
2337 }
2338
2339 // Return the instruction following the merged instruction, which is
2340 // the instruction following our unmerged load. Unless that's the add/sub
2341 // instruction we're merging, in which case it's the one after that.
2342 if (NextI == Update)
2343 NextI = next_nodbg(It: NextI, End: E);
2344
2345 int Value = Update->getOperand(i: 2).getImm();
2346 assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 &&
2347 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
2348 if (Update->getOpcode() == AArch64::SUBXri)
2349 Value = -Value;
2350
2351 unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(Opc: I->getOpcode())
2352 : getPostIndexedOpcode(Opc: I->getOpcode());
2353 MachineInstrBuilder MIB;
2354 int Scale, MinOffset, MaxOffset;
2355 getPrePostIndexedMemOpInfo(MI: *I, Scale, MinOffset, MaxOffset);
2356 if (!AArch64InstrInfo::isPairedLdSt(MI: *I)) {
2357 // Non-paired instruction.
2358 MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
2359 MCID: TII->get(Opcode: NewOpc))
2360 .add(MO: Update->getOperand(i: 0))
2361 .add(MO: getLdStRegOp(MI&: *I))
2362 .add(MO: AArch64InstrInfo::getLdStBaseOp(MI: *I))
2363 .addImm(Val: Value / Scale)
2364 .setMemRefs(I->memoperands())
2365 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2366 } else {
2367 // Paired instruction.
2368 MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
2369 MCID: TII->get(Opcode: NewOpc))
2370 .add(MO: Update->getOperand(i: 0))
2371 .add(MO: getLdStRegOp(MI&: *I, PairedRegOp: 0))
2372 .add(MO: getLdStRegOp(MI&: *I, PairedRegOp: 1))
2373 .add(MO: AArch64InstrInfo::getLdStBaseOp(MI: *I))
2374 .addImm(Val: Value / Scale)
2375 .setMemRefs(I->memoperands())
2376 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2377 }
2378
2379 if (IsPreIdx) {
2380 ++NumPreFolded;
2381 LLVM_DEBUG(dbgs() << "Creating pre-indexed load/store.");
2382 } else {
2383 ++NumPostFolded;
2384 LLVM_DEBUG(dbgs() << "Creating post-indexed load/store.");
2385 }
2386 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2387 LLVM_DEBUG(I->print(dbgs()));
2388 LLVM_DEBUG(dbgs() << " ");
2389 LLVM_DEBUG(Update->print(dbgs()));
2390 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2391 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
2392 LLVM_DEBUG(dbgs() << "\n");
2393
2394 // Erase the old instructions for the block.
2395 I->eraseFromParent();
2396 Update->eraseFromParent();
2397
2398 return NextI;
2399}
2400
2401MachineBasicBlock::iterator
2402AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I,
2403 MachineBasicBlock::iterator Update,
2404 unsigned Offset, int Scale) {
2405 assert((Update->getOpcode() == AArch64::MOVKWi) &&
2406 "Unexpected const mov instruction to merge!");
2407 MachineBasicBlock::iterator E = I->getParent()->end();
2408 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
2409 MachineBasicBlock::iterator PrevI = prev_nodbg(It: Update, Begin: E);
2410 MachineInstr &MemMI = *I;
2411 unsigned Mask = (1 << 12) * Scale - 1;
2412 unsigned Low = Offset & Mask;
2413 unsigned High = Offset - Low;
2414 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2415 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getReg();
2416 MachineInstrBuilder AddMIB, MemMIB;
2417
2418 // Add IndexReg, BaseReg, High (the BaseReg may be SP)
2419 AddMIB =
2420 BuildMI(BB&: *I->getParent(), I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
2421 .addDef(RegNo: IndexReg)
2422 .addUse(RegNo: BaseReg)
2423 .addImm(Val: High >> 12) // shifted value
2424 .addImm(Val: 12); // shift 12
2425 (void)AddMIB;
2426 // Ld/St DestReg, IndexReg, Imm12
2427 unsigned NewOpc = getBaseAddressOpcode(Opc: I->getOpcode());
2428 MemMIB = BuildMI(BB&: *I->getParent(), I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2429 .add(MO: getLdStRegOp(MI&: MemMI))
2430 .add(MO: AArch64InstrInfo::getLdStOffsetOp(MI: MemMI))
2431 .addImm(Val: Low / Scale)
2432 .setMemRefs(I->memoperands())
2433 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2434 (void)MemMIB;
2435
2436 ++NumConstOffsetFolded;
2437 LLVM_DEBUG(dbgs() << "Creating base address load/store.\n");
2438 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2439 LLVM_DEBUG(PrevI->print(dbgs()));
2440 LLVM_DEBUG(dbgs() << " ");
2441 LLVM_DEBUG(Update->print(dbgs()));
2442 LLVM_DEBUG(dbgs() << " ");
2443 LLVM_DEBUG(I->print(dbgs()));
2444 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2445 LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs()));
2446 LLVM_DEBUG(dbgs() << " ");
2447 LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs()));
2448 LLVM_DEBUG(dbgs() << "\n");
2449
2450 // Erase the old instructions for the block.
2451 I->eraseFromParent();
2452 PrevI->eraseFromParent();
2453 Update->eraseFromParent();
2454
2455 return NextI;
2456}
2457
2458bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
2459 MachineInstr &MI,
2460 unsigned BaseReg, int Offset) {
2461 switch (MI.getOpcode()) {
2462 default:
2463 break;
2464 case AArch64::SUBXri:
2465 case AArch64::ADDXri:
2466 // Make sure it's a vanilla immediate operand, not a relocation or
2467 // anything else we can't handle.
2468 if (!MI.getOperand(i: 2).isImm())
2469 break;
2470 // Watch out for 1 << 12 shifted value.
2471 if (AArch64_AM::getShiftValue(Imm: MI.getOperand(i: 3).getImm()))
2472 break;
2473
2474 // The update instruction source and destination register must be the
2475 // same as the load/store base register.
2476 if (MI.getOperand(i: 0).getReg() != BaseReg ||
2477 MI.getOperand(i: 1).getReg() != BaseReg)
2478 break;
2479
2480 int UpdateOffset = MI.getOperand(i: 2).getImm();
2481 if (MI.getOpcode() == AArch64::SUBXri)
2482 UpdateOffset = -UpdateOffset;
2483
2484 // The immediate must be a multiple of the scaling factor of the pre/post
2485 // indexed instruction.
2486 int Scale, MinOffset, MaxOffset;
2487 getPrePostIndexedMemOpInfo(MI: MemMI, Scale, MinOffset, MaxOffset);
2488 if (UpdateOffset % Scale != 0)
2489 break;
2490
2491 // Scaled offset must fit in the instruction immediate.
2492 int ScaledOffset = UpdateOffset / Scale;
2493 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
2494 break;
2495
2496 // If we have a non-zero Offset, we check that it matches the amount
2497 // we're adding to the register.
2498 if (!Offset || Offset == UpdateOffset)
2499 return true;
2500 break;
2501 }
2502 return false;
2503}
2504
2505bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI,
2506 MachineInstr &MI,
2507 unsigned IndexReg,
2508 unsigned &Offset) {
2509 // The update instruction source and destination register must be the
2510 // same as the load/store index register.
2511 if (MI.getOpcode() == AArch64::MOVKWi &&
2512 TRI->isSuperOrSubRegisterEq(RegA: IndexReg, RegB: MI.getOperand(i: 1).getReg())) {
2513
2514 // movz + movk hold a large offset of a Ld/St instruction.
2515 MachineBasicBlock::iterator B = MI.getParent()->begin();
2516 MachineBasicBlock::iterator MBBI = &MI;
2517 // Skip the scene when the MI is the first instruction of a block.
2518 if (MBBI == B)
2519 return false;
2520 MBBI = prev_nodbg(It: MBBI, Begin: B);
2521 MachineInstr &MovzMI = *MBBI;
2522 // Make sure the MOVKWi and MOVZWi set the same register.
2523 if (MovzMI.getOpcode() == AArch64::MOVZWi &&
2524 MovzMI.getOperand(i: 0).getReg() == MI.getOperand(i: 0).getReg()) {
2525 unsigned Low = MovzMI.getOperand(i: 1).getImm();
2526 unsigned High = MI.getOperand(i: 2).getImm() << MI.getOperand(i: 3).getImm();
2527 Offset = High + Low;
2528 // 12-bit optionally shifted immediates are legal for adds.
2529 return Offset >> 24 == 0;
2530 }
2531 }
2532 return false;
2533}
2534
2535MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
2536 MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
2537 MachineBasicBlock::iterator E = I->getParent()->end();
2538 MachineInstr &MemMI = *I;
2539 MachineBasicBlock::iterator MBBI = I;
2540
2541 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2542 int MIUnscaledOffset = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getImm() *
2543 TII->getMemScale(MI: MemMI);
2544
2545 // Scan forward looking for post-index opportunities. Updating instructions
2546 // can't be formed if the memory instruction doesn't have the offset we're
2547 // looking for.
2548 if (MIUnscaledOffset != UnscaledOffset)
2549 return E;
2550
2551 // If the base register overlaps a source/destination register, we can't
2552 // merge the update. This does not apply to tag store instructions which
2553 // ignore the address part of the source register.
2554 // This does not apply to STGPi as well, which does not have unpredictable
2555 // behavior in this case unlike normal stores, and always performs writeback
2556 // after reading the source register value.
2557 if (!isTagStore(MI: MemMI) && MemMI.getOpcode() != AArch64::STGPi) {
2558 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MI: MemMI);
2559 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
2560 Register DestReg = getLdStRegOp(MI&: MemMI, PairedRegOp: i).getReg();
2561 if (DestReg == BaseReg || TRI->isSubRegister(RegA: BaseReg, RegB: DestReg))
2562 return E;
2563 }
2564 }
2565
2566 // Track which register units have been modified and used between the first
2567 // insn (inclusive) and the second insn.
2568 ModifiedRegUnits.clear();
2569 UsedRegUnits.clear();
2570 MBBI = next_nodbg(It: MBBI, End: E);
2571
2572 // We can't post-increment the stack pointer if any instruction between
2573 // the memory access (I) and the increment (MBBI) can access the memory
2574 // region defined by [SP, MBBI].
2575 const bool BaseRegSP = BaseReg == AArch64::SP;
2576 if (BaseRegSP && needsWinCFI(MF: I->getMF())) {
2577 // FIXME: For now, we always block the optimization over SP in windows
2578 // targets as it requires to adjust the unwind/debug info, messing up
2579 // the unwind info can actually cause a miscompile.
2580 return E;
2581 }
2582
2583 unsigned Count = 0;
2584 MachineBasicBlock *CurMBB = I->getParent();
2585 // choice of next block to visit is liveins-based
2586 bool VisitSucc = CurMBB->getParent()->getRegInfo().tracksLiveness();
2587
2588 while (true) {
2589 for (MachineBasicBlock::iterator CurEnd = CurMBB->end();
2590 MBBI != CurEnd && Count < Limit; MBBI = next_nodbg(It: MBBI, End: CurEnd)) {
2591 MachineInstr &MI = *MBBI;
2592
2593 // Don't count transient instructions towards the search limit since there
2594 // may be different numbers of them if e.g. debug information is present.
2595 if (!MI.isTransient())
2596 ++Count;
2597
2598 // If we found a match, return it.
2599 if (isMatchingUpdateInsn(MemMI&: *I, MI, BaseReg, Offset: UnscaledOffset))
2600 return MBBI;
2601
2602 // Update the status of what the instruction clobbered and used.
2603 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
2604 TRI);
2605
2606 // Otherwise, if the base register is used or modified, we have no match,
2607 // so return early. If we are optimizing SP, do not allow instructions
2608 // that may load or store in between the load and the optimized value
2609 // update.
2610 if (!ModifiedRegUnits.available(Reg: BaseReg) ||
2611 !UsedRegUnits.available(Reg: BaseReg) ||
2612 (BaseRegSP && MBBI->mayLoadOrStore()))
2613 return E;
2614 }
2615
2616 if (!VisitSucc || Limit <= Count)
2617 break;
2618
2619 // Try to go downward to successors along a CF path w/o side enters
2620 // such that BaseReg is alive along it but not at its exits
2621 MachineBasicBlock *SuccToVisit = nullptr;
2622 unsigned LiveSuccCount = 0;
2623 for (MachineBasicBlock *Succ : CurMBB->successors()) {
2624 for (MCRegAliasIterator AI(BaseReg, TRI, true); AI.isValid(); ++AI) {
2625 if (Succ->isLiveIn(Reg: *AI)) {
2626 if (LiveSuccCount++)
2627 return E;
2628 if (Succ->pred_size() == 1)
2629 SuccToVisit = Succ;
2630 break;
2631 }
2632 }
2633 }
2634 if (!SuccToVisit)
2635 break;
2636 CurMBB = SuccToVisit;
2637 MBBI = CurMBB->begin();
2638 }
2639
2640 return E;
2641}
2642
2643MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
2644 MachineBasicBlock::iterator I, unsigned Limit, bool &MergeEither) {
2645 MachineBasicBlock::iterator B = I->getParent()->begin();
2646 MachineBasicBlock::iterator E = I->getParent()->end();
2647 MachineInstr &MemMI = *I;
2648 MachineBasicBlock::iterator MBBI = I;
2649 MachineFunction &MF = *MemMI.getMF();
2650
2651 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2652 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getImm();
2653
2654 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MI: MemMI);
2655 Register DestReg[] = {getLdStRegOp(MI&: MemMI, PairedRegOp: 0).getReg(),
2656 IsPairedInsn ? getLdStRegOp(MI&: MemMI, PairedRegOp: 1).getReg()
2657 : AArch64::NoRegister};
2658
2659 // If the load/store is the first instruction in the block, there's obviously
2660 // not any matching update. Ditto if the memory offset isn't zero.
2661 if (MBBI == B || Offset != 0)
2662 return E;
2663 // If the base register overlaps a destination register, we can't
2664 // merge the update.
2665 if (!isTagStore(MI: MemMI)) {
2666 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i)
2667 if (DestReg[i] == BaseReg || TRI->isSubRegister(RegA: BaseReg, RegB: DestReg[i]))
2668 return E;
2669 }
2670
2671 const bool BaseRegSP = BaseReg == AArch64::SP;
2672 if (BaseRegSP && needsWinCFI(MF: I->getMF())) {
2673 // FIXME: For now, we always block the optimization over SP in windows
2674 // targets as it requires to adjust the unwind/debug info, messing up
2675 // the unwind info can actually cause a miscompile.
2676 return E;
2677 }
2678
2679 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2680 unsigned RedZoneSize =
2681 Subtarget.getTargetLowering()->getRedZoneSize(F: MF.getFunction());
2682
2683 // Track which register units have been modified and used between the first
2684 // insn (inclusive) and the second insn.
2685 ModifiedRegUnits.clear();
2686 UsedRegUnits.clear();
2687 unsigned Count = 0;
2688 bool MemAccessBeforeSPPreInc = false;
2689 MergeEither = true;
2690 do {
2691 MBBI = prev_nodbg(It: MBBI, Begin: B);
2692 MachineInstr &MI = *MBBI;
2693
2694 // Don't count transient instructions towards the search limit since there
2695 // may be different numbers of them if e.g. debug information is present.
2696 if (!MI.isTransient())
2697 ++Count;
2698
2699 // If we found a match, return it.
2700 if (isMatchingUpdateInsn(MemMI&: *I, MI, BaseReg, Offset)) {
2701 // Check that the update value is within our red zone limit (which may be
2702 // zero).
2703 if (MemAccessBeforeSPPreInc && MBBI->getOperand(i: 2).getImm() > RedZoneSize)
2704 return E;
2705 return MBBI;
2706 }
2707
2708 // Update the status of what the instruction clobbered and used.
2709 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2710
2711 // Otherwise, if the base register is used or modified, we have no match, so
2712 // return early.
2713 if (!ModifiedRegUnits.available(Reg: BaseReg) ||
2714 !UsedRegUnits.available(Reg: BaseReg))
2715 return E;
2716
2717 // If we have a destination register (i.e. a load instruction) and a
2718 // destination register is used or modified, then we can only merge forward,
2719 // i.e. the combined instruction is put in the place of the memory
2720 // instruction. Same applies if we see a memory access or side effects.
2721 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects() ||
2722 (DestReg[0] != AArch64::NoRegister &&
2723 !(ModifiedRegUnits.available(Reg: DestReg[0]) &&
2724 UsedRegUnits.available(Reg: DestReg[0]))) ||
2725 (DestReg[1] != AArch64::NoRegister &&
2726 !(ModifiedRegUnits.available(Reg: DestReg[1]) &&
2727 UsedRegUnits.available(Reg: DestReg[1]))))
2728 MergeEither = false;
2729
2730 // Keep track if we have a memory access before an SP pre-increment, in this
2731 // case we need to validate later that the update amount respects the red
2732 // zone.
2733 if (BaseRegSP && MBBI->mayLoadOrStore())
2734 MemAccessBeforeSPPreInc = true;
2735 } while (MBBI != B && Count < Limit);
2736 return E;
2737}
2738
2739MachineBasicBlock::iterator
2740AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
2741 MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) {
2742 MachineBasicBlock::iterator B = I->getParent()->begin();
2743 MachineBasicBlock::iterator E = I->getParent()->end();
2744 MachineInstr &MemMI = *I;
2745 MachineBasicBlock::iterator MBBI = I;
2746
2747 // If the load is the first instruction in the block, there's obviously
2748 // not any matching load or store.
2749 if (MBBI == B)
2750 return E;
2751
2752 // Make sure the IndexReg is killed and the shift amount is zero.
2753 // TODO: Relex this restriction to extend, simplify processing now.
2754 if (!AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).isKill() ||
2755 !AArch64InstrInfo::getLdStAmountOp(MI: MemMI).isImm() ||
2756 (AArch64InstrInfo::getLdStAmountOp(MI: MemMI).getImm() != 0))
2757 return E;
2758
2759 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getReg();
2760
2761 // Track which register units have been modified and used between the first
2762 // insn (inclusive) and the second insn.
2763 ModifiedRegUnits.clear();
2764 UsedRegUnits.clear();
2765 unsigned Count = 0;
2766 do {
2767 MBBI = prev_nodbg(It: MBBI, Begin: B);
2768 MachineInstr &MI = *MBBI;
2769
2770 // Don't count transient instructions towards the search limit since there
2771 // may be different numbers of them if e.g. debug information is present.
2772 if (!MI.isTransient())
2773 ++Count;
2774
2775 // If we found a match, return it.
2776 if (isMatchingMovConstInsn(MemMI&: *I, MI, IndexReg, Offset)) {
2777 return MBBI;
2778 }
2779
2780 // Update the status of what the instruction clobbered and used.
2781 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2782
2783 // Otherwise, if the index register is used or modified, we have no match,
2784 // so return early.
2785 if (!ModifiedRegUnits.available(Reg: IndexReg) ||
2786 !UsedRegUnits.available(Reg: IndexReg))
2787 return E;
2788
2789 } while (MBBI != B && Count < Limit);
2790 return E;
2791}
2792
2793bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2794 MachineBasicBlock::iterator &MBBI) {
2795 MachineInstr &MI = *MBBI;
2796 // If this is a volatile load, don't mess with it.
2797 if (MI.hasOrderedMemoryRef())
2798 return false;
2799
2800 if (needsWinCFI(MF: MI.getMF()) && MI.getFlag(Flag: MachineInstr::FrameDestroy))
2801 return false;
2802
2803 // Make sure this is a reg+imm.
2804 // FIXME: It is possible to extend it to handle reg+reg cases.
2805 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
2806 return false;
2807
2808 // Look backward up to LdStLimit instructions.
2809 MachineBasicBlock::iterator StoreI;
2810 if (findMatchingStore(I: MBBI, Limit: LdStLimit, StoreI)) {
2811 ++NumLoadsFromStoresPromoted;
2812 // Promote the load. Keeping the iterator straight is a
2813 // pain, so we let the merge routine tell us what the next instruction
2814 // is after it's done mucking about.
2815 MBBI = promoteLoadFromStore(LoadI: MBBI, StoreI);
2816 return true;
2817 }
2818 return false;
2819}
2820
2821// Merge adjacent zero stores into a wider store.
2822bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2823 MachineBasicBlock::iterator &MBBI) {
2824 assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store.");
2825 MachineInstr &MI = *MBBI;
2826 MachineBasicBlock::iterator E = MI.getParent()->end();
2827
2828 if (!TII->isCandidateToMergeOrPair(MI))
2829 return false;
2830
2831 // Look ahead up to LdStLimit instructions for a mergeable instruction.
2832 LdStPairFlags Flags;
2833 MachineBasicBlock::iterator MergeMI =
2834 findMatchingInsn(I: MBBI, Flags, Limit: LdStLimit, /* FindNarrowMerge = */ true);
2835 if (MergeMI != E) {
2836 ++NumZeroStoresPromoted;
2837
2838 // Keeping the iterator straight is a pain, so we let the merge routine tell
2839 // us what the next instruction is after it's done mucking about.
2840 MBBI = mergeNarrowZeroStores(I: MBBI, MergeMI, Flags);
2841 return true;
2842 }
2843 return false;
2844}
2845
2846// Find loads and stores that can be merged into a single load or store pair
2847// instruction.
2848bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
2849 MachineInstr &MI = *MBBI;
2850 MachineBasicBlock::iterator E = MI.getParent()->end();
2851
2852 if (!TII->isCandidateToMergeOrPair(MI))
2853 return false;
2854
2855 // If disable-ldp feature is opted, do not emit ldp.
2856 if (MI.mayLoad() && Subtarget->hasDisableLdp())
2857 return false;
2858
2859 // If disable-stp feature is opted, do not emit stp.
2860 if (MI.mayStore() && Subtarget->hasDisableStp())
2861 return false;
2862
2863 // Early exit if the offset is not possible to match. (6 bits of positive
2864 // range, plus allow an extra one in case we find a later insn that matches
2865 // with Offset-1)
2866 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI);
2867 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2868 int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
2869 // Allow one more for offset.
2870 if (Offset > 0)
2871 Offset -= OffsetStride;
2872 if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
2873 return false;
2874
2875 // Look ahead up to LdStLimit instructions for a pairable instruction.
2876 LdStPairFlags Flags;
2877 MachineBasicBlock::iterator Paired =
2878 findMatchingInsn(I: MBBI, Flags, Limit: LdStLimit, /* FindNarrowMerge = */ false);
2879
2880 if (Paired == E)
2881 return false;
2882
2883 // Keeping the iterator straight is a pain, so we let the merge routine tell
2884 // us what the next instruction is after it's done mucking about.
2885 auto Prev = std::prev(x: MBBI);
2886
2887 // Fetch the memoperand of the load/store that is a candidate for combination.
2888 MachineMemOperand *MemOp =
2889 MI.memoperands_empty() ? nullptr : MI.memoperands().front();
2890
2891 // If a load/store arrives and ldp/stp-aligned-only feature is opted, check
2892 // that the alignment of the source pointer is at least double the alignment
2893 // of the type.
2894 if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) ||
2895 (MI.mayStore() && Subtarget->hasStpAlignedOnly())) {
2896 // If there is no size/align information, cancel the transformation.
2897 if (!MemOp || !MemOp->getMemoryType().isValid()) {
2898 NumFailedAlignmentCheck++;
2899 return false;
2900 }
2901
2902 // Get the needed alignments to check them if
2903 // ldp-aligned-only/stp-aligned-only features are opted.
2904 uint64_t MemAlignment = MemOp->getAlign().value();
2905 uint64_t TypeAlignment =
2906 Align(MemOp->getSize().getValue().getKnownMinValue()).value();
2907
2908 if (MemAlignment < 2 * TypeAlignment) {
2909 NumFailedAlignmentCheck++;
2910 return false;
2911 }
2912 }
2913
2914 ++NumPairCreated;
2915 if (TII->hasUnscaledLdStOffset(MI))
2916 ++NumUnscaledPairCreated;
2917
2918 MBBI = mergePairedInsns(I: MBBI, Paired, Flags);
2919 // Collect liveness info for instructions between Prev and the new position
2920 // MBBI.
2921 for (auto I = std::next(x: Prev); I != MBBI; I++)
2922 updateDefinedRegisters(MI&: *I, Units&: DefinedInBB, TRI);
2923
2924 return true;
2925}
2926
2927bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2928 (MachineBasicBlock::iterator &MBBI) {
2929 MachineInstr &MI = *MBBI;
2930 MachineBasicBlock::iterator E = MI.getParent()->end();
2931 MachineBasicBlock::iterator Update;
2932
2933 // Do not form post-inc addressing mode for volatile accesses. Instructions
2934 // performing register writeback do not set a valid instruction syndrome,
2935 // making it impossible to handle MMIO in protected hypervisors.
2936 // Exclude accesses based on the stack pointer, as these can't be MMIO.
2937 // Also exclude MTE tag store instructions.
2938 if (MBBI->hasOrderedMemoryRef() &&
2939 AArch64InstrInfo::getLdStBaseOp(MI).getReg() != AArch64::SP &&
2940 !isTagStore(MI) && MI.getOpcode() != AArch64::STGPi)
2941 return false;
2942
2943 // Look forward to try to form a post-index instruction. For example,
2944 // ldr x0, [x20]
2945 // add x20, x20, #32
2946 // merged into:
2947 // ldr x0, [x20], #32
2948 Update = findMatchingUpdateInsnForward(I: MBBI, UnscaledOffset: 0, Limit: UpdateLimit);
2949 if (Update != E) {
2950 // Merge the update into the ld/st.
2951 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/false,
2952 /*IsPreIdx=*/false,
2953 /*MergeEither=*/false)) {
2954 MBBI = *NextI;
2955 return true;
2956 }
2957 }
2958
2959 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2960 if (TII->hasUnscaledLdStOffset(Opc: MI.getOpcode()))
2961 return false;
2962
2963 // Look back to try to find a pre-index instruction. For example,
2964 // add x0, x0, #8
2965 // ldr x1, [x0]
2966 // merged into:
2967 // ldr x1, [x0, #8]!
2968 bool MergeEither;
2969 Update = findMatchingUpdateInsnBackward(I: MBBI, Limit: UpdateLimit, MergeEither);
2970 if (Update != E) {
2971 // Merge the update into the ld/st.
2972 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/true,
2973 /*IsPreIdx=*/true, MergeEither)) {
2974 MBBI = *NextI;
2975 return true;
2976 }
2977 }
2978
2979 // The immediate in the load/store is scaled by the size of the memory
2980 // operation. The immediate in the add we're looking for,
2981 // however, is not, so adjust here.
2982 int UnscaledOffset =
2983 AArch64InstrInfo::getLdStOffsetOp(MI).getImm() * TII->getMemScale(MI);
2984
2985 // Look forward to try to find a pre-index instruction. For example,
2986 // ldr x1, [x0, #64]
2987 // add x0, x0, #64
2988 // merged into:
2989 // ldr x1, [x0, #64]!
2990 Update = findMatchingUpdateInsnForward(I: MBBI, UnscaledOffset, Limit: UpdateLimit);
2991 if (Update != E) {
2992 // Merge the update into the ld/st.
2993 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/false,
2994 /*IsPreIdx=*/true,
2995 /*MergeEither=*/false)) {
2996 MBBI = *NextI;
2997 return true;
2998 }
2999 }
3000
3001 return false;
3002}
3003
3004bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
3005 int Scale) {
3006 MachineInstr &MI = *MBBI;
3007 MachineBasicBlock::iterator E = MI.getParent()->end();
3008 MachineBasicBlock::iterator Update;
3009
3010 // Don't know how to handle unscaled pre/post-index versions below, so bail.
3011 if (TII->hasUnscaledLdStOffset(Opc: MI.getOpcode()))
3012 return false;
3013
3014 // Look back to try to find a const offset for index LdSt instruction. For
3015 // example,
3016 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3017 // ldr x1, [x0, x8]
3018 // merged into:
3019 // add x8, x0, a * (1<<12)
3020 // ldr x1, [x8, imm12]
3021 unsigned Offset;
3022 Update = findMatchingConstOffsetBackward(I: MBBI, Limit: LdStConstLimit, Offset);
3023 if (Update != E && (Offset & (Scale - 1)) == 0) {
3024 // Merge the imm12 into the ld/st.
3025 MBBI = mergeConstOffsetInsn(I: MBBI, Update, Offset, Scale);
3026 return true;
3027 }
3028
3029 return false;
3030}
3031
3032// Map a GPR store opcode to its FPR equivalent at the same data width.
3033// Returns 0 if no mapping exists.
3034static unsigned getGPRToFPRStoreOpcode(unsigned GPRStoreOpc) {
3035 switch (GPRStoreOpc) {
3036 // Unsigned immediate.
3037 case AArch64::STRBBui:
3038 return AArch64::STRBui;
3039 case AArch64::STRHHui:
3040 return AArch64::STRHui;
3041 case AArch64::STRWui:
3042 return AArch64::STRSui;
3043 case AArch64::STRXui:
3044 return AArch64::STRDui;
3045 // Unscaled immediate.
3046 case AArch64::STURBBi:
3047 return AArch64::STURBi;
3048 case AArch64::STURHHi:
3049 return AArch64::STURHi;
3050 case AArch64::STURWi:
3051 return AArch64::STURSi;
3052 case AArch64::STURXi:
3053 return AArch64::STURDi;
3054 // Register offset.
3055 case AArch64::STRBBroW:
3056 return AArch64::STRBroW;
3057 case AArch64::STRBBroX:
3058 return AArch64::STRBroX;
3059 case AArch64::STRHHroW:
3060 return AArch64::STRHroW;
3061 case AArch64::STRHHroX:
3062 return AArch64::STRHroX;
3063 case AArch64::STRWroW:
3064 return AArch64::STRSroW;
3065 case AArch64::STRWroX:
3066 return AArch64::STRSroX;
3067 case AArch64::STRXroW:
3068 return AArch64::STRDroW;
3069 case AArch64::STRXroX:
3070 return AArch64::STRDroX;
3071 default:
3072 return 0;
3073 }
3074}
3075
3076// Given a UMOV-lane-0 opcode, return the sub-register index to extract from
3077// the vector register, or 0 if the opcode is not a supported UMOV.
3078static unsigned getUMOVSubRegIdx(unsigned UMOVOpc) {
3079 switch (UMOVOpc) {
3080 case AArch64::UMOVvi8_idx0:
3081 return AArch64::bsub;
3082 case AArch64::UMOVvi16_idx0:
3083 return AArch64::hsub;
3084 case AArch64::UMOVvi32_idx0:
3085 return AArch64::ssub;
3086 case AArch64::UMOVvi64_idx0:
3087 return AArch64::dsub;
3088 default:
3089 return 0;
3090 }
3091}
3092
3093bool AArch64LoadStoreOpt::tryToReplaceUMOVStore(
3094 MachineBasicBlock::iterator &MBBI) {
3095 MachineInstr &StoreMI = *MBBI;
3096
3097 unsigned FPRStoreOpc = getGPRToFPRStoreOpcode(GPRStoreOpc: StoreMI.getOpcode());
3098 if (!FPRStoreOpc)
3099 return false;
3100
3101 if (StoreMI.hasOrderedMemoryRef() || StoreMI.memoperands().size() != 1)
3102 return false;
3103
3104 MachineBasicBlock *MBB = StoreMI.getParent();
3105 MCPhysReg StoreValReg = StoreMI.getOperand(i: 0).getReg();
3106
3107 if (!StoreMI.getOperand(i: 0).isKill())
3108 return false;
3109
3110 // Bail out if the store uses the value register elsewhere (e.g., as the base
3111 // address in `str w8, [x8, #0]`).
3112 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3113 if (StoreMI.getOperand(i: I).isReg() &&
3114 TRI->regsOverlap(RegA: StoreMI.getOperand(i: I).getReg(), RegB: StoreValReg))
3115 return false;
3116
3117 // Scan backward to find the UMOV that defines the store's value register.
3118 MachineInstr *UMOVMI = nullptr;
3119 MachineBasicBlock::iterator B = MBB->begin();
3120 unsigned SubRegIdx = 0;
3121 unsigned Count = 0;
3122 for (auto It = MBBI; It != B;) {
3123 MachineInstr &MI = *--It;
3124 if (MI.isDebugInstr())
3125 continue;
3126 if (++Count > UMOVFoldLimit)
3127 return false;
3128 if (MI.readsRegister(Reg: StoreValReg, TRI))
3129 return false;
3130 if (MI.modifiesRegister(Reg: StoreValReg, TRI)) {
3131 SubRegIdx = getUMOVSubRegIdx(UMOVOpc: MI.getOpcode());
3132 if (!SubRegIdx)
3133 return false;
3134 UMOVMI = &MI;
3135 break;
3136 }
3137 }
3138 if (!UMOVMI)
3139 return false;
3140 MCPhysReg VecReg = UMOVMI->getOperand(i: 1).getReg();
3141 MCPhysReg FPRReg = TRI->getSubReg(Reg: VecReg, Idx: SubRegIdx);
3142 if ((*StoreMI.memoperands_begin())->getSizeInBits() !=
3143 TRI->getRegSizeInBits(RC: *TRI->getMinimalPhysRegClass(Reg: FPRReg)))
3144 return false;
3145
3146 // Check that no instruction between UMOV and store clobbers the vector
3147 // register. Also track whether VecReg is killed anywhere from the UMOV
3148 // (inclusive) through the intervening instructions -- we need this to decide
3149 // whether the FPR sub-register can be marked killed on the new store.
3150 bool VecRegKilled = UMOVMI->killsRegister(Reg: VecReg, TRI);
3151 for (auto It = std::next(x: UMOVMI->getIterator()); It != MBBI; ++It) {
3152 if (It->modifiesRegister(Reg: VecReg, TRI))
3153 return false;
3154 if (!VecRegKilled && It->killsRegister(Reg: VecReg, TRI))
3155 VecRegKilled = true;
3156 }
3157
3158 // Safe to proceed. Clear kill flags on the vector register between UMOV and
3159 // the new store so the FPR sub-register stays live.
3160 UMOVMI->clearRegisterKills(Reg: VecReg, RegInfo: TRI);
3161 for (auto It = std::next(x: UMOVMI->getIterator()); It != MBBI; ++It)
3162 It->clearRegisterKills(Reg: VecReg, RegInfo: TRI);
3163
3164 LLVM_DEBUG(dbgs() << "Folding UMOV + store: " << *UMOVMI << " + "
3165 << StoreMI);
3166
3167 auto MIB = BuildMI(BB&: *MBB, I: MBBI, MIMD: StoreMI.getDebugLoc(), MCID: TII->get(Opcode: FPRStoreOpc))
3168 .addReg(RegNo: FPRReg, Flags: getKillRegState(B: VecRegKilled));
3169 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3170 MIB.add(MO: StoreMI.getOperand(i: I));
3171 MIB.setMemRefs(StoreMI.memoperands());
3172
3173 MBBI = MBB->erase(I: MBBI);
3174 UMOVMI->eraseFromParent();
3175
3176 ++NumUMOVFoldedToFPRStore;
3177 return true;
3178}
3179
3180bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
3181 bool EnableNarrowZeroStOpt) {
3182 AArch64FunctionInfo &AFI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
3183
3184 bool Modified = false;
3185 // Six transformations to do here:
3186 // 1) Find loads that directly read from stores and promote them by
3187 // replacing with mov instructions. If the store is wider than the load,
3188 // the load will be replaced with a bitfield extract.
3189 // e.g.,
3190 // str w1, [x0, #4]
3191 // ldrh w2, [x0, #6]
3192 // ; becomes
3193 // str w1, [x0, #4]
3194 // lsr w2, w1, #16
3195 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3196 MBBI != E;) {
3197 if (isPromotableLoadFromStore(MI&: *MBBI) && tryToPromoteLoadFromStore(MBBI))
3198 Modified = true;
3199 else
3200 ++MBBI;
3201 }
3202 // 2) Merge adjacent zero stores into a wider store.
3203 // e.g.,
3204 // strh wzr, [x0]
3205 // strh wzr, [x0, #2]
3206 // ; becomes
3207 // str wzr, [x0]
3208 // e.g.,
3209 // str wzr, [x0]
3210 // str wzr, [x0, #4]
3211 // ; becomes
3212 // str xzr, [x0]
3213 if (EnableNarrowZeroStOpt)
3214 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3215 MBBI != E;) {
3216 if (isPromotableZeroStoreInst(MI&: *MBBI) && tryToMergeZeroStInst(MBBI))
3217 Modified = true;
3218 else
3219 ++MBBI;
3220 }
3221 // 3) Find loads and stores that can be merged into a single load or store
3222 // pair instruction.
3223 // When compiling for SVE 128, also try to combine SVE fill/spill
3224 // instructions into LDP/STP.
3225 // e.g.,
3226 // ldr x0, [x2]
3227 // ldr x1, [x2, #8]
3228 // ; becomes
3229 // ldp x0, x1, [x2]
3230 // e.g.,
3231 // ldr z0, [x2]
3232 // ldr z1, [x2, #1, mul vl]
3233 // ; becomes
3234 // ldp q0, q1, [x2]
3235
3236 if (MBB.getParent()->getRegInfo().tracksLiveness()) {
3237 DefinedInBB.clear();
3238 DefinedInBB.addLiveIns(MBB);
3239 }
3240
3241 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3242 MBBI != E;) {
3243 // Track currently live registers up to this point, to help with
3244 // searching for a rename register on demand.
3245 updateDefinedRegisters(MI&: *MBBI, Units&: DefinedInBB, TRI);
3246 if (TII->isPairableLdStInst(MI: *MBBI) && tryToPairLdStInst(MBBI))
3247 Modified = true;
3248 else
3249 ++MBBI;
3250 }
3251 // 4) Find base register updates that can be merged into the load or store
3252 // as a base-reg writeback.
3253 // e.g.,
3254 // ldr x0, [x2]
3255 // add x2, x2, #4
3256 // ; becomes
3257 // ldr x0, [x2], #4
3258 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3259 MBBI != E;) {
3260 if (isMergeableLdStUpdate(MI&: *MBBI, AFI) && tryToMergeLdStUpdate(MBBI))
3261 Modified = true;
3262 else
3263 ++MBBI;
3264 }
3265
3266 // 5) Find a register assigned with a const value that can be combined with
3267 // into the load or store. e.g.,
3268 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3269 // ldr x1, [x0, x8]
3270 // ; becomes
3271 // add x8, x0, a * (1<<12)
3272 // ldr x1, [x8, imm12]
3273 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3274 MBBI != E;) {
3275 int Scale;
3276 if (isMergeableIndexLdSt(MI&: *MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale))
3277 Modified = true;
3278 else
3279 ++MBBI;
3280 }
3281
3282 // 6) Replace UMOV (lane 0) + GPR store with a direct FPR sub-register store.
3283 // e.g.,
3284 // umov w8, v0.h[0]
3285 // strh w8, [x0]
3286 // ; becomes
3287 // str h0, [x0]
3288 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3289 MBBI != E;) {
3290 if (tryToReplaceUMOVStore(MBBI))
3291 Modified = true;
3292 else
3293 ++MBBI;
3294 }
3295
3296 return Modified;
3297}
3298
3299bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
3300 Subtarget = &Fn.getSubtarget<AArch64Subtarget>();
3301 TII = Subtarget->getInstrInfo();
3302 TRI = Subtarget->getRegisterInfo();
3303
3304 // Resize the modified and used register unit trackers. We do this once
3305 // per function and then clear the register units each time we optimize a load
3306 // or store.
3307 ModifiedRegUnits.init(TRI: *TRI);
3308 UsedRegUnits.init(TRI: *TRI);
3309 DefinedInBB.init(TRI: *TRI);
3310
3311 bool Modified = false;
3312 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
3313 for (auto &MBB : Fn) {
3314 auto M = optimizeBlock(MBB, EnableNarrowZeroStOpt: enableNarrowZeroStOpt);
3315 Modified |= M;
3316 }
3317
3318 return Modified;
3319}
3320
3321// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and
3322// stores near one another? Note: The pre-RA instruction scheduler already has
3323// hooks to try and schedule pairable loads/stores together to improve pairing
3324// opportunities. Thus, pre-RA pairing pass may not be worth the effort.
3325
3326// FIXME: When pairing store instructions it's very possible for this pass to
3327// hoist a store with a KILL marker above another use (without a KILL marker).
3328// The resulting IR is invalid, but nothing uses the KILL markers after this
3329// pass, so it's never caused a problem in practice.
3330
3331bool AArch64LoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
3332 if (skipFunction(F: MF.getFunction()))
3333 return false;
3334 AArch64LoadStoreOpt Impl;
3335 Impl.AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3336 return Impl.runOnMachineFunction(Fn&: MF);
3337}
3338
3339/// createAArch64LoadStoreOptimizationPass - returns an instance of the
3340/// load / store optimization pass.
3341FunctionPass *llvm::createAArch64LoadStoreOptLegacyPass() {
3342 return new AArch64LoadStoreOptLegacy();
3343}
3344
3345PreservedAnalyses
3346AArch64LoadStoreOptPass::run(MachineFunction &MF,
3347 MachineFunctionAnalysisManager &MFAM) {
3348 AArch64LoadStoreOpt Impl;
3349 Impl.AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3350 .getManager()
3351 .getResult<AAManager>(IR&: MF.getFunction());
3352 bool Changed = Impl.runOnMachineFunction(Fn&: MF);
3353 if (!Changed)
3354 return PreservedAnalyses::all();
3355 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
3356 PA.preserveSet<CFGAnalyses>();
3357 return PA;
3358}
3359