1//===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs load / store related peephole
10// optimizations. This pass should be run after register allocation.
11//
12// The pass runs after the PrologEpilogInserter where we emit the CFI
13// instructions. In order to preserve the correctness of the unwind information,
14// the pass should not change the order of any two instructions, one of which
15// has the FrameSetup/FrameDestroy flag or, alternatively, apply an add-hoc fix
16// to unwind information.
17//
18//===----------------------------------------------------------------------===//
19
20#include "AArch64InstrInfo.h"
21#include "AArch64MachineFunctionInfo.h"
22#include "AArch64Subtarget.h"
23#include "MCTargetDesc/AArch64AddressingModes.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineFunctionPass.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineOperand.h"
36#include "llvm/CodeGen/MachineRegisterInfo.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCDwarf.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/DebugCounter.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <functional>
48#include <iterator>
49#include <limits>
50#include <optional>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "aarch64-ldst-opt"
55
56STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
57STATISTIC(NumPostFolded, "Number of post-index updates folded");
58STATISTIC(NumPreFolded, "Number of pre-index updates folded");
59STATISTIC(NumUnscaledPairCreated,
60 "Number of load/store from unscaled generated");
61STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
62STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
63STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation "
64 "not passed the alignment check");
65STATISTIC(NumConstOffsetFolded,
66 "Number of const offset of index address folded");
67STATISTIC(NumUMOVFoldedToFPRStore,
68 "Number of UMOV + GPR stores folded to FPR stores");
69
70DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
71 "Controls which pairs are considered for renaming");
72
73#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
74
75namespace {
76
77using LdStPairFlags = struct LdStPairFlags {
78 // If a matching instruction is found, MergeForward is set to true if the
79 // merge is to remove the first instruction and replace the second with
80 // a pair-wise insn, and false if the reverse is true.
81 bool MergeForward = false;
82
83 // SExtIdx gives the index of the result of the load pair that must be
84 // extended. The value of SExtIdx assumes that the paired load produces the
85 // value in this order: (I, returned iterator), i.e., -1 means no value has
86 // to be extended, 0 means I, and 1 means the returned iterator.
87 int SExtIdx = -1;
88
89 // If not none, RenameReg can be used to rename the result register of the
90 // first store in a pair. Currently this only works when merging stores
91 // forward.
92 std::optional<MCPhysReg> RenameReg;
93
94 LdStPairFlags() = default;
95
96 void setMergeForward(bool V = true) { MergeForward = V; }
97 bool getMergeForward() const { return MergeForward; }
98
99 void setSExtIdx(int V) { SExtIdx = V; }
100 int getSExtIdx() const { return SExtIdx; }
101
102 void setRenameReg(MCPhysReg R) { RenameReg = R; }
103 void clearRenameReg() { RenameReg = std::nullopt; }
104 std::optional<MCPhysReg> getRenameReg() const { return RenameReg; }
105};
106
107struct AArch64LoadStoreOpt {
108 AliasAnalysis *AA;
109 const AArch64InstrInfo *TII;
110 const TargetRegisterInfo *TRI;
111 const AArch64Subtarget *Subtarget;
112
113 // Track which register units have been modified and used.
114 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
115 LiveRegUnits DefinedInBB;
116
117 // Scan the instructions looking for a load/store that can be combined
118 // with the current instruction into a load/store pair.
119 // Return the matching instruction if one is found, else MBB->end().
120 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
121 LdStPairFlags &Flags,
122 unsigned Limit,
123 bool FindNarrowMerge);
124
125 // Scan the instructions looking for a store that writes to the address from
126 // which the current load instruction reads. Return true if one is found.
127 bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
128 MachineBasicBlock::iterator &StoreI);
129
130 // Merge the two instructions indicated into a wider narrow store instruction.
131 MachineBasicBlock::iterator
132 mergeNarrowZeroStores(MachineBasicBlock::iterator I,
133 MachineBasicBlock::iterator MergeMI,
134 const LdStPairFlags &Flags);
135
136 // Merge the two instructions indicated into a single pair-wise instruction.
137 MachineBasicBlock::iterator
138 mergePairedInsns(MachineBasicBlock::iterator I,
139 MachineBasicBlock::iterator Paired,
140 const LdStPairFlags &Flags);
141
142 // Promote the load that reads directly from the address stored to.
143 MachineBasicBlock::iterator
144 promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
145 MachineBasicBlock::iterator StoreI);
146
147 // Scan the instruction list to find a base register update that can
148 // be combined with the current instruction (a load or store) using
149 // pre or post indexed addressing with writeback. Scan forwards.
150 MachineBasicBlock::iterator
151 findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
152 int UnscaledOffset, unsigned Limit);
153
154 // Scan the instruction list to find a register assigned with a const
155 // value that can be combined with the current instruction (a load or store)
156 // using base addressing with writeback. Scan backwards.
157 MachineBasicBlock::iterator
158 findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit,
159 unsigned &Offset);
160
161 // Scan the instruction list to find a base register update that can
162 // be combined with the current instruction (a load or store) using
163 // pre or post indexed addressing with writeback. Scan backwards.
164 // `MergeEither` is set to true if the combined instruction may be placed
165 // either at the location of the load/store instruction or at the location of
166 // the update instruction.
167 MachineBasicBlock::iterator
168 findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit,
169 bool &MergeEither);
170
171 // Find an instruction that updates the base register of the ld/st
172 // instruction.
173 bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
174 unsigned BaseReg, int Offset);
175
176 bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI,
177 unsigned IndexReg, unsigned &Offset);
178
179 // Merge a pre- or post-index base register update into a ld/st instruction.
180 std::optional<MachineBasicBlock::iterator>
181 mergeUpdateInsn(MachineBasicBlock::iterator I,
182 MachineBasicBlock::iterator Update, bool IsForward,
183 bool IsPreIdx, bool MergeEither);
184
185 MachineBasicBlock::iterator
186 mergeConstOffsetInsn(MachineBasicBlock::iterator I,
187 MachineBasicBlock::iterator Update, unsigned Offset,
188 int Scale);
189
190 // Find and merge zero store instructions.
191 bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
192
193 // Find and pair ldr/str instructions.
194 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
195
196 // Find and promote load instructions which read directly from store.
197 bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
198
199 // Find and merge a base register updates before or after a ld/st instruction.
200 bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
201
202 // Find and merge an index ldr/st instruction into a base ld/st instruction.
203 bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale);
204
205 // Replace a UMOV (lane 0) + GPR store with a direct FPR sub-register store.
206 bool tryToReplaceUMOVStore(MachineBasicBlock::iterator &MBBI);
207
208 bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
209
210 bool runOnMachineFunction(MachineFunction &MF);
211};
212
213struct AArch64LoadStoreOptLegacy : public MachineFunctionPass {
214 static char ID;
215
216 AArch64LoadStoreOptLegacy() : MachineFunctionPass(ID) {}
217
218 bool runOnMachineFunction(MachineFunction &Fn) override;
219
220 void getAnalysisUsage(AnalysisUsage &AU) const override {
221 AU.addRequired<AAResultsWrapperPass>();
222 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
223 MachineFunctionPass::getAnalysisUsage(AU);
224 }
225
226 MachineFunctionProperties getRequiredProperties() const override {
227 return MachineFunctionProperties().setNoVRegs();
228 }
229
230 StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; }
231};
232
233char AArch64LoadStoreOptLegacy::ID = 0;
234
235} // end anonymous namespace
236
237INITIALIZE_PASS(AArch64LoadStoreOptLegacy, "aarch64-ldst-opt",
238 AARCH64_LOAD_STORE_OPT_NAME, false, false)
239
240static bool isNarrowStore(unsigned Opc) {
241 switch (Opc) {
242 default:
243 return false;
244 case AArch64::STRBBui:
245 case AArch64::STURBBi:
246 case AArch64::STRHHui:
247 case AArch64::STURHHi:
248 return true;
249 }
250}
251
252// These instruction set memory tag and either keep memory contents unchanged or
253// set it to zero, ignoring the address part of the source register.
254static bool isTagStore(const MachineInstr &MI) {
255 switch (MI.getOpcode()) {
256 default:
257 return false;
258 case AArch64::STGi:
259 case AArch64::STZGi:
260 case AArch64::ST2Gi:
261 case AArch64::STZ2Gi:
262 return true;
263 }
264}
265
266static unsigned getMatchingNonSExtOpcode(unsigned Opc,
267 bool *IsValidLdStrOpc = nullptr) {
268 if (IsValidLdStrOpc)
269 *IsValidLdStrOpc = true;
270 switch (Opc) {
271 default:
272 if (IsValidLdStrOpc)
273 *IsValidLdStrOpc = false;
274 return std::numeric_limits<unsigned>::max();
275 case AArch64::STRDui:
276 case AArch64::STURDi:
277 case AArch64::STRDpre:
278 case AArch64::STRQui:
279 case AArch64::STURQi:
280 case AArch64::STRQpre:
281 case AArch64::STRBBui:
282 case AArch64::STURBBi:
283 case AArch64::STRHHui:
284 case AArch64::STURHHi:
285 case AArch64::STRWui:
286 case AArch64::STRWpre:
287 case AArch64::STURWi:
288 case AArch64::STRXui:
289 case AArch64::STRXpre:
290 case AArch64::STURXi:
291 case AArch64::STR_ZXI:
292 case AArch64::LDRDui:
293 case AArch64::LDURDi:
294 case AArch64::LDRDpre:
295 case AArch64::LDRQui:
296 case AArch64::LDURQi:
297 case AArch64::LDRQpre:
298 case AArch64::LDRWui:
299 case AArch64::LDURWi:
300 case AArch64::LDRWpre:
301 case AArch64::LDRXui:
302 case AArch64::LDURXi:
303 case AArch64::LDRXpre:
304 case AArch64::STRSui:
305 case AArch64::STURSi:
306 case AArch64::STRSpre:
307 case AArch64::LDRSui:
308 case AArch64::LDURSi:
309 case AArch64::LDRSpre:
310 case AArch64::LDR_ZXI:
311 return Opc;
312 case AArch64::LDRSWui:
313 return AArch64::LDRWui;
314 case AArch64::LDURSWi:
315 return AArch64::LDURWi;
316 case AArch64::LDRSWpre:
317 return AArch64::LDRWpre;
318 }
319}
320
321static unsigned getMatchingWideOpcode(unsigned Opc) {
322 switch (Opc) {
323 default:
324 llvm_unreachable("Opcode has no wide equivalent!");
325 case AArch64::STRBBui:
326 return AArch64::STRHHui;
327 case AArch64::STRHHui:
328 return AArch64::STRWui;
329 case AArch64::STURBBi:
330 return AArch64::STURHHi;
331 case AArch64::STURHHi:
332 return AArch64::STURWi;
333 case AArch64::STURWi:
334 return AArch64::STURXi;
335 case AArch64::STRWui:
336 return AArch64::STRXui;
337 }
338}
339
340static unsigned getMatchingPairOpcode(unsigned Opc) {
341 switch (Opc) {
342 default:
343 llvm_unreachable("Opcode has no pairwise equivalent!");
344 case AArch64::STRSui:
345 case AArch64::STURSi:
346 return AArch64::STPSi;
347 case AArch64::STRSpre:
348 return AArch64::STPSpre;
349 case AArch64::STRDui:
350 case AArch64::STURDi:
351 return AArch64::STPDi;
352 case AArch64::STRDpre:
353 return AArch64::STPDpre;
354 case AArch64::STRQui:
355 case AArch64::STURQi:
356 case AArch64::STR_ZXI:
357 return AArch64::STPQi;
358 case AArch64::STRQpre:
359 return AArch64::STPQpre;
360 case AArch64::STRWui:
361 case AArch64::STURWi:
362 return AArch64::STPWi;
363 case AArch64::STRWpre:
364 return AArch64::STPWpre;
365 case AArch64::STRXui:
366 case AArch64::STURXi:
367 return AArch64::STPXi;
368 case AArch64::STRXpre:
369 return AArch64::STPXpre;
370 case AArch64::LDRSui:
371 case AArch64::LDURSi:
372 return AArch64::LDPSi;
373 case AArch64::LDRSpre:
374 return AArch64::LDPSpre;
375 case AArch64::LDRDui:
376 case AArch64::LDURDi:
377 return AArch64::LDPDi;
378 case AArch64::LDRDpre:
379 return AArch64::LDPDpre;
380 case AArch64::LDRQui:
381 case AArch64::LDURQi:
382 case AArch64::LDR_ZXI:
383 return AArch64::LDPQi;
384 case AArch64::LDRQpre:
385 return AArch64::LDPQpre;
386 case AArch64::LDRWui:
387 case AArch64::LDURWi:
388 return AArch64::LDPWi;
389 case AArch64::LDRWpre:
390 return AArch64::LDPWpre;
391 case AArch64::LDRXui:
392 case AArch64::LDURXi:
393 return AArch64::LDPXi;
394 case AArch64::LDRXpre:
395 return AArch64::LDPXpre;
396 case AArch64::LDRSWui:
397 case AArch64::LDURSWi:
398 return AArch64::LDPSWi;
399 case AArch64::LDRSWpre:
400 return AArch64::LDPSWpre;
401 }
402}
403
404static unsigned isMatchingStore(MachineInstr &LoadInst,
405 MachineInstr &StoreInst) {
406 unsigned LdOpc = LoadInst.getOpcode();
407 unsigned StOpc = StoreInst.getOpcode();
408 switch (LdOpc) {
409 default:
410 llvm_unreachable("Unsupported load instruction!");
411 case AArch64::LDRBBui:
412 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
413 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
414 case AArch64::LDURBBi:
415 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
416 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
417 case AArch64::LDRHHui:
418 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
419 StOpc == AArch64::STRXui;
420 case AArch64::LDURHHi:
421 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
422 StOpc == AArch64::STURXi;
423 case AArch64::LDRWui:
424 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
425 case AArch64::LDURWi:
426 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
427 case AArch64::LDRXui:
428 return StOpc == AArch64::STRXui;
429 case AArch64::LDURXi:
430 return StOpc == AArch64::STURXi;
431 }
432}
433
434static unsigned getPreIndexedOpcode(unsigned Opc) {
435 // FIXME: We don't currently support creating pre-indexed loads/stores when
436 // the load or store is the unscaled version. If we decide to perform such an
437 // optimization in the future the cases for the unscaled loads/stores will
438 // need to be added here.
439 switch (Opc) {
440 default:
441 llvm_unreachable("Opcode has no pre-indexed equivalent!");
442 case AArch64::STRBui:
443 return AArch64::STRBpre;
444 case AArch64::STRHui:
445 return AArch64::STRHpre;
446 case AArch64::STRSui:
447 return AArch64::STRSpre;
448 case AArch64::STRDui:
449 return AArch64::STRDpre;
450 case AArch64::STRQui:
451 return AArch64::STRQpre;
452 case AArch64::STRBBui:
453 return AArch64::STRBBpre;
454 case AArch64::STRHHui:
455 return AArch64::STRHHpre;
456 case AArch64::STRWui:
457 return AArch64::STRWpre;
458 case AArch64::STRXui:
459 return AArch64::STRXpre;
460 case AArch64::LDRBui:
461 return AArch64::LDRBpre;
462 case AArch64::LDRHui:
463 return AArch64::LDRHpre;
464 case AArch64::LDRSui:
465 return AArch64::LDRSpre;
466 case AArch64::LDRDui:
467 return AArch64::LDRDpre;
468 case AArch64::LDRQui:
469 return AArch64::LDRQpre;
470 case AArch64::LDRBBui:
471 return AArch64::LDRBBpre;
472 case AArch64::LDRHHui:
473 return AArch64::LDRHHpre;
474 case AArch64::LDRWui:
475 return AArch64::LDRWpre;
476 case AArch64::LDRXui:
477 return AArch64::LDRXpre;
478 case AArch64::LDRSWui:
479 return AArch64::LDRSWpre;
480 case AArch64::LDPSi:
481 return AArch64::LDPSpre;
482 case AArch64::LDPSWi:
483 return AArch64::LDPSWpre;
484 case AArch64::LDPDi:
485 return AArch64::LDPDpre;
486 case AArch64::LDPQi:
487 return AArch64::LDPQpre;
488 case AArch64::LDPWi:
489 return AArch64::LDPWpre;
490 case AArch64::LDPXi:
491 return AArch64::LDPXpre;
492 case AArch64::STPSi:
493 return AArch64::STPSpre;
494 case AArch64::STPDi:
495 return AArch64::STPDpre;
496 case AArch64::STPQi:
497 return AArch64::STPQpre;
498 case AArch64::STPWi:
499 return AArch64::STPWpre;
500 case AArch64::STPXi:
501 return AArch64::STPXpre;
502 case AArch64::STGi:
503 return AArch64::STGPreIndex;
504 case AArch64::STZGi:
505 return AArch64::STZGPreIndex;
506 case AArch64::ST2Gi:
507 return AArch64::ST2GPreIndex;
508 case AArch64::STZ2Gi:
509 return AArch64::STZ2GPreIndex;
510 case AArch64::STGPi:
511 return AArch64::STGPpre;
512 }
513}
514
515static unsigned getBaseAddressOpcode(unsigned Opc) {
516 // TODO: Add more index address stores.
517 switch (Opc) {
518 default:
519 llvm_unreachable("Opcode has no base address equivalent!");
520 case AArch64::LDRBroX:
521 return AArch64::LDRBui;
522 case AArch64::LDRBBroX:
523 return AArch64::LDRBBui;
524 case AArch64::LDRSBXroX:
525 return AArch64::LDRSBXui;
526 case AArch64::LDRSBWroX:
527 return AArch64::LDRSBWui;
528 case AArch64::LDRHroX:
529 return AArch64::LDRHui;
530 case AArch64::LDRHHroX:
531 return AArch64::LDRHHui;
532 case AArch64::LDRSHXroX:
533 return AArch64::LDRSHXui;
534 case AArch64::LDRSHWroX:
535 return AArch64::LDRSHWui;
536 case AArch64::LDRWroX:
537 return AArch64::LDRWui;
538 case AArch64::LDRSroX:
539 return AArch64::LDRSui;
540 case AArch64::LDRSWroX:
541 return AArch64::LDRSWui;
542 case AArch64::LDRDroX:
543 return AArch64::LDRDui;
544 case AArch64::LDRXroX:
545 return AArch64::LDRXui;
546 case AArch64::LDRQroX:
547 return AArch64::LDRQui;
548 }
549}
550
551static unsigned getPostIndexedOpcode(unsigned Opc) {
552 switch (Opc) {
553 default:
554 llvm_unreachable("Opcode has no post-indexed wise equivalent!");
555 case AArch64::STRBui:
556 return AArch64::STRBpost;
557 case AArch64::STRHui:
558 return AArch64::STRHpost;
559 case AArch64::STRSui:
560 case AArch64::STURSi:
561 return AArch64::STRSpost;
562 case AArch64::STRDui:
563 case AArch64::STURDi:
564 return AArch64::STRDpost;
565 case AArch64::STRQui:
566 case AArch64::STURQi:
567 return AArch64::STRQpost;
568 case AArch64::STRBBui:
569 return AArch64::STRBBpost;
570 case AArch64::STRHHui:
571 return AArch64::STRHHpost;
572 case AArch64::STRWui:
573 case AArch64::STURWi:
574 return AArch64::STRWpost;
575 case AArch64::STRXui:
576 case AArch64::STURXi:
577 return AArch64::STRXpost;
578 case AArch64::LDRBui:
579 return AArch64::LDRBpost;
580 case AArch64::LDRHui:
581 return AArch64::LDRHpost;
582 case AArch64::LDRSui:
583 case AArch64::LDURSi:
584 return AArch64::LDRSpost;
585 case AArch64::LDRDui:
586 case AArch64::LDURDi:
587 return AArch64::LDRDpost;
588 case AArch64::LDRQui:
589 case AArch64::LDURQi:
590 return AArch64::LDRQpost;
591 case AArch64::LDRBBui:
592 return AArch64::LDRBBpost;
593 case AArch64::LDRHHui:
594 return AArch64::LDRHHpost;
595 case AArch64::LDRWui:
596 case AArch64::LDURWi:
597 return AArch64::LDRWpost;
598 case AArch64::LDRXui:
599 case AArch64::LDURXi:
600 return AArch64::LDRXpost;
601 case AArch64::LDRSWui:
602 return AArch64::LDRSWpost;
603 case AArch64::LDPSi:
604 return AArch64::LDPSpost;
605 case AArch64::LDPSWi:
606 return AArch64::LDPSWpost;
607 case AArch64::LDPDi:
608 return AArch64::LDPDpost;
609 case AArch64::LDPQi:
610 return AArch64::LDPQpost;
611 case AArch64::LDPWi:
612 return AArch64::LDPWpost;
613 case AArch64::LDPXi:
614 return AArch64::LDPXpost;
615 case AArch64::STPSi:
616 return AArch64::STPSpost;
617 case AArch64::STPDi:
618 return AArch64::STPDpost;
619 case AArch64::STPQi:
620 return AArch64::STPQpost;
621 case AArch64::STPWi:
622 return AArch64::STPWpost;
623 case AArch64::STPXi:
624 return AArch64::STPXpost;
625 case AArch64::STGi:
626 return AArch64::STGPostIndex;
627 case AArch64::STZGi:
628 return AArch64::STZGPostIndex;
629 case AArch64::ST2Gi:
630 return AArch64::ST2GPostIndex;
631 case AArch64::STZ2Gi:
632 return AArch64::STZ2GPostIndex;
633 case AArch64::STGPi:
634 return AArch64::STGPpost;
635 }
636}
637
638static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI) {
639
640 unsigned OpcA = FirstMI.getOpcode();
641 unsigned OpcB = MI.getOpcode();
642
643 switch (OpcA) {
644 default:
645 return false;
646 case AArch64::STRSpre:
647 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
648 case AArch64::STRDpre:
649 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
650 case AArch64::STRQpre:
651 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
652 case AArch64::STRWpre:
653 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
654 case AArch64::STRXpre:
655 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
656 case AArch64::LDRSpre:
657 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
658 case AArch64::LDRDpre:
659 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
660 case AArch64::LDRQpre:
661 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
662 case AArch64::LDRWpre:
663 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
664 case AArch64::LDRXpre:
665 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
666 case AArch64::LDRSWpre:
667 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
668 }
669}
670
671// Returns the scale and offset range of pre/post indexed variants of MI.
672static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
673 int &MinOffset, int &MaxOffset) {
674 bool IsPaired = AArch64InstrInfo::isPairedLdSt(MI);
675 bool IsTagStore = isTagStore(MI);
676 // ST*G and all paired ldst have the same scale in pre/post-indexed variants
677 // as in the "unsigned offset" variant.
678 // All other pre/post indexed ldst instructions are unscaled.
679 Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
680
681 if (IsPaired) {
682 MinOffset = -64;
683 MaxOffset = 63;
684 } else {
685 MinOffset = -256;
686 MaxOffset = 255;
687 }
688}
689
690static MachineOperand &getLdStRegOp(MachineInstr &MI,
691 unsigned PairedRegOp = 0) {
692 assert(PairedRegOp < 2 && "Unexpected register operand idx.");
693 bool IsPreLdSt = AArch64InstrInfo::isPreLdSt(MI);
694 if (IsPreLdSt)
695 PairedRegOp += 1;
696 unsigned Idx =
697 AArch64InstrInfo::isPairedLdSt(MI) || IsPreLdSt ? PairedRegOp : 0;
698 return MI.getOperand(i: Idx);
699}
700
701static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst,
702 MachineInstr &StoreInst,
703 const AArch64InstrInfo *TII) {
704 assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
705 int LoadSize = TII->getMemScale(MI: LoadInst);
706 int StoreSize = TII->getMemScale(MI: StoreInst);
707 int UnscaledStOffset =
708 TII->hasUnscaledLdStOffset(MI&: StoreInst)
709 ? AArch64InstrInfo::getLdStOffsetOp(MI: StoreInst).getImm()
710 : AArch64InstrInfo::getLdStOffsetOp(MI: StoreInst).getImm() * StoreSize;
711 int UnscaledLdOffset =
712 TII->hasUnscaledLdStOffset(MI&: LoadInst)
713 ? AArch64InstrInfo::getLdStOffsetOp(MI: LoadInst).getImm()
714 : AArch64InstrInfo::getLdStOffsetOp(MI: LoadInst).getImm() * LoadSize;
715 return (UnscaledStOffset <= UnscaledLdOffset) &&
716 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
717}
718
719static bool isPromotableZeroStoreInst(MachineInstr &MI) {
720 unsigned Opc = MI.getOpcode();
721 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
722 isNarrowStore(Opc)) &&
723 getLdStRegOp(MI).getReg() == AArch64::WZR;
724}
725
726static bool isPromotableLoadFromStore(MachineInstr &MI) {
727 switch (MI.getOpcode()) {
728 default:
729 return false;
730 // Scaled instructions.
731 case AArch64::LDRBBui:
732 case AArch64::LDRHHui:
733 case AArch64::LDRWui:
734 case AArch64::LDRXui:
735 // Unscaled instructions.
736 case AArch64::LDURBBi:
737 case AArch64::LDURHHi:
738 case AArch64::LDURWi:
739 case AArch64::LDURXi:
740 return true;
741 }
742}
743
744static bool isMergeableLdStUpdate(MachineInstr &MI, AArch64FunctionInfo &AFI) {
745 unsigned Opc = MI.getOpcode();
746 switch (Opc) {
747 default:
748 return false;
749 // Scaled instructions.
750 case AArch64::STRBui:
751 case AArch64::STRHui:
752 case AArch64::STRSui:
753 case AArch64::STRDui:
754 case AArch64::STRQui:
755 case AArch64::STRXui:
756 case AArch64::STRWui:
757 case AArch64::STRHHui:
758 case AArch64::STRBBui:
759 case AArch64::LDRBui:
760 case AArch64::LDRHui:
761 case AArch64::LDRSui:
762 case AArch64::LDRDui:
763 case AArch64::LDRQui:
764 case AArch64::LDRXui:
765 case AArch64::LDRWui:
766 case AArch64::LDRHHui:
767 case AArch64::LDRBBui:
768 case AArch64::STGi:
769 case AArch64::STZGi:
770 case AArch64::ST2Gi:
771 case AArch64::STZ2Gi:
772 case AArch64::STGPi:
773 // Unscaled instructions.
774 case AArch64::STURSi:
775 case AArch64::STURDi:
776 case AArch64::STURQi:
777 case AArch64::STURWi:
778 case AArch64::STURXi:
779 case AArch64::LDURSi:
780 case AArch64::LDURDi:
781 case AArch64::LDURQi:
782 case AArch64::LDURWi:
783 case AArch64::LDURXi:
784 // Paired instructions.
785 case AArch64::LDPSi:
786 case AArch64::LDPSWi:
787 case AArch64::LDPDi:
788 case AArch64::LDPQi:
789 case AArch64::LDPWi:
790 case AArch64::LDPXi:
791 case AArch64::STPSi:
792 case AArch64::STPDi:
793 case AArch64::STPQi:
794 case AArch64::STPWi:
795 case AArch64::STPXi:
796 // Make sure this is a reg+imm (as opposed to an address reloc).
797 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
798 return false;
799
800 // When using stack tagging, simple sp+imm loads and stores are not
801 // tag-checked, but pre- and post-indexed versions of them are, so we can't
802 // replace the former with the latter. This transformation would be valid
803 // if the load/store accesses an untagged stack slot, but we don't have
804 // that information available after frame indices have been eliminated.
805 if (AFI.isMTETagged() &&
806 AArch64InstrInfo::getLdStBaseOp(MI).getReg() == AArch64::SP)
807 return false;
808
809 return true;
810 }
811}
812
813// Make sure this is a reg+reg Ld/St
814static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
815 unsigned Opc = MI.getOpcode();
816 switch (Opc) {
817 default:
818 return false;
819 // Scaled instructions.
820 // TODO: Add more index address stores.
821 case AArch64::LDRBroX:
822 case AArch64::LDRBBroX:
823 case AArch64::LDRSBXroX:
824 case AArch64::LDRSBWroX:
825 Scale = 1;
826 return true;
827 case AArch64::LDRHroX:
828 case AArch64::LDRHHroX:
829 case AArch64::LDRSHXroX:
830 case AArch64::LDRSHWroX:
831 Scale = 2;
832 return true;
833 case AArch64::LDRWroX:
834 case AArch64::LDRSroX:
835 case AArch64::LDRSWroX:
836 Scale = 4;
837 return true;
838 case AArch64::LDRDroX:
839 case AArch64::LDRXroX:
840 Scale = 8;
841 return true;
842 case AArch64::LDRQroX:
843 Scale = 16;
844 return true;
845 }
846}
847
848static bool isRewritableImplicitDef(const MachineInstr &MI,
849 const MachineOperand &MO) {
850 switch (MI.getOpcode()) {
851 default:
852 return MO.isRenamable();
853 case AArch64::ORRWrs:
854 case AArch64::ADDWri:
855 return true;
856 }
857}
858
859MachineBasicBlock::iterator
860AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
861 MachineBasicBlock::iterator MergeMI,
862 const LdStPairFlags &Flags) {
863 assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
864 "Expected promotable zero stores.");
865
866 MachineBasicBlock::iterator E = I->getParent()->end();
867 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
868 // If NextI is the second of the two instructions to be merged, we need
869 // to skip one further. Either way we merge will invalidate the iterator,
870 // and we don't need to scan the new instruction, as it's a pairwise
871 // instruction, which we're not considering for further action anyway.
872 if (NextI == MergeMI)
873 NextI = next_nodbg(It: NextI, End: E);
874
875 unsigned Opc = I->getOpcode();
876 unsigned MergeMIOpc = MergeMI->getOpcode();
877 bool IsScaled = !TII->hasUnscaledLdStOffset(Opc);
878 bool IsMergedMIScaled = !TII->hasUnscaledLdStOffset(Opc: MergeMIOpc);
879 int OffsetStride = IsScaled ? TII->getMemScale(MI: *I) : 1;
880 int MergeMIOffsetStride = IsMergedMIScaled ? TII->getMemScale(MI: *MergeMI) : 1;
881
882 bool MergeForward = Flags.getMergeForward();
883 // Insert our new paired instruction after whichever of the paired
884 // instructions MergeForward indicates.
885 MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
886 // Also based on MergeForward is from where we copy the base register operand
887 // so we get the flags compatible with the input code.
888 const MachineOperand &BaseRegOp =
889 MergeForward ? AArch64InstrInfo::getLdStBaseOp(MI: *MergeMI)
890 : AArch64InstrInfo::getLdStBaseOp(MI: *I);
891
892 // Which register is Rt and which is Rt2 depends on the offset order.
893 int64_t IOffsetInBytes =
894 AArch64InstrInfo::getLdStOffsetOp(MI: *I).getImm() * OffsetStride;
895 int64_t MIOffsetInBytes =
896 AArch64InstrInfo::getLdStOffsetOp(MI: *MergeMI).getImm() *
897 MergeMIOffsetStride;
898 // Select final offset based on the offset order.
899 int64_t OffsetImm;
900 if (IOffsetInBytes > MIOffsetInBytes)
901 OffsetImm = MIOffsetInBytes;
902 else
903 OffsetImm = IOffsetInBytes;
904
905 int NewOpcode = getMatchingWideOpcode(Opc);
906 // Adjust final offset on scaled stores because the new instruction
907 // has a different scale.
908 if (!TII->hasUnscaledLdStOffset(Opc: NewOpcode)) {
909 int NewOffsetStride = TII->getMemScale(Opc: NewOpcode);
910 assert(((OffsetImm % NewOffsetStride) == 0) &&
911 "Offset should be a multiple of the store memory scale");
912 OffsetImm = OffsetImm / NewOffsetStride;
913 }
914
915 // Construct the new instruction.
916 DebugLoc DL = I->getDebugLoc();
917 MachineBasicBlock *MBB = I->getParent();
918 MachineInstrBuilder MIB;
919 MIB = BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: NewOpcode))
920 .addReg(RegNo: isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
921 .add(MO: BaseRegOp)
922 .addImm(Val: OffsetImm)
923 .cloneMergedMemRefs(OtherMIs: {&*I, &*MergeMI})
924 .setMIFlags(I->mergeFlagsWith(Other: *MergeMI));
925 (void)MIB;
926
927 LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
928 LLVM_DEBUG(I->print(dbgs()));
929 LLVM_DEBUG(dbgs() << " ");
930 LLVM_DEBUG(MergeMI->print(dbgs()));
931 LLVM_DEBUG(dbgs() << " with instruction:\n ");
932 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
933 LLVM_DEBUG(dbgs() << "\n");
934
935 // Erase the old instructions.
936 I->eraseFromParent();
937 MergeMI->eraseFromParent();
938 return NextI;
939}
940
941// Apply Fn to all instructions between MI and the beginning of the block, until
942// a def for DefReg is reached. Returns true, iff Fn returns true for all
943// visited instructions. Stop after visiting Limit iterations.
944static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
945 const TargetRegisterInfo *TRI, unsigned Limit,
946 std::function<bool(MachineInstr &, bool)> &Fn) {
947 auto MBB = MI.getParent();
948 for (MachineInstr &I :
949 instructionsWithoutDebug(It: MI.getReverseIterator(), End: MBB->instr_rend())) {
950 if (!Limit)
951 return false;
952 --Limit;
953
954 bool isDef = any_of(Range: I.operands(), P: [DefReg, TRI](MachineOperand &MOP) {
955 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
956 TRI->regsOverlap(RegA: MOP.getReg(), RegB: DefReg);
957 });
958 if (!Fn(I, isDef))
959 return false;
960 if (isDef)
961 break;
962 }
963 return true;
964}
965
966static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units,
967 const TargetRegisterInfo *TRI) {
968
969 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
970 if (MOP.isReg() && MOP.isKill())
971 Units.removeReg(Reg: MOP.getReg());
972
973 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
974 if (MOP.isReg() && !MOP.isKill())
975 Units.addReg(Reg: MOP.getReg());
976}
977
978/// This function will add a new entry into the debugValueSubstitutions table
979/// when two instruction have been merged into a new one represented by \p
980/// MergedInstr.
981static void addDebugSubstitutionsToTable(MachineFunction *MF,
982 unsigned InstrNumToSet,
983 MachineInstr &OriginalInstr,
984 MachineInstr &MergedInstr) {
985
986 // Figure out the Operand Index of the destination register of the
987 // OriginalInstr in the new MergedInstr.
988 auto Reg = OriginalInstr.getOperand(i: 0).getReg();
989 unsigned OperandNo = 0;
990 bool RegFound = false;
991 for (const auto Op : MergedInstr.operands()) {
992 if (Op.getReg() == Reg) {
993 RegFound = true;
994 break;
995 }
996 OperandNo++;
997 }
998
999 if (RegFound)
1000 MF->makeDebugValueSubstitution({OriginalInstr.peekDebugInstrNum(), 0},
1001 {InstrNumToSet, OperandNo});
1002}
1003
1004MachineBasicBlock::iterator
1005AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
1006 MachineBasicBlock::iterator Paired,
1007 const LdStPairFlags &Flags) {
1008 MachineBasicBlock::iterator E = I->getParent()->end();
1009 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
1010 // If NextI is the second of the two instructions to be merged, we need
1011 // to skip one further. Either way we merge will invalidate the iterator,
1012 // and we don't need to scan the new instruction, as it's a pairwise
1013 // instruction, which we're not considering for further action anyway.
1014 if (NextI == Paired)
1015 NextI = next_nodbg(It: NextI, End: E);
1016
1017 int SExtIdx = Flags.getSExtIdx();
1018 unsigned Opc =
1019 SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(Opc: I->getOpcode());
1020 bool IsUnscaled = TII->hasUnscaledLdStOffset(Opc);
1021 int OffsetStride = IsUnscaled ? TII->getMemScale(MI: *I) : 1;
1022
1023 bool MergeForward = Flags.getMergeForward();
1024
1025 std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
1026 if (RenameReg) {
1027 MCRegister RegToRename = getLdStRegOp(MI&: *I).getReg();
1028 DefinedInBB.addReg(Reg: *RenameReg);
1029
1030 // Return the sub/super register for RenameReg, matching the size of
1031 // OriginalReg.
1032 auto GetMatchingSubReg =
1033 [this, RenameReg](const TargetRegisterClass *C) -> MCPhysReg {
1034 for (MCPhysReg SubOrSuper :
1035 TRI->sub_and_superregs_inclusive(Reg: *RenameReg)) {
1036 if (C->contains(Reg: SubOrSuper))
1037 return SubOrSuper;
1038 }
1039 llvm_unreachable("Should have found matching sub or super register!");
1040 };
1041
1042 std::function<bool(MachineInstr &, bool)> UpdateMIs =
1043 [this, RegToRename, GetMatchingSubReg, MergeForward](MachineInstr &MI,
1044 bool IsDef) {
1045 if (IsDef) {
1046 bool SeenDef = false;
1047 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1048 MachineOperand &MOP = MI.getOperand(i: OpIdx);
1049 // Rename the first explicit definition and all implicit
1050 // definitions matching RegToRename.
1051 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1052 (!MergeForward || !SeenDef ||
1053 (MOP.isDef() && MOP.isImplicit())) &&
1054 TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename)) {
1055 assert((MOP.isImplicit() ||
1056 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1057 "Need renamable operands");
1058 Register MatchingReg;
1059 if (const TargetRegisterClass *RC =
1060 MI.getRegClassConstraint(OpIdx, TII, TRI))
1061 MatchingReg = GetMatchingSubReg(RC);
1062 else {
1063 if (!isRewritableImplicitDef(MI, MO: MOP))
1064 continue;
1065 MatchingReg = GetMatchingSubReg(
1066 TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1067 }
1068 MOP.setReg(MatchingReg);
1069 SeenDef = true;
1070 }
1071 }
1072 } else {
1073 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1074 MachineOperand &MOP = MI.getOperand(i: OpIdx);
1075 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1076 TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename)) {
1077 assert((MOP.isImplicit() ||
1078 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1079 "Need renamable operands");
1080 Register MatchingReg;
1081 if (const TargetRegisterClass *RC =
1082 MI.getRegClassConstraint(OpIdx, TII, TRI))
1083 MatchingReg = GetMatchingSubReg(RC);
1084 else
1085 MatchingReg = GetMatchingSubReg(
1086 TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1087 assert(MatchingReg.isValid() &&
1088 "Cannot find matching regs for renaming");
1089 MOP.setReg(MatchingReg);
1090 }
1091 }
1092 }
1093 LLVM_DEBUG(dbgs() << "Renamed " << MI);
1094 return true;
1095 };
1096 forAllMIsUntilDef(MI&: MergeForward ? *I : *Paired->getPrevNode(), DefReg: RegToRename,
1097 TRI, UINT32_MAX, Fn&: UpdateMIs);
1098
1099#if !defined(NDEBUG)
1100 // For forward merging store:
1101 // Make sure the register used for renaming is not used between the
1102 // paired instructions. That would trash the content before the new
1103 // paired instruction.
1104 MCPhysReg RegToCheck = *RenameReg;
1105 // For backward merging load:
1106 // Make sure the register being renamed is not used between the
1107 // paired instructions. That would trash the content after the new
1108 // paired instruction.
1109 if (!MergeForward)
1110 RegToCheck = RegToRename;
1111 for (auto &MI :
1112 iterator_range<MachineInstrBundleIterator<llvm::MachineInstr>>(
1113 MergeForward ? std::next(I) : I,
1114 MergeForward ? std::next(Paired) : Paired))
1115 assert(all_of(MI.operands(),
1116 [this, RegToCheck](const MachineOperand &MOP) {
1117 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1118 MOP.isUndef() ||
1119 !TRI->regsOverlap(MOP.getReg(), RegToCheck);
1120 }) &&
1121 "Rename register used between paired instruction, trashing the "
1122 "content");
1123#endif
1124 }
1125
1126 // Insert our new paired instruction after whichever of the paired
1127 // instructions MergeForward indicates.
1128 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
1129 // Also based on MergeForward is from where we copy the base register operand
1130 // so we get the flags compatible with the input code.
1131 const MachineOperand &BaseRegOp =
1132 MergeForward ? AArch64InstrInfo::getLdStBaseOp(MI: *Paired)
1133 : AArch64InstrInfo::getLdStBaseOp(MI: *I);
1134
1135 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: *I).getImm();
1136 int PairedOffset = AArch64InstrInfo::getLdStOffsetOp(MI: *Paired).getImm();
1137 bool PairedIsUnscaled = TII->hasUnscaledLdStOffset(Opc: Paired->getOpcode());
1138 if (IsUnscaled != PairedIsUnscaled) {
1139 // We're trying to pair instructions that differ in how they are scaled. If
1140 // I is scaled then scale the offset of Paired accordingly. Otherwise, do
1141 // the opposite (i.e., make Paired's offset unscaled).
1142 int MemSize = TII->getMemScale(MI: *Paired);
1143 if (PairedIsUnscaled) {
1144 // If the unscaled offset isn't a multiple of the MemSize, we can't
1145 // pair the operations together.
1146 assert(!(PairedOffset % TII->getMemScale(*Paired)) &&
1147 "Offset should be a multiple of the stride!");
1148 PairedOffset /= MemSize;
1149 } else {
1150 PairedOffset *= MemSize;
1151 }
1152 }
1153
1154 // Which register is Rt and which is Rt2 depends on the offset order.
1155 // However, for pre load/stores the Rt should be the one of the pre
1156 // load/store.
1157 MachineInstr *RtMI, *Rt2MI;
1158 if (Offset == PairedOffset + OffsetStride &&
1159 !AArch64InstrInfo::isPreLdSt(MI: *I)) {
1160 RtMI = &*Paired;
1161 Rt2MI = &*I;
1162 // Here we swapped the assumption made for SExtIdx.
1163 // I.e., we turn ldp I, Paired into ldp Paired, I.
1164 // Update the index accordingly.
1165 if (SExtIdx != -1)
1166 SExtIdx = (SExtIdx + 1) % 2;
1167 } else {
1168 RtMI = &*I;
1169 Rt2MI = &*Paired;
1170 }
1171 int OffsetImm = AArch64InstrInfo::getLdStOffsetOp(MI: *RtMI).getImm();
1172 // Scale the immediate offset, if necessary.
1173 if (TII->hasUnscaledLdStOffset(Opc: RtMI->getOpcode())) {
1174 assert(!(OffsetImm % TII->getMemScale(*RtMI)) &&
1175 "Unscaled offset cannot be scaled.");
1176 OffsetImm /= TII->getMemScale(MI: *RtMI);
1177 }
1178
1179 // Construct the new instruction.
1180 MachineInstrBuilder MIB;
1181 DebugLoc DL = I->getDebugLoc();
1182 MachineBasicBlock *MBB = I->getParent();
1183 MachineOperand RegOp0 = getLdStRegOp(MI&: *RtMI);
1184 MachineOperand RegOp1 = getLdStRegOp(MI&: *Rt2MI);
1185 MachineOperand &PairedRegOp = RtMI == &*Paired ? RegOp0 : RegOp1;
1186 // Kill flags may become invalid when moving stores for pairing.
1187 if (RegOp0.isUse()) {
1188 if (!MergeForward) {
1189 // Clear kill flags on store if moving upwards. Example:
1190 // STRWui kill %w0, ...
1191 // USE %w1
1192 // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards
1193 // We are about to move the store of w1, so its kill flag may become
1194 // invalid; not the case for w0.
1195 // Since w1 is used between the stores, the kill flag on w1 is cleared
1196 // after merging.
1197 // STPWi kill %w0, %w1, ...
1198 // USE %w1
1199 for (auto It = std::next(x: I); It != Paired && PairedRegOp.isKill(); ++It)
1200 if (It->readsRegister(Reg: PairedRegOp.getReg(), TRI))
1201 PairedRegOp.setIsKill(false);
1202 } else {
1203 // Clear kill flags of the first stores register. Example:
1204 // STRWui %w1, ...
1205 // USE kill %w1 ; need to clear kill flag when moving STRWui downwards
1206 // STRW %w0
1207 Register Reg = getLdStRegOp(MI&: *I).getReg();
1208 for (MachineInstr &MI :
1209 make_range(x: std::next(x: I->getIterator()), y: Paired->getIterator()))
1210 MI.clearRegisterKills(Reg, RegInfo: TRI);
1211 }
1212 }
1213
1214 unsigned int MatchPairOpcode = getMatchingPairOpcode(Opc);
1215 MIB = BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: MatchPairOpcode));
1216
1217 // Adds the pre-index operand for pre-indexed ld/st pairs.
1218 if (AArch64InstrInfo::isPreLdSt(MI: *RtMI))
1219 MIB.addReg(RegNo: BaseRegOp.getReg(), Flags: RegState::Define);
1220
1221 MIB.add(MO: RegOp0)
1222 .add(MO: RegOp1)
1223 .add(MO: BaseRegOp)
1224 .addImm(Val: OffsetImm)
1225 .cloneMergedMemRefs(OtherMIs: {&*I, &*Paired})
1226 .setMIFlags(I->mergeFlagsWith(Other: *Paired));
1227
1228 (void)MIB;
1229
1230 LLVM_DEBUG(
1231 dbgs() << "Creating pair load/store. Replacing instructions:\n ");
1232 LLVM_DEBUG(I->print(dbgs()));
1233 LLVM_DEBUG(dbgs() << " ");
1234 LLVM_DEBUG(Paired->print(dbgs()));
1235 LLVM_DEBUG(dbgs() << " with instruction:\n ");
1236 if (SExtIdx != -1) {
1237 // Generate the sign extension for the proper result of the ldp.
1238 // I.e., with X1, that would be:
1239 // %w1 = KILL %w1, implicit-def %x1
1240 // %x1 = SBFMXri killed %x1, 0, 31
1241 MachineOperand &DstMO = MIB->getOperand(i: SExtIdx);
1242 // Right now, DstMO has the extended register, since it comes from an
1243 // extended opcode.
1244 Register DstRegX = DstMO.getReg();
1245 // Get the W variant of that register.
1246 Register DstRegW = TRI->getSubReg(Reg: DstRegX, Idx: AArch64::sub_32);
1247 // Update the result of LDP to use the W instead of the X variant.
1248 DstMO.setReg(DstRegW);
1249 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1250 LLVM_DEBUG(dbgs() << "\n");
1251 // Make the machine verifier happy by providing a definition for
1252 // the X register.
1253 // Insert this definition right after the generated LDP, i.e., before
1254 // InsertionPoint.
1255 MachineInstrBuilder MIBKill =
1256 BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::KILL), DestReg: DstRegW)
1257 .addReg(RegNo: DstRegW)
1258 .addReg(RegNo: DstRegX, Flags: RegState::Define);
1259 MIBKill->getOperand(i: 2).setImplicit();
1260 // Create the sign extension.
1261 MachineInstrBuilder MIBSXTW =
1262 BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::SBFMXri), DestReg: DstRegX)
1263 .addReg(RegNo: DstRegX)
1264 .addImm(Val: 0)
1265 .addImm(Val: 31);
1266 (void)MIBSXTW;
1267
1268 // In the case of a sign-extend, where we have something like:
1269 // debugValueSubstitutions:[]
1270 // $w1 = LDRWui $x0, 1, debug-instr-number 1
1271 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1272 // $x0 = LDRSWui $x0, 0, debug-instr-number 2
1273 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1274
1275 // It will be converted to:
1276 // debugValueSubstitutions:[]
1277 // $w0, $w1 = LDPWi $x0, 0
1278 // $w0 = KILL $w0, implicit-def $x0
1279 // $x0 = SBFMXri $x0, 0, 31
1280 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1281 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1282
1283 // We want the final result to look like:
1284 // debugValueSubstitutions:
1285 // - { srcinst: 1, srcop: 0, dstinst: 4, dstop: 1, subreg: 0 }
1286 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1287 // $w0, $w1 = LDPWi $x0, 0, debug-instr-number 4
1288 // $w0 = KILL $w0, implicit-def $x0
1289 // $x0 = SBFMXri $x0, 0, 31, debug-instr-number 3
1290 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1291 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1292
1293 // $x0 is where the final value is stored, so the sign extend (SBFMXri)
1294 // instruction contains the final value we care about we give it a new
1295 // debug-instr-number 3. Whereas, $w1 contains the final value that we care
1296 // about, therefore the LDP instruction is also given a new
1297 // debug-instr-number 4. We have to add these substitutions to the
1298 // debugValueSubstitutions table. However, we also have to ensure that the
1299 // OpIndex that pointed to debug-instr-number 1 gets updated to 1, because
1300 // $w1 is the second operand of the LDP instruction.
1301
1302 if (I->peekDebugInstrNum()) {
1303 // If I is the instruction which got sign extended and has a
1304 // debug-instr-number, give the SBFMXri instruction a new
1305 // debug-instr-number, and update the debugValueSubstitutions table with
1306 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1307 // instruction a new debug-instr-number, and update the
1308 // debugValueSubstitutions table with the new debug-instr-number and
1309 // OpIndex pair.
1310 unsigned NewInstrNum;
1311 if (DstRegX == I->getOperand(i: 0).getReg()) {
1312 NewInstrNum = MIBSXTW->getDebugInstrNum();
1313 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *I,
1314 MergedInstr&: *MIBSXTW);
1315 } else {
1316 NewInstrNum = MIB->getDebugInstrNum();
1317 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *I, MergedInstr&: *MIB);
1318 }
1319 }
1320 if (Paired->peekDebugInstrNum()) {
1321 // If Paired is the instruction which got sign extended and has a
1322 // debug-instr-number, give the SBFMXri instruction a new
1323 // debug-instr-number, and update the debugValueSubstitutions table with
1324 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1325 // instruction a new debug-instr-number, and update the
1326 // debugValueSubstitutions table with the new debug-instr-number and
1327 // OpIndex pair.
1328 unsigned NewInstrNum;
1329 if (DstRegX == Paired->getOperand(i: 0).getReg()) {
1330 NewInstrNum = MIBSXTW->getDebugInstrNum();
1331 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *Paired,
1332 MergedInstr&: *MIBSXTW);
1333 } else {
1334 NewInstrNum = MIB->getDebugInstrNum();
1335 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *Paired,
1336 MergedInstr&: *MIB);
1337 }
1338 }
1339
1340 LLVM_DEBUG(dbgs() << " Extend operand:\n ");
1341 LLVM_DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs()));
1342 } else if (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI) {
1343 // We are combining SVE fill/spill to LDP/STP, so we need to use the Q
1344 // variant of the registers.
1345 MachineOperand &MOp0 = MIB->getOperand(i: 0);
1346 MachineOperand &MOp1 = MIB->getOperand(i: 1);
1347 assert(AArch64::ZPRRegClass.contains(MOp0.getReg()) &&
1348 AArch64::ZPRRegClass.contains(MOp1.getReg()) && "Invalid register.");
1349 MOp0.setReg(AArch64::Q0 + (MOp0.getReg() - AArch64::Z0));
1350 MOp1.setReg(AArch64::Q0 + (MOp1.getReg() - AArch64::Z0));
1351 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1352 } else {
1353
1354 // In the case that the merge doesn't result in a sign-extend, if we have
1355 // something like:
1356 // debugValueSubstitutions:[]
1357 // $x1 = LDRXui $x0, 1, debug-instr-number 1
1358 // DBG_INSTR_REF !13, dbg-instr-ref(1, 0), debug-location !11
1359 // $x0 = LDRXui killed $x0, 0, debug-instr-number 2
1360 // DBG_INSTR_REF !14, dbg-instr-ref(2, 0), debug-location !11
1361
1362 // It will be converted to:
1363 // debugValueSubstitutions: []
1364 // $x0, $x1 = LDPXi $x0, 0
1365 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1366 // DBG_INSTR_REF !13, dbg-instr-ref(2, 0), debug-location !14
1367
1368 // We want the final result to look like:
1369 // debugValueSubstitutions:
1370 // - { srcinst: 1, srcop: 0, dstinst: 3, dstop: 1, subreg: 0 }
1371 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1372 // $x0, $x1 = LDPXi $x0, 0, debug-instr-number 3
1373 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1374 // DBG_INSTR_REF !12, dbg-instr-ref(2, 0), debug-location !14
1375
1376 // Here all that needs to be done is, that the LDP instruction needs to be
1377 // updated with a new debug-instr-number, we then need to add entries into
1378 // the debugSubstitutions table to map the old instr-refs to the new ones.
1379
1380 // Assign new DebugInstrNum to the Paired instruction.
1381 if (I->peekDebugInstrNum()) {
1382 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1383 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewDebugInstrNum, OriginalInstr&: *I,
1384 MergedInstr&: *MIB);
1385 }
1386 if (Paired->peekDebugInstrNum()) {
1387 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1388 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewDebugInstrNum, OriginalInstr&: *Paired,
1389 MergedInstr&: *MIB);
1390 }
1391
1392 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1393 }
1394 LLVM_DEBUG(dbgs() << "\n");
1395
1396 if (MergeForward)
1397 for (const MachineOperand &MOP : phys_regs_and_masks(MI: *I))
1398 if (MOP.isReg() && MOP.isKill())
1399 DefinedInBB.addReg(Reg: MOP.getReg());
1400
1401 // Copy over any implicit-def operands. This is like MI.copyImplicitOps, but
1402 // only copies implicit defs and makes sure that each operand is only added
1403 // once in case of duplicates.
1404 auto CopyImplicitOps = [&](MachineBasicBlock::iterator MI1,
1405 MachineBasicBlock::iterator MI2) {
1406 SmallSetVector<Register, 4> Ops;
1407 for (const MachineOperand &MO :
1408 llvm::drop_begin(RangeOrContainer: MI1->operands(), N: MI1->getDesc().getNumOperands()))
1409 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1410 Ops.insert(X: MO.getReg());
1411 for (const MachineOperand &MO :
1412 llvm::drop_begin(RangeOrContainer: MI2->operands(), N: MI2->getDesc().getNumOperands()))
1413 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1414 Ops.insert(X: MO.getReg());
1415 for (auto Op : Ops)
1416 MIB.addDef(RegNo: Op, Flags: RegState::Implicit);
1417 };
1418 CopyImplicitOps(I, Paired);
1419
1420 // Erase the old instructions.
1421 I->eraseFromParent();
1422 Paired->eraseFromParent();
1423
1424 return NextI;
1425}
1426
1427MachineBasicBlock::iterator
1428AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
1429 MachineBasicBlock::iterator StoreI) {
1430 MachineBasicBlock::iterator NextI =
1431 next_nodbg(It: LoadI, End: LoadI->getParent()->end());
1432
1433 int LoadSize = TII->getMemScale(MI: *LoadI);
1434 int StoreSize = TII->getMemScale(MI: *StoreI);
1435 Register LdRt = getLdStRegOp(MI&: *LoadI).getReg();
1436 const MachineOperand &StMO = getLdStRegOp(MI&: *StoreI);
1437 Register StRt = getLdStRegOp(MI&: *StoreI).getReg();
1438 bool IsStoreXReg = TRI->getRegClass(i: AArch64::GPR64RegClassID)->contains(Reg: StRt);
1439
1440 assert((IsStoreXReg ||
1441 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1442 "Unexpected RegClass");
1443
1444 MachineInstr *BitExtMI;
1445 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1446 // Remove the load, if the destination register of the loads is the same
1447 // register for stored value.
1448 if (StRt == LdRt && LoadSize == 8) {
1449 for (MachineInstr &MI : make_range(x: StoreI->getIterator(),
1450 y: LoadI->getIterator())) {
1451 if (MI.killsRegister(Reg: StRt, TRI)) {
1452 MI.clearRegisterKills(Reg: StRt, RegInfo: TRI);
1453 break;
1454 }
1455 }
1456 LLVM_DEBUG(dbgs() << "Remove load instruction:\n ");
1457 LLVM_DEBUG(LoadI->print(dbgs()));
1458 LLVM_DEBUG(dbgs() << "\n");
1459 LoadI->eraseFromParent();
1460 return NextI;
1461 }
1462 // Replace the load with a mov if the load and store are in the same size.
1463 BitExtMI =
1464 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1465 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), DestReg: LdRt)
1466 .addReg(RegNo: IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1467 .add(MO: StMO)
1468 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
1469 .setMIFlags(LoadI->getFlags());
1470 } else {
1471 // FIXME: Currently we disable this transformation in big-endian targets as
1472 // performance and correctness are verified only in little-endian.
1473 if (!Subtarget->isLittleEndian())
1474 return NextI;
1475 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI&: *LoadI);
1476 assert(IsUnscaled == TII->hasUnscaledLdStOffset(*StoreI) &&
1477 "Unsupported ld/st match");
1478 assert(LoadSize <= StoreSize && "Invalid load size");
1479 int UnscaledLdOffset =
1480 IsUnscaled
1481 ? AArch64InstrInfo::getLdStOffsetOp(MI: *LoadI).getImm()
1482 : AArch64InstrInfo::getLdStOffsetOp(MI: *LoadI).getImm() * LoadSize;
1483 int UnscaledStOffset =
1484 IsUnscaled
1485 ? AArch64InstrInfo::getLdStOffsetOp(MI: *StoreI).getImm()
1486 : AArch64InstrInfo::getLdStOffsetOp(MI: *StoreI).getImm() * StoreSize;
1487 int Width = LoadSize * 8;
1488 Register DestReg =
1489 IsStoreXReg ? Register(TRI->getMatchingSuperReg(
1490 Reg: LdRt, SubIdx: AArch64::sub_32, RC: &AArch64::GPR64RegClass))
1491 : LdRt;
1492
1493 assert((UnscaledLdOffset >= UnscaledStOffset &&
1494 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1495 "Invalid offset");
1496
1497 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1498 int Imms = Immr + Width - 1;
1499 if (UnscaledLdOffset == UnscaledStOffset) {
1500 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N
1501 | ((Immr) << 6) // immr
1502 | ((Imms) << 0) // imms
1503 ;
1504
1505 BitExtMI =
1506 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1507 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1508 DestReg)
1509 .add(MO: StMO)
1510 .addImm(Val: AndMaskEncoded)
1511 .setMIFlags(LoadI->getFlags());
1512 } else if (IsStoreXReg && Imms == 31) {
1513 // Use the 32 bit variant of UBFM if it's the LSR alias of the
1514 // instruction.
1515 assert(Immr <= Imms && "Expected LSR alias of UBFM");
1516 BitExtMI = BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1517 MCID: TII->get(Opcode: AArch64::UBFMWri),
1518 DestReg: TRI->getSubReg(Reg: DestReg, Idx: AArch64::sub_32))
1519 .addReg(RegNo: TRI->getSubReg(Reg: StRt, Idx: AArch64::sub_32))
1520 .addImm(Val: Immr)
1521 .addImm(Val: Imms)
1522 .setMIFlags(LoadI->getFlags());
1523 } else {
1524 BitExtMI =
1525 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1526 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1527 DestReg)
1528 .add(MO: StMO)
1529 .addImm(Val: Immr)
1530 .addImm(Val: Imms)
1531 .setMIFlags(LoadI->getFlags());
1532 }
1533 }
1534
1535 // Clear kill flags between store and load.
1536 for (MachineInstr &MI : make_range(x: StoreI->getIterator(),
1537 y: BitExtMI->getIterator()))
1538 if (MI.killsRegister(Reg: StRt, TRI)) {
1539 MI.clearRegisterKills(Reg: StRt, RegInfo: TRI);
1540 break;
1541 }
1542
1543 LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
1544 LLVM_DEBUG(StoreI->print(dbgs()));
1545 LLVM_DEBUG(dbgs() << " ");
1546 LLVM_DEBUG(LoadI->print(dbgs()));
1547 LLVM_DEBUG(dbgs() << " with instructions:\n ");
1548 LLVM_DEBUG(StoreI->print(dbgs()));
1549 LLVM_DEBUG(dbgs() << " ");
1550 LLVM_DEBUG((BitExtMI)->print(dbgs()));
1551 LLVM_DEBUG(dbgs() << "\n");
1552
1553 // Erase the old instructions.
1554 LoadI->eraseFromParent();
1555 return NextI;
1556}
1557
1558static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
1559 // Convert the byte-offset used by unscaled into an "element" offset used
1560 // by the scaled pair load/store instructions.
1561 if (IsUnscaled) {
1562 // If the byte-offset isn't a multiple of the stride, there's no point
1563 // trying to match it.
1564 if (Offset % OffsetStride)
1565 return false;
1566 Offset /= OffsetStride;
1567 }
1568 return Offset <= 63 && Offset >= -64;
1569}
1570
1571// Do alignment, specialized to power of 2 and for signed ints,
1572// avoiding having to do a C-style cast from uint_64t to int when
1573// using alignTo from include/llvm/Support/MathExtras.h.
1574// FIXME: Move this function to include/MathExtras.h?
1575static int alignTo(int Num, int PowOf2) {
1576 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1577}
1578
1579static bool mayAlias(MachineInstr &MIa,
1580 SmallVectorImpl<MachineInstr *> &MemInsns,
1581 AliasAnalysis *AA) {
1582 for (MachineInstr *MIb : MemInsns) {
1583 if (MIa.mayAlias(AA, Other: *MIb, /*UseTBAA*/ false)) {
1584 LLVM_DEBUG(dbgs() << "Aliasing with: "; MIb->dump());
1585 return true;
1586 }
1587 }
1588
1589 LLVM_DEBUG(dbgs() << "No aliases found\n");
1590 return false;
1591}
1592
1593bool AArch64LoadStoreOpt::findMatchingStore(
1594 MachineBasicBlock::iterator I, unsigned Limit,
1595 MachineBasicBlock::iterator &StoreI) {
1596 MachineBasicBlock::iterator B = I->getParent()->begin();
1597 MachineBasicBlock::iterator MBBI = I;
1598 MachineInstr &LoadMI = *I;
1599 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: LoadMI).getReg();
1600
1601 // If the load is the first instruction in the block, there's obviously
1602 // not any matching store.
1603 if (MBBI == B)
1604 return false;
1605
1606 // Track which register units have been modified and used between the first
1607 // insn and the second insn.
1608 ModifiedRegUnits.clear();
1609 UsedRegUnits.clear();
1610
1611 unsigned Count = 0;
1612 do {
1613 MBBI = prev_nodbg(It: MBBI, Begin: B);
1614 MachineInstr &MI = *MBBI;
1615
1616 // Don't count transient instructions towards the search limit since there
1617 // may be different numbers of them if e.g. debug information is present.
1618 if (!MI.isTransient())
1619 ++Count;
1620
1621 // If the load instruction reads directly from the address to which the
1622 // store instruction writes and the stored value is not modified, we can
1623 // promote the load. Since we do not handle stores with pre-/post-index,
1624 // it's unnecessary to check if BaseReg is modified by the store itself.
1625 // Also we can't handle stores without an immediate offset operand,
1626 // while the operand might be the address for a global variable.
1627 if (MI.mayStore() && isMatchingStore(LoadInst&: LoadMI, StoreInst&: MI) &&
1628 BaseReg == AArch64InstrInfo::getLdStBaseOp(MI).getReg() &&
1629 AArch64InstrInfo::getLdStOffsetOp(MI).isImm() &&
1630 isLdOffsetInRangeOfSt(LoadInst&: LoadMI, StoreInst&: MI, TII) &&
1631 ModifiedRegUnits.available(Reg: getLdStRegOp(MI).getReg())) {
1632 StoreI = MBBI;
1633 return true;
1634 }
1635
1636 if (MI.isCall())
1637 return false;
1638
1639 // Update modified / uses register units.
1640 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1641
1642 // Otherwise, if the base register is modified, we have no match, so
1643 // return early.
1644 if (!ModifiedRegUnits.available(Reg: BaseReg))
1645 return false;
1646
1647 // If we encounter a store aliased with the load, return early.
1648 if (MI.mayStore() && LoadMI.mayAlias(AA, Other: MI, /*UseTBAA*/ false))
1649 return false;
1650 } while (MBBI != B && Count < Limit);
1651 return false;
1652}
1653
1654// Returns true if FirstMI and MI are candidates for merging or pairing.
1655// Otherwise, returns false.
1656static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
1657 LdStPairFlags &Flags,
1658 const AArch64InstrInfo *TII) {
1659 // If this is volatile or if pairing is suppressed, not a candidate.
1660 if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
1661 return false;
1662
1663 // We should have already checked FirstMI for pair suppression and volatility.
1664 assert(!FirstMI.hasOrderedMemoryRef() &&
1665 !TII->isLdStPairSuppressed(FirstMI) &&
1666 "FirstMI shouldn't get here if either of these checks are true.");
1667
1668 if ((MI.getFlag(Flag: MachineInstr::FrameSetup) ||
1669 MI.getFlag(Flag: MachineInstr::FrameDestroy)) &&
1670 MI.getMF()->needsWinCFI())
1671 return false;
1672
1673 unsigned OpcA = FirstMI.getOpcode();
1674 unsigned OpcB = MI.getOpcode();
1675
1676 // Opcodes match: If the opcodes are pre ld/st there is nothing more to check.
1677 if (OpcA == OpcB)
1678 return !AArch64InstrInfo::isPreLdSt(MI: FirstMI);
1679
1680 // Bail out if one of the opcodes is SVE fill/spill, as we currently don't
1681 // allow pairing them with other instructions.
1682 if (OpcA == AArch64::LDR_ZXI || OpcA == AArch64::STR_ZXI ||
1683 OpcB == AArch64::LDR_ZXI || OpcB == AArch64::STR_ZXI)
1684 return false;
1685
1686 // Two pre ld/st of different opcodes cannot be merged either
1687 if (AArch64InstrInfo::isPreLdSt(MI: FirstMI) && AArch64InstrInfo::isPreLdSt(MI))
1688 return false;
1689
1690 // Try to match a sign-extended load/store with a zero-extended load/store.
1691 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1692 unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc: OpcA, IsValidLdStrOpc: &IsValidLdStrOpc);
1693 assert(IsValidLdStrOpc &&
1694 "Given Opc should be a Load or Store with an immediate");
1695 // OpcA will be the first instruction in the pair.
1696 if (NonSExtOpc == getMatchingNonSExtOpcode(Opc: OpcB, IsValidLdStrOpc: &PairIsValidLdStrOpc)) {
1697 Flags.setSExtIdx(NonSExtOpc == OpcA ? 1 : 0);
1698 return true;
1699 }
1700
1701 // If the second instruction isn't even a mergable/pairable load/store, bail
1702 // out.
1703 if (!PairIsValidLdStrOpc)
1704 return false;
1705
1706 // Narrow stores do not have a matching pair opcodes, so constrain their
1707 // merging to zero stores.
1708 if (isNarrowStore(Opc: OpcA) || isNarrowStore(Opc: OpcB))
1709 return getLdStRegOp(MI&: FirstMI).getReg() == AArch64::WZR &&
1710 getLdStRegOp(MI).getReg() == AArch64::WZR &&
1711 TII->getMemScale(MI: FirstMI) == TII->getMemScale(MI);
1712
1713 // The STR<S,D,Q,W,X>pre - STR<S,D,Q,W,X>ui and
1714 // LDR<S,D,Q,W,X,SW>pre-LDR<S,D,Q,W,X,SW>ui
1715 // are candidate pairs that can be merged.
1716 if (isPreLdStPairCandidate(FirstMI, MI))
1717 return true;
1718
1719 // Try to match an unscaled load/store with a scaled load/store.
1720 return TII->hasUnscaledLdStOffset(Opc: OpcA) != TII->hasUnscaledLdStOffset(Opc: OpcB) &&
1721 getMatchingPairOpcode(Opc: OpcA) == getMatchingPairOpcode(Opc: OpcB);
1722
1723 // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
1724}
1725
1726static bool canRenameMOP(const MachineInstr &MI, const MachineOperand &MOP,
1727 const TargetRegisterInfo *TRI) {
1728 if (MOP.isReg()) {
1729 auto *RegClass = TRI->getMinimalPhysRegClass(Reg: MOP.getReg());
1730 // Renaming registers with multiple disjunct sub-registers (e.g. the
1731 // result of a LD3) means that all sub-registers are renamed, potentially
1732 // impacting other instructions we did not check. Bail out.
1733 // Note that this relies on the structure of the AArch64 register file. In
1734 // particular, a subregister cannot be written without overwriting the
1735 // whole register.
1736 if (RegClass->HasDisjunctSubRegs && RegClass->CoveredBySubRegs &&
1737 (TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::dsub0) ||
1738 TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::qsub0) ||
1739 TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::zsub0))) {
1740 LLVM_DEBUG(
1741 dbgs()
1742 << " Cannot rename operands with multiple disjunct subregisters ("
1743 << MOP << ")\n");
1744 return false;
1745 }
1746
1747 // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
1748 // them must be known. For example, in ORRWrs the implicit-def
1749 // corresponds to the result register.
1750 if (MOP.isImplicit() && MOP.isDef()) {
1751 if (!isRewritableImplicitDef(MI, MO: MOP))
1752 return false;
1753 return TRI->isSuperOrSubRegisterEq(RegA: MI.getOperand(i: 0).getReg(),
1754 RegB: MOP.getReg());
1755 }
1756
1757 // Implicit uses of calls and instruction description are fixed and cannot
1758 // be renamed.
1759 if (MOP.isImplicit() &&
1760 (MI.isCall() || MI.getDesc().hasImplicitUseOfPhysReg(Reg: MOP.getReg())))
1761 return false;
1762 }
1763 return MOP.isImplicit() ||
1764 (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
1765}
1766
1767static bool
1768canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
1769 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1770 const TargetRegisterInfo *TRI) {
1771 if (!FirstMI.mayStore())
1772 return false;
1773
1774 // Check if we can find an unused register which we can use to rename
1775 // the register used by the first load/store.
1776
1777 auto RegToRename = getLdStRegOp(MI&: FirstMI).getReg();
1778 // For now, we only rename if the store operand gets killed at the store.
1779 if (!getLdStRegOp(MI&: FirstMI).isKill() &&
1780 !any_of(Range: FirstMI.operands(),
1781 P: [TRI, RegToRename](const MachineOperand &MOP) {
1782 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1783 MOP.isImplicit() && MOP.isKill() &&
1784 TRI->regsOverlap(RegA: RegToRename, RegB: MOP.getReg());
1785 })) {
1786 LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI);
1787 return false;
1788 }
1789
1790 bool FoundDef = false;
1791
1792 // For each instruction between FirstMI and the previous def for RegToRename,
1793 // we
1794 // * check if we can rename RegToRename in this instruction
1795 // * collect the registers used and required register classes for RegToRename.
1796 std::function<bool(MachineInstr &, bool)> CheckMIs = [&](MachineInstr &MI,
1797 bool IsDef) {
1798 LLVM_DEBUG(dbgs() << "Checking " << MI);
1799 // Currently we do not try to rename across frame-setup instructions.
1800 if (MI.getFlag(Flag: MachineInstr::FrameSetup)) {
1801 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1802 << "currently\n");
1803 return false;
1804 }
1805
1806 UsedInBetween.accumulate(MI);
1807
1808 // For a definition, check that we can rename the definition and exit the
1809 // loop.
1810 FoundDef = IsDef;
1811
1812 // For defs, check if we can rename the first def of RegToRename.
1813 if (FoundDef) {
1814 // For some pseudo instructions, we might not generate code in the end
1815 // (e.g. KILL) and we would end up without a correct def for the rename
1816 // register.
1817 // TODO: This might be overly conservative and we could handle those cases
1818 // in multiple ways:
1819 // 1. Insert an extra copy, to materialize the def.
1820 // 2. Skip pseudo-defs until we find an non-pseudo def.
1821 if (MI.isPseudo()) {
1822 LLVM_DEBUG(dbgs() << " Cannot rename pseudo/bundle instruction\n");
1823 return false;
1824 }
1825
1826 for (auto &MOP : MI.operands()) {
1827 if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
1828 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1829 continue;
1830 if (!canRenameMOP(MI, MOP, TRI)) {
1831 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1832 return false;
1833 }
1834 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1835 }
1836 return true;
1837 } else {
1838 for (auto &MOP : MI.operands()) {
1839 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1840 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1841 continue;
1842
1843 if (!canRenameMOP(MI, MOP, TRI)) {
1844 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1845 return false;
1846 }
1847 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1848 }
1849 }
1850 return true;
1851 };
1852
1853 const AArch64Options &CLOpts =
1854 FirstMI.getMF()->getSubtarget<AArch64Subtarget>().getCLOpts();
1855 if (!forAllMIsUntilDef(MI&: FirstMI, DefReg: RegToRename, TRI,
1856 Limit: CLOpts.load_store_scan_limit, Fn&: CheckMIs))
1857 return false;
1858
1859 if (!FoundDef) {
1860 LLVM_DEBUG(dbgs() << " Did not find definition for register in BB\n");
1861 return false;
1862 }
1863 return true;
1864}
1865
1866// We want to merge the second load into the first by rewriting the usages of
1867// the same reg between first (incl.) and second (excl.). We don't need to care
1868// about any insns before FirstLoad or after SecondLoad.
1869// 1. The second load writes new value into the same reg.
1870// - The renaming is impossible to impact later use of the reg.
1871// - The second load always trash the value written by the first load which
1872// means the reg must be killed before the second load.
1873// 2. The first load must be a def for the same reg so we don't need to look
1874// into anything before it.
1875static bool canRenameUntilSecondLoad(
1876 MachineInstr &FirstLoad, MachineInstr &SecondLoad,
1877 LiveRegUnits &UsedInBetween,
1878 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1879 const TargetRegisterInfo *TRI) {
1880 if (FirstLoad.isPseudo())
1881 return false;
1882
1883 UsedInBetween.accumulate(MI: FirstLoad);
1884 auto RegToRename = getLdStRegOp(MI&: FirstLoad).getReg();
1885 bool Success = std::all_of(
1886 first: FirstLoad.getIterator(), last: SecondLoad.getIterator(),
1887 pred: [&](MachineInstr &MI) {
1888 LLVM_DEBUG(dbgs() << "Checking " << MI);
1889 // Currently we do not try to rename across frame-setup instructions.
1890 if (MI.getFlag(Flag: MachineInstr::FrameSetup)) {
1891 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1892 << "currently\n");
1893 return false;
1894 }
1895
1896 for (auto &MOP : MI.operands()) {
1897 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1898 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1899 continue;
1900 if (!canRenameMOP(MI, MOP, TRI)) {
1901 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1902 return false;
1903 }
1904 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1905 }
1906
1907 return true;
1908 });
1909 return Success;
1910}
1911
1912// Check if we can find a physical register for renaming \p Reg. This register
1913// must:
1914// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
1915// defined registers up to the point where the renamed register will be used,
1916// * not used in \p UsedInBetween; UsedInBetween must contain all accessed
1917// registers in the range the rename register will be used,
1918// * is available in all used register classes (checked using RequiredClasses).
1919static std::optional<MCPhysReg> tryToFindRegisterToRename(
1920 const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB,
1921 LiveRegUnits &UsedInBetween,
1922 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1923 const TargetRegisterInfo *TRI) {
1924 const MachineRegisterInfo &RegInfo = MF.getRegInfo();
1925
1926 // Checks if any sub- or super-register of PR is callee saved.
1927 auto AnySubOrSuperRegCalleePreserved = [&MF, TRI](MCPhysReg PR) {
1928 return any_of(Range: TRI->sub_and_superregs_inclusive(Reg: PR),
1929 P: [&MF, TRI](MCPhysReg SubOrSuper) {
1930 return TRI->isCalleeSavedPhysReg(PhysReg: SubOrSuper, MF);
1931 });
1932 };
1933
1934 // Check if PR or one of its sub- or super-registers can be used for all
1935 // required register classes.
1936 auto CanBeUsedForAllClasses = [&RequiredClasses, TRI](MCPhysReg PR) {
1937 return all_of(Range&: RequiredClasses, P: [PR, TRI](const TargetRegisterClass *C) {
1938 return any_of(
1939 Range: TRI->sub_and_superregs_inclusive(Reg: PR),
1940 P: [C](MCPhysReg SubOrSuper) { return C->contains(Reg: SubOrSuper); });
1941 });
1942 };
1943
1944 auto *RegClass = TRI->getMinimalPhysRegClass(Reg);
1945 for (const MCPhysReg &PR : *RegClass) {
1946 if (DefinedInBB.available(Reg: PR) && UsedInBetween.available(Reg: PR) &&
1947 !RegInfo.isReserved(PhysReg: PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1948 CanBeUsedForAllClasses(PR)) {
1949 DefinedInBB.addReg(Reg: PR);
1950 LLVM_DEBUG(dbgs() << "Found rename register " << printReg(PR, TRI)
1951 << "\n");
1952 return {PR};
1953 }
1954 }
1955 LLVM_DEBUG(dbgs() << "No rename register found from "
1956 << TRI->getRegClassName(RegClass) << "\n");
1957 return std::nullopt;
1958}
1959
1960// For store pairs: returns a register from FirstMI to the beginning of the
1961// block that can be renamed.
1962// For load pairs: returns a register from FirstMI to MI that can be renamed.
1963static std::optional<MCPhysReg> findRenameRegForSameLdStRegPair(
1964 std::optional<bool> MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI,
1965 Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween,
1966 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1967 const TargetRegisterInfo *TRI) {
1968 std::optional<MCPhysReg> RenameReg;
1969 if (!DebugCounter::shouldExecute(Counter&: RegRenamingCounter))
1970 return RenameReg;
1971
1972 auto *RegClass = TRI->getMinimalPhysRegClass(Reg: getLdStRegOp(MI&: FirstMI).getReg());
1973 MachineFunction &MF = *FirstMI.getParent()->getParent();
1974 if (!RegClass || !MF.getRegInfo().tracksLiveness())
1975 return RenameReg;
1976
1977 const bool IsLoad = FirstMI.mayLoad();
1978
1979 if (!MaybeCanRename) {
1980 if (IsLoad)
1981 MaybeCanRename = {canRenameUntilSecondLoad(FirstLoad&: FirstMI, SecondLoad&: MI, UsedInBetween,
1982 RequiredClasses, TRI)};
1983 else
1984 MaybeCanRename = {
1985 canRenameUpToDef(FirstMI, UsedInBetween, RequiredClasses, TRI)};
1986 }
1987
1988 if (*MaybeCanRename) {
1989 RenameReg = tryToFindRegisterToRename(MF, Reg, DefinedInBB, UsedInBetween,
1990 RequiredClasses, TRI);
1991 }
1992 return RenameReg;
1993}
1994
1995/// Scan the instructions looking for a load/store that can be combined with the
1996/// current instruction into a wider equivalent or a load/store pair.
1997MachineBasicBlock::iterator
1998AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
1999 LdStPairFlags &Flags, unsigned Limit,
2000 bool FindNarrowMerge) {
2001 MachineBasicBlock::iterator E = I->getParent()->end();
2002 MachineBasicBlock::iterator MBBI = I;
2003 MachineInstr &FirstMI = *I;
2004 MBBI = next_nodbg(It: MBBI, End: E);
2005
2006 bool MayLoad = FirstMI.mayLoad();
2007 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI&: FirstMI);
2008 Register Reg = getLdStRegOp(MI&: FirstMI).getReg();
2009 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: FirstMI).getReg();
2010 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: FirstMI).getImm();
2011 int OffsetStride = IsUnscaled ? TII->getMemScale(MI: FirstMI) : 1;
2012 bool IsPromotableZeroStore = isPromotableZeroStoreInst(MI&: FirstMI);
2013
2014 std::optional<bool> MaybeCanRename;
2015 if (!Subtarget->getCLOpts().load_store_renaming)
2016 MaybeCanRename = {false};
2017
2018 SmallPtrSet<const TargetRegisterClass *, 5> RequiredClasses;
2019 LiveRegUnits UsedInBetween;
2020 UsedInBetween.init(TRI: *TRI);
2021
2022 Flags.clearRenameReg();
2023
2024 // Track which register units have been modified and used between the first
2025 // insn (inclusive) and the second insn.
2026 ModifiedRegUnits.clear();
2027 UsedRegUnits.clear();
2028
2029 // Remember any instructions that read/write memory between FirstMI and MI.
2030 SmallVector<MachineInstr *, 4> MemInsns;
2031
2032 LLVM_DEBUG(dbgs() << "Find match for: "; FirstMI.dump());
2033 for (unsigned Count = 0; MBBI != E && Count < Limit;
2034 MBBI = next_nodbg(It: MBBI, End: E)) {
2035 MachineInstr &MI = *MBBI;
2036 LLVM_DEBUG(dbgs() << "Analysing 2nd insn: "; MI.dump());
2037
2038 UsedInBetween.accumulate(MI);
2039
2040 // Don't count transient instructions towards the search limit since there
2041 // may be different numbers of them if e.g. debug information is present.
2042 if (!MI.isTransient())
2043 ++Count;
2044
2045 Flags.setSExtIdx(-1);
2046 if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
2047 AArch64InstrInfo::getLdStOffsetOp(MI).isImm()) {
2048 assert(MI.mayLoadOrStore() && "Expected memory operation.");
2049 // If we've found another instruction with the same opcode, check to see
2050 // if the base and offset are compatible with our starting instruction.
2051 // These instructions all have scaled immediate operands, so we just
2052 // check for +1/-1. Make sure to check the new instruction offset is
2053 // actually an immediate and not a symbolic reference destined for
2054 // a relocation.
2055 Register MIBaseReg = AArch64InstrInfo::getLdStBaseOp(MI).getReg();
2056 int MIOffset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2057 bool MIIsUnscaled = TII->hasUnscaledLdStOffset(MI);
2058 if (IsUnscaled != MIIsUnscaled) {
2059 // We're trying to pair instructions that differ in how they are scaled.
2060 // If FirstMI is scaled then scale the offset of MI accordingly.
2061 // Otherwise, do the opposite (i.e., make MI's offset unscaled).
2062 int MemSize = TII->getMemScale(MI);
2063 if (MIIsUnscaled) {
2064 // If the unscaled offset isn't a multiple of the MemSize, we can't
2065 // pair the operations together: bail and keep looking.
2066 if (MIOffset % MemSize) {
2067 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2068 UsedRegUnits, TRI);
2069 MemInsns.push_back(Elt: &MI);
2070 continue;
2071 }
2072 MIOffset /= MemSize;
2073 } else {
2074 MIOffset *= MemSize;
2075 }
2076 }
2077
2078 bool IsPreLdSt = isPreLdStPairCandidate(FirstMI, MI);
2079
2080 if (BaseReg == MIBaseReg) {
2081 // If the offset of the second ld/st is not equal to the size of the
2082 // destination register it can’t be paired with a pre-index ld/st
2083 // pair. Additionally if the base reg is used or modified the operations
2084 // can't be paired: bail and keep looking.
2085 if (IsPreLdSt) {
2086 bool IsOutOfBounds = MIOffset != TII->getMemScale(MI);
2087 bool IsBaseRegUsed = !UsedRegUnits.available(
2088 Reg: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2089 bool IsBaseRegModified = !ModifiedRegUnits.available(
2090 Reg: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2091 // If the stored value and the address of the second instruction is
2092 // the same, it needs to be using the updated register and therefore
2093 // it must not be folded.
2094 bool IsMIRegTheSame =
2095 TRI->regsOverlap(RegA: getLdStRegOp(MI).getReg(),
2096 RegB: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2097 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
2098 IsMIRegTheSame) {
2099 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2100 UsedRegUnits, TRI);
2101 MemInsns.push_back(Elt: &MI);
2102 continue;
2103 }
2104 } else {
2105 if ((Offset != MIOffset + OffsetStride) &&
2106 (Offset + OffsetStride != MIOffset)) {
2107 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2108 UsedRegUnits, TRI);
2109 MemInsns.push_back(Elt: &MI);
2110 continue;
2111 }
2112 }
2113
2114 int MinOffset = Offset < MIOffset ? Offset : MIOffset;
2115 if (FindNarrowMerge) {
2116 // If the alignment requirements of the scaled wide load/store
2117 // instruction can't express the offset of the scaled narrow input,
2118 // bail and keep looking. For promotable zero stores, allow only when
2119 // the stored value is the same (i.e., WZR).
2120 if ((!IsUnscaled && alignTo(Num: MinOffset, PowOf2: 2) != MinOffset) ||
2121 (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) {
2122 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2123 UsedRegUnits, TRI);
2124 MemInsns.push_back(Elt: &MI);
2125 continue;
2126 }
2127 } else {
2128 // Pairwise instructions have a 7-bit signed offset field. Single
2129 // insns have a 12-bit unsigned offset field. If the resultant
2130 // immediate offset of merging these instructions is out of range for
2131 // a pairwise instruction, bail and keep looking.
2132 if (!inBoundsForPair(IsUnscaled, Offset: MinOffset, OffsetStride)) {
2133 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2134 UsedRegUnits, TRI);
2135 MemInsns.push_back(Elt: &MI);
2136 LLVM_DEBUG(dbgs() << "Offset doesn't fit in immediate, "
2137 << "keep looking.\n");
2138 continue;
2139 }
2140 // If the alignment requirements of the paired (scaled) instruction
2141 // can't express the offset of the unscaled input, bail and keep
2142 // looking.
2143 if (IsUnscaled && (alignTo(Num: MinOffset, PowOf2: OffsetStride) != MinOffset)) {
2144 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2145 UsedRegUnits, TRI);
2146 MemInsns.push_back(Elt: &MI);
2147 LLVM_DEBUG(dbgs()
2148 << "Offset doesn't fit due to alignment requirements, "
2149 << "keep looking.\n");
2150 continue;
2151 }
2152 }
2153
2154 // If the BaseReg has been modified, then we cannot do the optimization.
2155 // For example, in the following pattern
2156 // ldr x1 [x2]
2157 // ldr x2 [x3]
2158 // ldr x4 [x2, #8],
2159 // the first and third ldr cannot be converted to ldp x1, x4, [x2]
2160 if (!ModifiedRegUnits.available(Reg: BaseReg))
2161 return E;
2162
2163 const bool SameLoadReg = MayLoad && TRI->isSuperOrSubRegisterEq(
2164 RegA: Reg, RegB: getLdStRegOp(MI).getReg());
2165
2166 // If the Rt of the second instruction (destination register of the
2167 // load) was not modified or used between the two instructions and none
2168 // of the instructions between the second and first alias with the
2169 // second, we can combine the second into the first.
2170 bool RtNotModified =
2171 ModifiedRegUnits.available(Reg: getLdStRegOp(MI).getReg());
2172 bool RtNotUsed = !(MI.mayLoad() && !SameLoadReg &&
2173 !UsedRegUnits.available(Reg: getLdStRegOp(MI).getReg()));
2174
2175 LLVM_DEBUG(dbgs() << "Checking, can combine 2nd into 1st insn:\n"
2176 << "Reg '" << getLdStRegOp(MI) << "' not modified: "
2177 << (RtNotModified ? "true" : "false") << "\n"
2178 << "Reg '" << getLdStRegOp(MI) << "' not used: "
2179 << (RtNotUsed ? "true" : "false") << "\n");
2180
2181 if (RtNotModified && RtNotUsed && !mayAlias(MIa&: MI, MemInsns, AA)) {
2182 // For pairs loading into the same reg, try to find a renaming
2183 // opportunity to allow the renaming of Reg between FirstMI and MI
2184 // and combine MI into FirstMI; otherwise bail and keep looking.
2185 if (SameLoadReg) {
2186 std::optional<MCPhysReg> RenameReg =
2187 findRenameRegForSameLdStRegPair(MaybeCanRename, FirstMI, MI,
2188 Reg, DefinedInBB, UsedInBetween,
2189 RequiredClasses, TRI);
2190 if (!RenameReg) {
2191 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2192 UsedRegUnits, TRI);
2193 MemInsns.push_back(Elt: &MI);
2194 LLVM_DEBUG(dbgs() << "Can't find reg for renaming, "
2195 << "keep looking.\n");
2196 continue;
2197 }
2198 Flags.setRenameReg(*RenameReg);
2199 }
2200
2201 Flags.setMergeForward(false);
2202 if (!SameLoadReg)
2203 Flags.clearRenameReg();
2204 return MBBI;
2205 }
2206
2207 // Likewise, if the Rt of the first instruction is not modified or used
2208 // between the two instructions and none of the instructions between the
2209 // first and the second alias with the first, we can combine the first
2210 // into the second.
2211 RtNotModified = !(
2212 MayLoad && !UsedRegUnits.available(Reg: getLdStRegOp(MI&: FirstMI).getReg()));
2213
2214 LLVM_DEBUG(dbgs() << "Checking, can combine 1st into 2nd insn:\n"
2215 << "Reg '" << getLdStRegOp(FirstMI)
2216 << "' not modified: "
2217 << (RtNotModified ? "true" : "false") << "\n");
2218
2219 if (RtNotModified && !mayAlias(MIa&: FirstMI, MemInsns, AA)) {
2220 if (ModifiedRegUnits.available(Reg: getLdStRegOp(MI&: FirstMI).getReg())) {
2221 Flags.setMergeForward(true);
2222 Flags.clearRenameReg();
2223 return MBBI;
2224 }
2225
2226 std::optional<MCPhysReg> RenameReg = findRenameRegForSameLdStRegPair(
2227 MaybeCanRename, FirstMI, MI, Reg, DefinedInBB, UsedInBetween,
2228 RequiredClasses, TRI);
2229 if (RenameReg) {
2230 Flags.setMergeForward(true);
2231 Flags.setRenameReg(*RenameReg);
2232 return MBBI;
2233 }
2234 }
2235 LLVM_DEBUG(dbgs() << "Unable to combine these instructions due to "
2236 << "interference in between, keep looking.\n");
2237 }
2238 }
2239
2240 // If the instruction wasn't a matching load or store. Stop searching if we
2241 // encounter a call instruction that might modify memory.
2242 if (MI.isCall()) {
2243 LLVM_DEBUG(dbgs() << "Found a call, stop looking.\n");
2244 return E;
2245 }
2246
2247 // Update modified / uses register units.
2248 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2249
2250 // Otherwise, if the base register is modified, we have no match, so
2251 // return early.
2252 if (!ModifiedRegUnits.available(Reg: BaseReg)) {
2253 LLVM_DEBUG(dbgs() << "Base reg is modified, stop looking.\n");
2254 return E;
2255 }
2256
2257 // Update list of instructions that read/write memory.
2258 if (MI.mayLoadOrStore())
2259 MemInsns.push_back(Elt: &MI);
2260 }
2261 return E;
2262}
2263
2264static MachineBasicBlock::iterator
2265maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI) {
2266 assert((MI.getOpcode() == AArch64::SUBXri ||
2267 MI.getOpcode() == AArch64::ADDXri) &&
2268 "Expected a register update instruction");
2269 auto End = MI.getParent()->end();
2270 if (MaybeCFI == End ||
2271 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
2272 !(MI.getFlag(Flag: MachineInstr::FrameSetup) ||
2273 MI.getFlag(Flag: MachineInstr::FrameDestroy)) ||
2274 MI.getOperand(i: 0).getReg() != AArch64::SP)
2275 return End;
2276
2277 const MachineFunction &MF = *MI.getParent()->getParent();
2278 unsigned CFIIndex = MaybeCFI->getOperand(i: 0).getCFIIndex();
2279 const MCCFIInstruction &CFI = MF.getFrameInstructions()[CFIIndex];
2280 switch (CFI.getOperation()) {
2281 case MCCFIInstruction::OpDefCfa:
2282 case MCCFIInstruction::OpDefCfaOffset:
2283 return MaybeCFI;
2284 default:
2285 return End;
2286 }
2287}
2288
2289std::optional<MachineBasicBlock::iterator> AArch64LoadStoreOpt::mergeUpdateInsn(
2290 MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update,
2291 bool IsForward, bool IsPreIdx, bool MergeEither) {
2292 assert((Update->getOpcode() == AArch64::ADDXri ||
2293 Update->getOpcode() == AArch64::SUBXri) &&
2294 "Unexpected base register update instruction to merge!");
2295 MachineBasicBlock::iterator E = I->getParent()->end();
2296 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
2297
2298 // If updating the SP and the following instruction is CFA offset related CFI,
2299 // make sure the CFI follows the SP update either by merging at the location
2300 // of the update or by moving the CFI after the merged instruction. If unable
2301 // to do so, bail.
2302 MachineBasicBlock::iterator InsertPt = I;
2303 if (IsForward) {
2304 assert(IsPreIdx);
2305 if (auto CFI = maybeMoveCFI(MI&: *Update, MaybeCFI: next_nodbg(It: Update, End: E)); CFI != E) {
2306 if (MergeEither) {
2307 InsertPt = Update;
2308 } else {
2309 // Take care not to reorder CFIs.
2310 if (std::any_of(first: std::next(x: CFI), last: I, pred: [](const auto &Insn) {
2311 return Insn.getOpcode() == TargetOpcode::CFI_INSTRUCTION;
2312 }))
2313 return std::nullopt;
2314
2315 MachineBasicBlock *MBB = InsertPt->getParent();
2316 MBB->splice(Where: std::next(x: InsertPt), Other: MBB, From: CFI);
2317 }
2318 }
2319 }
2320
2321 // Return the instruction following the merged instruction, which is
2322 // the instruction following our unmerged load. Unless that's the add/sub
2323 // instruction we're merging, in which case it's the one after that.
2324 if (NextI == Update)
2325 NextI = next_nodbg(It: NextI, End: E);
2326
2327 int Value = Update->getOperand(i: 2).getImm();
2328 assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 &&
2329 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
2330 if (Update->getOpcode() == AArch64::SUBXri)
2331 Value = -Value;
2332
2333 unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(Opc: I->getOpcode())
2334 : getPostIndexedOpcode(Opc: I->getOpcode());
2335 MachineInstrBuilder MIB;
2336 int Scale, MinOffset, MaxOffset;
2337 getPrePostIndexedMemOpInfo(MI: *I, Scale, MinOffset, MaxOffset);
2338 if (!AArch64InstrInfo::isPairedLdSt(MI: *I)) {
2339 // Non-paired instruction.
2340 MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
2341 MCID: TII->get(Opcode: NewOpc))
2342 .add(MO: Update->getOperand(i: 0))
2343 .add(MO: getLdStRegOp(MI&: *I))
2344 .add(MO: AArch64InstrInfo::getLdStBaseOp(MI: *I))
2345 .addImm(Val: Value / Scale)
2346 .setMemRefs(I->memoperands())
2347 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2348 } else {
2349 // Paired instruction.
2350 MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
2351 MCID: TII->get(Opcode: NewOpc))
2352 .add(MO: Update->getOperand(i: 0))
2353 .add(MO: getLdStRegOp(MI&: *I, PairedRegOp: 0))
2354 .add(MO: getLdStRegOp(MI&: *I, PairedRegOp: 1))
2355 .add(MO: AArch64InstrInfo::getLdStBaseOp(MI: *I))
2356 .addImm(Val: Value / Scale)
2357 .setMemRefs(I->memoperands())
2358 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2359 }
2360
2361 if (IsPreIdx) {
2362 ++NumPreFolded;
2363 LLVM_DEBUG(dbgs() << "Creating pre-indexed load/store.");
2364 } else {
2365 ++NumPostFolded;
2366 LLVM_DEBUG(dbgs() << "Creating post-indexed load/store.");
2367 }
2368 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2369 LLVM_DEBUG(I->print(dbgs()));
2370 LLVM_DEBUG(dbgs() << " ");
2371 LLVM_DEBUG(Update->print(dbgs()));
2372 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2373 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
2374 LLVM_DEBUG(dbgs() << "\n");
2375
2376 // Erase the old instructions for the block.
2377 I->eraseFromParent();
2378 Update->eraseFromParent();
2379
2380 return NextI;
2381}
2382
2383MachineBasicBlock::iterator
2384AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I,
2385 MachineBasicBlock::iterator Update,
2386 unsigned Offset, int Scale) {
2387 assert((Update->getOpcode() == AArch64::MOVKWi) &&
2388 "Unexpected const mov instruction to merge!");
2389 MachineBasicBlock::iterator E = I->getParent()->end();
2390 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
2391 MachineBasicBlock::iterator PrevI = prev_nodbg(It: Update, Begin: E);
2392 MachineInstr &MemMI = *I;
2393 unsigned Mask = (1 << 12) * Scale - 1;
2394 unsigned Low = Offset & Mask;
2395 unsigned High = Offset - Low;
2396 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2397 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getReg();
2398 MachineInstrBuilder AddMIB, MemMIB;
2399
2400 // Add IndexReg, BaseReg, High (the BaseReg may be SP)
2401 AddMIB =
2402 BuildMI(BB&: *I->getParent(), I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
2403 .addDef(RegNo: IndexReg)
2404 .addUse(RegNo: BaseReg)
2405 .addImm(Val: High >> 12) // shifted value
2406 .addImm(Val: 12); // shift 12
2407 (void)AddMIB;
2408 // Ld/St DestReg, IndexReg, Imm12
2409 unsigned NewOpc = getBaseAddressOpcode(Opc: I->getOpcode());
2410 MemMIB = BuildMI(BB&: *I->getParent(), I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2411 .add(MO: getLdStRegOp(MI&: MemMI))
2412 .add(MO: AArch64InstrInfo::getLdStOffsetOp(MI: MemMI))
2413 .addImm(Val: Low / Scale)
2414 .setMemRefs(I->memoperands())
2415 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2416 (void)MemMIB;
2417
2418 ++NumConstOffsetFolded;
2419 LLVM_DEBUG(dbgs() << "Creating base address load/store.\n");
2420 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2421 LLVM_DEBUG(PrevI->print(dbgs()));
2422 LLVM_DEBUG(dbgs() << " ");
2423 LLVM_DEBUG(Update->print(dbgs()));
2424 LLVM_DEBUG(dbgs() << " ");
2425 LLVM_DEBUG(I->print(dbgs()));
2426 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2427 LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs()));
2428 LLVM_DEBUG(dbgs() << " ");
2429 LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs()));
2430 LLVM_DEBUG(dbgs() << "\n");
2431
2432 // Erase the old instructions for the block.
2433 I->eraseFromParent();
2434 PrevI->eraseFromParent();
2435 Update->eraseFromParent();
2436
2437 return NextI;
2438}
2439
2440bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
2441 MachineInstr &MI,
2442 unsigned BaseReg, int Offset) {
2443 switch (MI.getOpcode()) {
2444 default:
2445 break;
2446 case AArch64::SUBXri:
2447 case AArch64::ADDXri:
2448 // Make sure it's a vanilla immediate operand, not a relocation or
2449 // anything else we can't handle.
2450 if (!MI.getOperand(i: 2).isImm())
2451 break;
2452 // Watch out for 1 << 12 shifted value.
2453 if (AArch64_AM::getShiftValue(Imm: MI.getOperand(i: 3).getImm()))
2454 break;
2455
2456 // The update instruction source and destination register must be the
2457 // same as the load/store base register.
2458 if (MI.getOperand(i: 0).getReg() != BaseReg ||
2459 MI.getOperand(i: 1).getReg() != BaseReg)
2460 break;
2461
2462 int UpdateOffset = MI.getOperand(i: 2).getImm();
2463 if (MI.getOpcode() == AArch64::SUBXri)
2464 UpdateOffset = -UpdateOffset;
2465
2466 // The immediate must be a multiple of the scaling factor of the pre/post
2467 // indexed instruction.
2468 int Scale, MinOffset, MaxOffset;
2469 getPrePostIndexedMemOpInfo(MI: MemMI, Scale, MinOffset, MaxOffset);
2470 if (UpdateOffset % Scale != 0)
2471 break;
2472
2473 // Scaled offset must fit in the instruction immediate.
2474 int ScaledOffset = UpdateOffset / Scale;
2475 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
2476 break;
2477
2478 // If we have a non-zero Offset, we check that it matches the amount
2479 // we're adding to the register.
2480 if (!Offset || Offset == UpdateOffset)
2481 return true;
2482 break;
2483 }
2484 return false;
2485}
2486
2487bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI,
2488 MachineInstr &MI,
2489 unsigned IndexReg,
2490 unsigned &Offset) {
2491 // The update instruction source and destination register must be the
2492 // same as the load/store index register.
2493 if (MI.getOpcode() == AArch64::MOVKWi &&
2494 TRI->isSuperOrSubRegisterEq(RegA: IndexReg, RegB: MI.getOperand(i: 1).getReg())) {
2495
2496 // movz + movk hold a large offset of a Ld/St instruction.
2497 MachineBasicBlock::iterator B = MI.getParent()->begin();
2498 MachineBasicBlock::iterator MBBI = &MI;
2499 // Skip the scene when the MI is the first instruction of a block.
2500 if (MBBI == B)
2501 return false;
2502 MBBI = prev_nodbg(It: MBBI, Begin: B);
2503 MachineInstr &MovzMI = *MBBI;
2504 // Make sure the MOVKWi and MOVZWi set the same register.
2505 if (MovzMI.getOpcode() == AArch64::MOVZWi &&
2506 MovzMI.getOperand(i: 0).getReg() == MI.getOperand(i: 0).getReg()) {
2507 unsigned Low = MovzMI.getOperand(i: 1).getImm();
2508 unsigned High = MI.getOperand(i: 2).getImm() << MI.getOperand(i: 3).getImm();
2509 Offset = High + Low;
2510 // 12-bit optionally shifted immediates are legal for adds.
2511 return Offset >> 24 == 0;
2512 }
2513 }
2514 return false;
2515}
2516
2517MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
2518 MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
2519 MachineBasicBlock::iterator E = I->getParent()->end();
2520 MachineInstr &MemMI = *I;
2521 MachineBasicBlock::iterator MBBI = I;
2522
2523 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2524 int MIUnscaledOffset = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getImm() *
2525 TII->getMemScale(MI: MemMI);
2526
2527 // Scan forward looking for post-index opportunities. Updating instructions
2528 // can't be formed if the memory instruction doesn't have the offset we're
2529 // looking for.
2530 if (MIUnscaledOffset != UnscaledOffset)
2531 return E;
2532
2533 // If the base register overlaps a source/destination register, we can't
2534 // merge the update. This does not apply to tag store instructions which
2535 // ignore the address part of the source register.
2536 // This does not apply to STGPi as well, which does not have unpredictable
2537 // behavior in this case unlike normal stores, and always performs writeback
2538 // after reading the source register value.
2539 if (!isTagStore(MI: MemMI) && MemMI.getOpcode() != AArch64::STGPi) {
2540 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MI: MemMI);
2541 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
2542 Register DestReg = getLdStRegOp(MI&: MemMI, PairedRegOp: i).getReg();
2543 if (DestReg == BaseReg || TRI->isSubRegister(RegA: BaseReg, RegB: DestReg))
2544 return E;
2545 }
2546 }
2547
2548 // Track which register units have been modified and used between the first
2549 // insn (inclusive) and the second insn.
2550 ModifiedRegUnits.clear();
2551 UsedRegUnits.clear();
2552 MBBI = next_nodbg(It: MBBI, End: E);
2553
2554 // We can't post-increment the stack pointer if any instruction between
2555 // the memory access (I) and the increment (MBBI) can access the memory
2556 // region defined by [SP, MBBI].
2557 const bool BaseRegSP = BaseReg == AArch64::SP;
2558 if (BaseRegSP && I->getMF()->needsWinCFI()) {
2559 // FIXME: For now, we always block the optimization over SP in windows
2560 // targets as it requires to adjust the unwind/debug info, messing up
2561 // the unwind info can actually cause a miscompile.
2562 return E;
2563 }
2564
2565 unsigned Count = 0;
2566 MachineBasicBlock *CurMBB = I->getParent();
2567 // choice of next block to visit is liveins-based
2568 bool VisitSucc = CurMBB->getParent()->getRegInfo().tracksLiveness();
2569
2570 while (true) {
2571 for (MachineBasicBlock::iterator CurEnd = CurMBB->end();
2572 MBBI != CurEnd && Count < Limit; MBBI = next_nodbg(It: MBBI, End: CurEnd)) {
2573 MachineInstr &MI = *MBBI;
2574
2575 // Don't count transient instructions towards the search limit since there
2576 // may be different numbers of them if e.g. debug information is present.
2577 if (!MI.isTransient())
2578 ++Count;
2579
2580 // If we found a match, return it.
2581 if (isMatchingUpdateInsn(MemMI&: *I, MI, BaseReg, Offset: UnscaledOffset))
2582 return MBBI;
2583
2584 // Update the status of what the instruction clobbered and used.
2585 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
2586 TRI);
2587
2588 // Otherwise, if the base register is used or modified, we have no match,
2589 // so return early. If we are optimizing SP, do not allow instructions
2590 // that may load or store in between the load and the optimized value
2591 // update.
2592 if (!ModifiedRegUnits.available(Reg: BaseReg) ||
2593 !UsedRegUnits.available(Reg: BaseReg) ||
2594 (BaseRegSP && MBBI->mayLoadOrStore()))
2595 return E;
2596 }
2597
2598 if (!VisitSucc || Limit <= Count)
2599 break;
2600
2601 // Try to go downward to successors along a CF path w/o side enters
2602 // such that BaseReg is alive along it but not at its exits
2603 MachineBasicBlock *SuccToVisit = nullptr;
2604 unsigned LiveSuccCount = 0;
2605 for (MachineBasicBlock *Succ : CurMBB->successors()) {
2606 for (MCRegAliasIterator AI(BaseReg, TRI, true); AI.isValid(); ++AI) {
2607 if (Succ->isLiveIn(Reg: *AI)) {
2608 if (LiveSuccCount++)
2609 return E;
2610 if (Succ->pred_size() == 1)
2611 SuccToVisit = Succ;
2612 break;
2613 }
2614 }
2615 }
2616 if (!SuccToVisit)
2617 break;
2618 CurMBB = SuccToVisit;
2619 MBBI = skipDebugInstructionsForward(It: CurMBB->begin(), End: CurMBB->end());
2620 }
2621
2622 return E;
2623}
2624
2625MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
2626 MachineBasicBlock::iterator I, unsigned Limit, bool &MergeEither) {
2627 MachineBasicBlock::iterator B = I->getParent()->begin();
2628 MachineBasicBlock::iterator E = I->getParent()->end();
2629 MachineInstr &MemMI = *I;
2630 MachineBasicBlock::iterator MBBI = I;
2631 MachineFunction &MF = *MemMI.getMF();
2632
2633 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2634 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getImm();
2635
2636 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MI: MemMI);
2637 Register DestReg[] = {getLdStRegOp(MI&: MemMI, PairedRegOp: 0).getReg(),
2638 IsPairedInsn ? getLdStRegOp(MI&: MemMI, PairedRegOp: 1).getReg()
2639 : Register()};
2640
2641 // If the load/store is the first instruction in the block, there's obviously
2642 // not any matching update. Ditto if the memory offset isn't zero.
2643 if (MBBI == B || Offset != 0)
2644 return E;
2645 // If the base register overlaps a destination register, we can't
2646 // merge the update.
2647 if (!isTagStore(MI: MemMI)) {
2648 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i)
2649 if (DestReg[i] == BaseReg || TRI->isSubRegister(RegA: BaseReg, RegB: DestReg[i]))
2650 return E;
2651 }
2652
2653 const bool BaseRegSP = BaseReg == AArch64::SP;
2654 if (BaseRegSP && I->getMF()->needsWinCFI()) {
2655 // FIXME: For now, we always block the optimization over SP in windows
2656 // targets as it requires to adjust the unwind/debug info, messing up
2657 // the unwind info can actually cause a miscompile.
2658 return E;
2659 }
2660
2661 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2662 unsigned RedZoneSize =
2663 Subtarget.getTargetLowering()->getRedZoneSize(F: MF.getFunction());
2664
2665 // Track which register units have been modified and used between the first
2666 // insn (inclusive) and the second insn.
2667 ModifiedRegUnits.clear();
2668 UsedRegUnits.clear();
2669 unsigned Count = 0;
2670 bool MemAccessBeforeSPPreInc = false;
2671 MergeEither = true;
2672 do {
2673 MBBI = prev_nodbg(It: MBBI, Begin: B);
2674 MachineInstr &MI = *MBBI;
2675
2676 // Don't count transient instructions towards the search limit since there
2677 // may be different numbers of them if e.g. debug information is present.
2678 if (!MI.isTransient())
2679 ++Count;
2680
2681 // If we found a match, return it.
2682 if (isMatchingUpdateInsn(MemMI&: *I, MI, BaseReg, Offset)) {
2683 // Check that the update value is within our red zone limit (which may be
2684 // zero).
2685 if (MemAccessBeforeSPPreInc && MBBI->getOperand(i: 2).getImm() > RedZoneSize)
2686 return E;
2687 return MBBI;
2688 }
2689
2690 // Update the status of what the instruction clobbered and used.
2691 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2692
2693 // Otherwise, if the base register is used or modified, we have no match, so
2694 // return early.
2695 if (!ModifiedRegUnits.available(Reg: BaseReg) ||
2696 !UsedRegUnits.available(Reg: BaseReg))
2697 return E;
2698
2699 // If we have a destination register (i.e. a load instruction) and a
2700 // destination register is used or modified, then we can only merge forward,
2701 // i.e. the combined instruction is put in the place of the memory
2702 // instruction. Same applies if we see a memory access or side effects.
2703 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects() ||
2704 (DestReg[0].isValid() && !(ModifiedRegUnits.available(Reg: DestReg[0]) &&
2705 UsedRegUnits.available(Reg: DestReg[0]))) ||
2706 (DestReg[1].isValid() && !(ModifiedRegUnits.available(Reg: DestReg[1]) &&
2707 UsedRegUnits.available(Reg: DestReg[1]))))
2708 MergeEither = false;
2709
2710 // Keep track if we have a memory access before an SP pre-increment, in this
2711 // case we need to validate later that the update amount respects the red
2712 // zone.
2713 if (BaseRegSP && MBBI->mayLoadOrStore())
2714 MemAccessBeforeSPPreInc = true;
2715 } while (MBBI != B && Count < Limit);
2716 return E;
2717}
2718
2719MachineBasicBlock::iterator
2720AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
2721 MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) {
2722 MachineBasicBlock::iterator B = I->getParent()->begin();
2723 MachineBasicBlock::iterator E = I->getParent()->end();
2724 MachineInstr &MemMI = *I;
2725 MachineBasicBlock::iterator MBBI = I;
2726
2727 // If the load is the first instruction in the block, there's obviously
2728 // not any matching load or store.
2729 if (MBBI == B)
2730 return E;
2731
2732 // Make sure the IndexReg is killed and the shift amount is zero.
2733 // TODO: Relex this restriction to extend, simplify processing now.
2734 if (!AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).isKill() ||
2735 !AArch64InstrInfo::getLdStAmountOp(MI: MemMI).isImm() ||
2736 (AArch64InstrInfo::getLdStAmountOp(MI: MemMI).getImm() != 0))
2737 return E;
2738
2739 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getReg();
2740
2741 // Track which register units have been modified and used between the first
2742 // insn (inclusive) and the second insn.
2743 ModifiedRegUnits.clear();
2744 UsedRegUnits.clear();
2745 unsigned Count = 0;
2746 do {
2747 MBBI = prev_nodbg(It: MBBI, Begin: B);
2748 MachineInstr &MI = *MBBI;
2749
2750 // Don't count transient instructions towards the search limit since there
2751 // may be different numbers of them if e.g. debug information is present.
2752 if (!MI.isTransient())
2753 ++Count;
2754
2755 // If we found a match, return it.
2756 if (isMatchingMovConstInsn(MemMI&: *I, MI, IndexReg, Offset)) {
2757 return MBBI;
2758 }
2759
2760 // Update the status of what the instruction clobbered and used.
2761 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2762
2763 // Otherwise, if the index register is used or modified, we have no match,
2764 // so return early.
2765 if (!ModifiedRegUnits.available(Reg: IndexReg) ||
2766 !UsedRegUnits.available(Reg: IndexReg))
2767 return E;
2768
2769 } while (MBBI != B && Count < Limit);
2770 return E;
2771}
2772
2773bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2774 MachineBasicBlock::iterator &MBBI) {
2775 MachineInstr &MI = *MBBI;
2776 // If this is a volatile load, don't mess with it.
2777 if (MI.hasOrderedMemoryRef())
2778 return false;
2779
2780 if (MI.getFlag(Flag: MachineInstr::FrameDestroy) && MI.getMF()->needsWinCFI())
2781 return false;
2782
2783 // Make sure this is a reg+imm.
2784 // FIXME: It is possible to extend it to handle reg+reg cases.
2785 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
2786 return false;
2787
2788 // Look backward up to the scan limit.
2789 MachineBasicBlock::iterator StoreI;
2790 if (findMatchingStore(I: MBBI, Limit: Subtarget->getCLOpts().load_store_scan_limit,
2791 StoreI)) {
2792 ++NumLoadsFromStoresPromoted;
2793 // Promote the load. Keeping the iterator straight is a
2794 // pain, so we let the merge routine tell us what the next instruction
2795 // is after it's done mucking about.
2796 MBBI = promoteLoadFromStore(LoadI: MBBI, StoreI);
2797 return true;
2798 }
2799 return false;
2800}
2801
2802// Merge adjacent zero stores into a wider store.
2803bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2804 MachineBasicBlock::iterator &MBBI) {
2805 assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store.");
2806 MachineInstr &MI = *MBBI;
2807 MachineBasicBlock::iterator E = MI.getParent()->end();
2808
2809 if (!TII->isCandidateToMergeOrPair(MI))
2810 return false;
2811
2812 // Look ahead up to the scan limit for a mergeable instruction.
2813 LdStPairFlags Flags;
2814 MachineBasicBlock::iterator MergeMI = findMatchingInsn(
2815 I: MBBI, Flags, Limit: Subtarget->getCLOpts().load_store_scan_limit,
2816 /*FindNarrowMerge=*/true);
2817 if (MergeMI != E) {
2818 ++NumZeroStoresPromoted;
2819
2820 // Keeping the iterator straight is a pain, so we let the merge routine tell
2821 // us what the next instruction is after it's done mucking about.
2822 MBBI = mergeNarrowZeroStores(I: MBBI, MergeMI, Flags);
2823 return true;
2824 }
2825 return false;
2826}
2827
2828// Find loads and stores that can be merged into a single load or store pair
2829// instruction.
2830bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
2831 MachineInstr &MI = *MBBI;
2832 MachineBasicBlock::iterator E = MI.getParent()->end();
2833
2834 if (!TII->isCandidateToMergeOrPair(MI))
2835 return false;
2836
2837 // If disable-ldp feature is opted, do not emit ldp.
2838 if (MI.mayLoad() && Subtarget->hasDisableLdp())
2839 return false;
2840
2841 // If disable-stp feature is opted, do not emit stp.
2842 if (MI.mayStore() && Subtarget->hasDisableStp())
2843 return false;
2844
2845 // Early exit if the offset is not possible to match. (6 bits of positive
2846 // range, plus allow an extra one in case we find a later insn that matches
2847 // with Offset-1)
2848 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI);
2849 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2850 int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
2851 // Allow one more for offset.
2852 if (Offset > 0)
2853 Offset -= OffsetStride;
2854 if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
2855 return false;
2856
2857 // Look ahead up to the scan limit for a pairable instruction.
2858 LdStPairFlags Flags;
2859 MachineBasicBlock::iterator Paired = findMatchingInsn(
2860 I: MBBI, Flags, Limit: Subtarget->getCLOpts().load_store_scan_limit,
2861 /*FindNarrowMerge=*/false);
2862
2863 if (Paired == E)
2864 return false;
2865
2866 // Keeping the iterator straight is a pain, so we let the merge routine tell
2867 // us what the next instruction is after it's done mucking about.
2868 auto Prev = std::prev(x: MBBI);
2869
2870 // Fetch the memoperand of the load/store that is a candidate for combination.
2871 MachineMemOperand *MemOp =
2872 MI.memoperands_empty() ? nullptr : MI.memoperands().front();
2873
2874 // If a load/store arrives and ldp/stp-aligned-only feature is opted, check
2875 // that the alignment of the source pointer is at least double the alignment
2876 // of the type.
2877 if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) ||
2878 (MI.mayStore() && Subtarget->hasStpAlignedOnly())) {
2879 // If there is no size/align information, cancel the transformation.
2880 if (!MemOp || !MemOp->getMemoryType().isValid()) {
2881 NumFailedAlignmentCheck++;
2882 return false;
2883 }
2884
2885 // Get the needed alignments to check them if
2886 // ldp-aligned-only/stp-aligned-only features are opted.
2887 uint64_t MemAlignment = MemOp->getAlign().value();
2888 uint64_t TypeAlignment =
2889 Align(MemOp->getSize().getValue().getKnownMinValue()).value();
2890
2891 if (MemAlignment < 2 * TypeAlignment) {
2892 NumFailedAlignmentCheck++;
2893 return false;
2894 }
2895 }
2896
2897 ++NumPairCreated;
2898 if (TII->hasUnscaledLdStOffset(MI))
2899 ++NumUnscaledPairCreated;
2900
2901 MBBI = mergePairedInsns(I: MBBI, Paired, Flags);
2902 // Collect liveness info for instructions between Prev and the new position
2903 // MBBI.
2904 for (auto I = std::next(x: Prev); I != MBBI; I++)
2905 updateDefinedRegisters(MI&: *I, Units&: DefinedInBB, TRI);
2906
2907 return true;
2908}
2909
2910bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2911 (MachineBasicBlock::iterator &MBBI) {
2912 MachineInstr &MI = *MBBI;
2913 MachineBasicBlock::iterator E = MI.getParent()->end();
2914 MachineBasicBlock::iterator Update;
2915
2916 // Do not form post-inc addressing mode for volatile accesses. Instructions
2917 // performing register writeback do not set a valid instruction syndrome,
2918 // making it impossible to handle MMIO in protected hypervisors.
2919 // Exclude accesses based on the stack pointer, as these can't be MMIO.
2920 // Also exclude MTE tag store instructions.
2921 if (MBBI->hasOrderedMemoryRef() &&
2922 AArch64InstrInfo::getLdStBaseOp(MI).getReg() != AArch64::SP &&
2923 !isTagStore(MI) && MI.getOpcode() != AArch64::STGPi)
2924 return false;
2925
2926 // Look forward to try to form a post-index instruction. For example,
2927 // ldr x0, [x20]
2928 // add x20, x20, #32
2929 // merged into:
2930 // ldr x0, [x20], #32
2931 Update = findMatchingUpdateInsnForward(
2932 I: MBBI, UnscaledOffset: 0, Limit: Subtarget->getCLOpts().update_scan_limit);
2933 if (Update != E) {
2934 // Merge the update into the ld/st.
2935 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/false,
2936 /*IsPreIdx=*/false,
2937 /*MergeEither=*/false)) {
2938 MBBI = *NextI;
2939 return true;
2940 }
2941 }
2942
2943 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2944 if (TII->hasUnscaledLdStOffset(Opc: MI.getOpcode()))
2945 return false;
2946
2947 // Look back to try to find a pre-index instruction. For example,
2948 // add x0, x0, #8
2949 // ldr x1, [x0]
2950 // merged into:
2951 // ldr x1, [x0, #8]!
2952 bool MergeEither;
2953 Update = findMatchingUpdateInsnBackward(
2954 I: MBBI, Limit: Subtarget->getCLOpts().update_scan_limit, MergeEither);
2955 if (Update != E) {
2956 // Merge the update into the ld/st.
2957 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/true,
2958 /*IsPreIdx=*/true, MergeEither)) {
2959 MBBI = *NextI;
2960 return true;
2961 }
2962 }
2963
2964 // The immediate in the load/store is scaled by the size of the memory
2965 // operation. The immediate in the add we're looking for,
2966 // however, is not, so adjust here.
2967 int UnscaledOffset =
2968 AArch64InstrInfo::getLdStOffsetOp(MI).getImm() * TII->getMemScale(MI);
2969
2970 // Look forward to try to find a pre-index instruction. For example,
2971 // ldr x1, [x0, #64]
2972 // add x0, x0, #64
2973 // merged into:
2974 // ldr x1, [x0, #64]!
2975 Update = findMatchingUpdateInsnForward(
2976 I: MBBI, UnscaledOffset, Limit: Subtarget->getCLOpts().update_scan_limit);
2977 if (Update != E) {
2978 // Merge the update into the ld/st.
2979 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/false,
2980 /*IsPreIdx=*/true,
2981 /*MergeEither=*/false)) {
2982 MBBI = *NextI;
2983 return true;
2984 }
2985 }
2986
2987 return false;
2988}
2989
2990bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
2991 int Scale) {
2992 MachineInstr &MI = *MBBI;
2993 MachineBasicBlock::iterator E = MI.getParent()->end();
2994 MachineBasicBlock::iterator Update;
2995
2996 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2997 if (TII->hasUnscaledLdStOffset(Opc: MI.getOpcode()))
2998 return false;
2999
3000 // Look back to try to find a const offset for index LdSt instruction. For
3001 // example,
3002 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3003 // ldr x1, [x0, x8]
3004 // merged into:
3005 // add x8, x0, a * (1<<12)
3006 // ldr x1, [x8, imm12]
3007 unsigned Offset;
3008 Update = findMatchingConstOffsetBackward(
3009 I: MBBI, Limit: Subtarget->getCLOpts().load_store_const_scan_limit, Offset);
3010 if (Update != E && (Offset & (Scale - 1)) == 0) {
3011 // Merge the imm12 into the ld/st.
3012 MBBI = mergeConstOffsetInsn(I: MBBI, Update, Offset, Scale);
3013 return true;
3014 }
3015
3016 return false;
3017}
3018
3019// Map a GPR store opcode to its FPR equivalent at the same data width.
3020// Returns 0 if no mapping exists.
3021static unsigned getGPRToFPRStoreOpcode(unsigned GPRStoreOpc) {
3022 switch (GPRStoreOpc) {
3023 // Unsigned immediate.
3024 case AArch64::STRBBui:
3025 return AArch64::STRBui;
3026 case AArch64::STRHHui:
3027 return AArch64::STRHui;
3028 case AArch64::STRWui:
3029 return AArch64::STRSui;
3030 case AArch64::STRXui:
3031 return AArch64::STRDui;
3032 // Unscaled immediate.
3033 case AArch64::STURBBi:
3034 return AArch64::STURBi;
3035 case AArch64::STURHHi:
3036 return AArch64::STURHi;
3037 case AArch64::STURWi:
3038 return AArch64::STURSi;
3039 case AArch64::STURXi:
3040 return AArch64::STURDi;
3041 // Register offset.
3042 case AArch64::STRBBroW:
3043 return AArch64::STRBroW;
3044 case AArch64::STRBBroX:
3045 return AArch64::STRBroX;
3046 case AArch64::STRHHroW:
3047 return AArch64::STRHroW;
3048 case AArch64::STRHHroX:
3049 return AArch64::STRHroX;
3050 case AArch64::STRWroW:
3051 return AArch64::STRSroW;
3052 case AArch64::STRWroX:
3053 return AArch64::STRSroX;
3054 case AArch64::STRXroW:
3055 return AArch64::STRDroW;
3056 case AArch64::STRXroX:
3057 return AArch64::STRDroX;
3058 default:
3059 return 0;
3060 }
3061}
3062
3063// Given a UMOV-lane-0 opcode, return the sub-register index to extract from
3064// the vector register, or 0 if the opcode is not a supported UMOV.
3065static unsigned getUMOVSubRegIdx(unsigned UMOVOpc) {
3066 switch (UMOVOpc) {
3067 case AArch64::UMOVvi8_idx0:
3068 return AArch64::bsub;
3069 case AArch64::UMOVvi16_idx0:
3070 return AArch64::hsub;
3071 case AArch64::UMOVvi32_idx0:
3072 return AArch64::ssub;
3073 case AArch64::UMOVvi64_idx0:
3074 return AArch64::dsub;
3075 default:
3076 return 0;
3077 }
3078}
3079
3080bool AArch64LoadStoreOpt::tryToReplaceUMOVStore(
3081 MachineBasicBlock::iterator &MBBI) {
3082 MachineInstr &StoreMI = *MBBI;
3083
3084 unsigned FPRStoreOpc = getGPRToFPRStoreOpcode(GPRStoreOpc: StoreMI.getOpcode());
3085 if (!FPRStoreOpc)
3086 return false;
3087
3088 if (StoreMI.hasOrderedMemoryRef() || StoreMI.memoperands().size() != 1)
3089 return false;
3090
3091 MachineBasicBlock *MBB = StoreMI.getParent();
3092 MCPhysReg StoreValReg = StoreMI.getOperand(i: 0).getReg();
3093
3094 if (!StoreMI.getOperand(i: 0).isKill())
3095 return false;
3096
3097 // Bail out if the store uses the value register elsewhere (e.g., as the base
3098 // address in `str w8, [x8, #0]`).
3099 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3100 if (StoreMI.getOperand(i: I).isReg() &&
3101 TRI->regsOverlap(RegA: StoreMI.getOperand(i: I).getReg(), RegB: StoreValReg))
3102 return false;
3103
3104 // Scan backward to find the UMOV that defines the store's value register.
3105 MachineInstr *UMOVMI = nullptr;
3106 MachineBasicBlock::iterator B = MBB->begin();
3107 unsigned SubRegIdx = 0;
3108 unsigned Count = 0;
3109 for (auto It = MBBI; It != B;) {
3110 MachineInstr &MI = *--It;
3111 if (MI.isDebugInstr())
3112 continue;
3113 if (++Count > Subtarget->getCLOpts().umov_fold_scan_limit)
3114 return false;
3115 if (MI.readsRegister(Reg: StoreValReg, TRI))
3116 return false;
3117 if (MI.modifiesRegister(Reg: StoreValReg, TRI)) {
3118 SubRegIdx = getUMOVSubRegIdx(UMOVOpc: MI.getOpcode());
3119 if (!SubRegIdx)
3120 return false;
3121 UMOVMI = &MI;
3122 break;
3123 }
3124 }
3125 if (!UMOVMI)
3126 return false;
3127 MCPhysReg VecReg = UMOVMI->getOperand(i: 1).getReg();
3128 MCPhysReg FPRReg = TRI->getSubReg(Reg: VecReg, Idx: SubRegIdx);
3129 if ((*StoreMI.memoperands_begin())->getSizeInBits() !=
3130 TRI->getRegSizeInBits(RC: *TRI->getMinimalPhysRegClass(Reg: FPRReg)))
3131 return false;
3132
3133 // Check that no instruction between UMOV and store clobbers the vector
3134 // register. Also track whether VecReg is killed anywhere from the UMOV
3135 // (inclusive) through the intervening instructions -- we need this to decide
3136 // whether the FPR sub-register can be marked killed on the new store.
3137 bool VecRegKilled = UMOVMI->killsRegister(Reg: VecReg, TRI);
3138 for (auto It = std::next(x: UMOVMI->getIterator()); It != MBBI; ++It) {
3139 if (It->modifiesRegister(Reg: VecReg, TRI))
3140 return false;
3141 if (!VecRegKilled && It->killsRegister(Reg: VecReg, TRI))
3142 VecRegKilled = true;
3143 }
3144
3145 // Safe to proceed. Clear kill flags on the vector register between UMOV and
3146 // the new store so the FPR sub-register stays live.
3147 UMOVMI->clearRegisterKills(Reg: VecReg, RegInfo: TRI);
3148 for (auto It = std::next(x: UMOVMI->getIterator()); It != MBBI; ++It)
3149 It->clearRegisterKills(Reg: VecReg, RegInfo: TRI);
3150
3151 LLVM_DEBUG(dbgs() << "Folding UMOV + store: " << *UMOVMI << " + "
3152 << StoreMI);
3153
3154 auto MIB = BuildMI(BB&: *MBB, I: MBBI, MIMD: StoreMI.getDebugLoc(), MCID: TII->get(Opcode: FPRStoreOpc))
3155 .addReg(RegNo: FPRReg, Flags: getKillRegState(B: VecRegKilled));
3156 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3157 MIB.add(MO: StoreMI.getOperand(i: I));
3158 MIB.setMemRefs(StoreMI.memoperands());
3159
3160 MBBI = MBB->erase(I: MBBI);
3161 UMOVMI->eraseFromParent();
3162
3163 ++NumUMOVFoldedToFPRStore;
3164 return true;
3165}
3166
3167bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
3168 bool EnableNarrowZeroStOpt) {
3169 AArch64FunctionInfo &AFI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
3170
3171 bool Modified = false;
3172 // Six transformations to do here:
3173 // 1) Find loads that directly read from stores and promote them by
3174 // replacing with mov instructions. If the store is wider than the load,
3175 // the load will be replaced with a bitfield extract.
3176 // e.g.,
3177 // str w1, [x0, #4]
3178 // ldrh w2, [x0, #6]
3179 // ; becomes
3180 // str w1, [x0, #4]
3181 // lsr w2, w1, #16
3182 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3183 MBBI != E;) {
3184 if (isPromotableLoadFromStore(MI&: *MBBI) && tryToPromoteLoadFromStore(MBBI))
3185 Modified = true;
3186 else
3187 ++MBBI;
3188 }
3189 // 2) Merge adjacent zero stores into a wider store.
3190 // e.g.,
3191 // strh wzr, [x0]
3192 // strh wzr, [x0, #2]
3193 // ; becomes
3194 // str wzr, [x0]
3195 // e.g.,
3196 // str wzr, [x0]
3197 // str wzr, [x0, #4]
3198 // ; becomes
3199 // str xzr, [x0]
3200 if (EnableNarrowZeroStOpt)
3201 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3202 MBBI != E;) {
3203 if (isPromotableZeroStoreInst(MI&: *MBBI) && tryToMergeZeroStInst(MBBI))
3204 Modified = true;
3205 else
3206 ++MBBI;
3207 }
3208 // 3) Find loads and stores that can be merged into a single load or store
3209 // pair instruction.
3210 // When compiling for SVE 128, also try to combine SVE fill/spill
3211 // instructions into LDP/STP.
3212 // e.g.,
3213 // ldr x0, [x2]
3214 // ldr x1, [x2, #8]
3215 // ; becomes
3216 // ldp x0, x1, [x2]
3217 // e.g.,
3218 // ldr z0, [x2]
3219 // ldr z1, [x2, #1, mul vl]
3220 // ; becomes
3221 // ldp q0, q1, [x2]
3222
3223 if (MBB.getParent()->getRegInfo().tracksLiveness()) {
3224 DefinedInBB.clear();
3225 DefinedInBB.addLiveIns(MBB);
3226 }
3227
3228 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3229 MBBI != E;) {
3230 // Track currently live registers up to this point, to help with
3231 // searching for a rename register on demand.
3232 updateDefinedRegisters(MI&: *MBBI, Units&: DefinedInBB, TRI);
3233 if (TII->isPairableLdStInst(MI: *MBBI) && tryToPairLdStInst(MBBI))
3234 Modified = true;
3235 else
3236 ++MBBI;
3237 }
3238 // 4) Find base register updates that can be merged into the load or store
3239 // as a base-reg writeback.
3240 // e.g.,
3241 // ldr x0, [x2]
3242 // add x2, x2, #4
3243 // ; becomes
3244 // ldr x0, [x2], #4
3245 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3246 MBBI != E;) {
3247 if (isMergeableLdStUpdate(MI&: *MBBI, AFI) && tryToMergeLdStUpdate(MBBI))
3248 Modified = true;
3249 else
3250 ++MBBI;
3251 }
3252
3253 // 5) Find a register assigned with a const value that can be combined with
3254 // into the load or store. e.g.,
3255 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3256 // ldr x1, [x0, x8]
3257 // ; becomes
3258 // add x8, x0, a * (1<<12)
3259 // ldr x1, [x8, imm12]
3260 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3261 MBBI != E;) {
3262 int Scale;
3263 if (isMergeableIndexLdSt(MI&: *MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale))
3264 Modified = true;
3265 else
3266 ++MBBI;
3267 }
3268
3269 // 6) Replace UMOV (lane 0) + GPR store with a direct FPR sub-register store.
3270 // e.g.,
3271 // umov w8, v0.h[0]
3272 // strh w8, [x0]
3273 // ; becomes
3274 // str h0, [x0]
3275 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3276 MBBI != E;) {
3277 if (tryToReplaceUMOVStore(MBBI))
3278 Modified = true;
3279 else
3280 ++MBBI;
3281 }
3282
3283 return Modified;
3284}
3285
3286bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
3287 Subtarget = &Fn.getSubtarget<AArch64Subtarget>();
3288 TII = Subtarget->getInstrInfo();
3289 TRI = Subtarget->getRegisterInfo();
3290
3291 // Resize the modified and used register unit trackers. We do this once
3292 // per function and then clear the register units each time we optimize a load
3293 // or store.
3294 ModifiedRegUnits.init(TRI: *TRI);
3295 UsedRegUnits.init(TRI: *TRI);
3296 DefinedInBB.init(TRI: *TRI);
3297
3298 bool Modified = false;
3299 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
3300 for (auto &MBB : Fn) {
3301 auto M = optimizeBlock(MBB, EnableNarrowZeroStOpt: enableNarrowZeroStOpt);
3302 Modified |= M;
3303 }
3304
3305 return Modified;
3306}
3307
3308// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and
3309// stores near one another? Note: The pre-RA instruction scheduler already has
3310// hooks to try and schedule pairable loads/stores together to improve pairing
3311// opportunities. Thus, pre-RA pairing pass may not be worth the effort.
3312
3313// FIXME: When pairing store instructions it's very possible for this pass to
3314// hoist a store with a KILL marker above another use (without a KILL marker).
3315// The resulting IR is invalid, but nothing uses the KILL markers after this
3316// pass, so it's never caused a problem in practice.
3317
3318bool AArch64LoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
3319 if (skipFunction(F: MF.getFunction()))
3320 return false;
3321 AArch64LoadStoreOpt Impl;
3322 Impl.AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3323 return Impl.runOnMachineFunction(Fn&: MF);
3324}
3325
3326/// createAArch64LoadStoreOptimizationPass - returns an instance of the
3327/// load / store optimization pass.
3328FunctionPass *llvm::createAArch64LoadStoreOptLegacyPass() {
3329 return new AArch64LoadStoreOptLegacy();
3330}
3331
3332PreservedAnalyses
3333AArch64LoadStoreOptPass::run(MachineFunction &MF,
3334 MachineFunctionAnalysisManager &MFAM) {
3335 AArch64LoadStoreOpt Impl;
3336 Impl.AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3337 .getManager()
3338 .getResult<AAManager>(IR&: MF.getFunction());
3339 bool Changed = Impl.runOnMachineFunction(Fn&: MF);
3340 if (!Changed)
3341 return PreservedAnalyses::all();
3342 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
3343 PA.preserveSet<CFGAnalyses>();
3344 return PA;
3345}
3346