1//===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs load / store related peephole
10// optimizations. This pass should be run after register allocation.
11//
12// The pass runs after the PrologEpilogInserter where we emit the CFI
13// instructions. In order to preserve the correctness of the unwind information,
14// the pass should not change the order of any two instructions, one of which
15// has the FrameSetup/FrameDestroy flag or, alternatively, apply an add-hoc fix
16// to unwind information.
17//
18//===----------------------------------------------------------------------===//
19
20#include "AArch64InstrInfo.h"
21#include "AArch64MachineFunctionInfo.h"
22#include "AArch64Subtarget.h"
23#include "MCTargetDesc/AArch64AddressingModes.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineFunctionPass.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineOperand.h"
36#include "llvm/CodeGen/MachineRegisterInfo.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCDwarf.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/DebugCounter.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <functional>
48#include <iterator>
49#include <limits>
50#include <optional>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "aarch64-ldst-opt"
55
56STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
57STATISTIC(NumPostFolded, "Number of post-index updates folded");
58STATISTIC(NumPreFolded, "Number of pre-index updates folded");
59STATISTIC(NumUnscaledPairCreated,
60 "Number of load/store from unscaled generated");
61STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
62STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
63STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation "
64 "not passed the alignment check");
65STATISTIC(NumConstOffsetFolded,
66 "Number of const offset of index address folded");
67STATISTIC(NumUMOVFoldedToFPRStore,
68 "Number of UMOV + GPR stores folded to FPR stores");
69
70DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
71 "Controls which pairs are considered for renaming");
72
73#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
74
75namespace {
76
77using LdStPairFlags = struct LdStPairFlags {
78 // If a matching instruction is found, MergeForward is set to true if the
79 // merge is to remove the first instruction and replace the second with
80 // a pair-wise insn, and false if the reverse is true.
81 bool MergeForward = false;
82
83 // SExtIdx gives the index of the result of the load pair that must be
84 // extended. The value of SExtIdx assumes that the paired load produces the
85 // value in this order: (I, returned iterator), i.e., -1 means no value has
86 // to be extended, 0 means I, and 1 means the returned iterator.
87 int SExtIdx = -1;
88
89 // If not none, RenameReg can be used to rename the result register of the
90 // first store in a pair. Currently this only works when merging stores
91 // forward.
92 std::optional<MCPhysReg> RenameReg;
93
94 LdStPairFlags() = default;
95
96 void setMergeForward(bool V = true) { MergeForward = V; }
97 bool getMergeForward() const { return MergeForward; }
98
99 void setSExtIdx(int V) { SExtIdx = V; }
100 int getSExtIdx() const { return SExtIdx; }
101
102 void setRenameReg(MCPhysReg R) { RenameReg = R; }
103 void clearRenameReg() { RenameReg = std::nullopt; }
104 std::optional<MCPhysReg> getRenameReg() const { return RenameReg; }
105};
106
107struct AArch64LoadStoreOpt {
108 AliasAnalysis *AA;
109 const AArch64InstrInfo *TII;
110 const TargetRegisterInfo *TRI;
111 const AArch64Subtarget *Subtarget;
112
113 // Track which register units have been modified and used.
114 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
115 LiveRegUnits DefinedInBB;
116
117 // Scan the instructions looking for a load/store that can be combined
118 // with the current instruction into a load/store pair.
119 // Return the matching instruction if one is found, else MBB->end().
120 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
121 LdStPairFlags &Flags,
122 unsigned Limit,
123 bool FindNarrowMerge);
124
125 // Scan the instructions looking for a store that writes to the address from
126 // which the current load instruction reads. Return true if one is found.
127 bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
128 MachineBasicBlock::iterator &StoreI);
129
130 // Merge the two instructions indicated into a wider narrow store instruction.
131 MachineBasicBlock::iterator
132 mergeNarrowZeroStores(MachineBasicBlock::iterator I,
133 MachineBasicBlock::iterator MergeMI,
134 const LdStPairFlags &Flags);
135
136 // Merge the two instructions indicated into a single pair-wise instruction.
137 MachineBasicBlock::iterator
138 mergePairedInsns(MachineBasicBlock::iterator I,
139 MachineBasicBlock::iterator Paired,
140 const LdStPairFlags &Flags);
141
142 // Promote the load that reads directly from the address stored to.
143 MachineBasicBlock::iterator
144 promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
145 MachineBasicBlock::iterator StoreI);
146
147 // Scan the instruction list to find a base register update that can
148 // be combined with the current instruction (a load or store) using
149 // pre or post indexed addressing with writeback. Scan forwards.
150 MachineBasicBlock::iterator
151 findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
152 int UnscaledOffset, unsigned Limit);
153
154 // Scan the instruction list to find a register assigned with a const
155 // value that can be combined with the current instruction (a load or store)
156 // using base addressing with writeback. Scan backwards.
157 MachineBasicBlock::iterator
158 findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit,
159 unsigned &Offset);
160
161 // Scan the instruction list to find a base register update that can
162 // be combined with the current instruction (a load or store) using
163 // pre or post indexed addressing with writeback. Scan backwards.
164 // `MergeEither` is set to true if the combined instruction may be placed
165 // either at the location of the load/store instruction or at the location of
166 // the update instruction.
167 MachineBasicBlock::iterator
168 findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit,
169 bool &MergeEither);
170
171 // Find an instruction that updates the base register of the ld/st
172 // instruction.
173 bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
174 unsigned BaseReg, int Offset);
175
176 bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI,
177 unsigned IndexReg, unsigned &Offset);
178
179 // Merge a pre- or post-index base register update into a ld/st instruction.
180 std::optional<MachineBasicBlock::iterator>
181 mergeUpdateInsn(MachineBasicBlock::iterator I,
182 MachineBasicBlock::iterator Update, bool IsForward,
183 bool IsPreIdx, bool MergeEither);
184
185 MachineBasicBlock::iterator
186 mergeConstOffsetInsn(MachineBasicBlock::iterator I,
187 MachineBasicBlock::iterator Update, unsigned Offset,
188 int Scale);
189
190 // Find and merge zero store instructions.
191 bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
192
193 // Find and pair ldr/str instructions.
194 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
195
196 // Find and promote load instructions which read directly from store.
197 bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
198
199 // Find and merge a base register updates before or after a ld/st instruction.
200 bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
201
202 // Find and merge an index ldr/st instruction into a base ld/st instruction.
203 bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale);
204
205 // Replace a UMOV (lane 0) + GPR store with a direct FPR sub-register store.
206 bool tryToReplaceUMOVStore(MachineBasicBlock::iterator &MBBI);
207
208 bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
209
210 bool runOnMachineFunction(MachineFunction &MF);
211};
212
213struct AArch64LoadStoreOptLegacy : public MachineFunctionPass {
214 static char ID;
215
216 AArch64LoadStoreOptLegacy() : MachineFunctionPass(ID) {}
217
218 bool runOnMachineFunction(MachineFunction &Fn) override;
219
220 void getAnalysisUsage(AnalysisUsage &AU) const override {
221 AU.addRequired<AAResultsWrapperPass>();
222 MachineFunctionPass::getAnalysisUsage(AU);
223 }
224
225 MachineFunctionProperties getRequiredProperties() const override {
226 return MachineFunctionProperties().setNoVRegs();
227 }
228
229 StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; }
230};
231
232char AArch64LoadStoreOptLegacy::ID = 0;
233
234} // end anonymous namespace
235
236INITIALIZE_PASS(AArch64LoadStoreOptLegacy, "aarch64-ldst-opt",
237 AARCH64_LOAD_STORE_OPT_NAME, false, false)
238
239static bool isNarrowStore(unsigned Opc) {
240 switch (Opc) {
241 default:
242 return false;
243 case AArch64::STRBBui:
244 case AArch64::STURBBi:
245 case AArch64::STRHHui:
246 case AArch64::STURHHi:
247 return true;
248 }
249}
250
251// These instruction set memory tag and either keep memory contents unchanged or
252// set it to zero, ignoring the address part of the source register.
253static bool isTagStore(const MachineInstr &MI) {
254 switch (MI.getOpcode()) {
255 default:
256 return false;
257 case AArch64::STGi:
258 case AArch64::STZGi:
259 case AArch64::ST2Gi:
260 case AArch64::STZ2Gi:
261 return true;
262 }
263}
264
265static unsigned getMatchingNonSExtOpcode(unsigned Opc,
266 bool *IsValidLdStrOpc = nullptr) {
267 if (IsValidLdStrOpc)
268 *IsValidLdStrOpc = true;
269 switch (Opc) {
270 default:
271 if (IsValidLdStrOpc)
272 *IsValidLdStrOpc = false;
273 return std::numeric_limits<unsigned>::max();
274 case AArch64::STRDui:
275 case AArch64::STURDi:
276 case AArch64::STRDpre:
277 case AArch64::STRQui:
278 case AArch64::STURQi:
279 case AArch64::STRQpre:
280 case AArch64::STRBBui:
281 case AArch64::STURBBi:
282 case AArch64::STRHHui:
283 case AArch64::STURHHi:
284 case AArch64::STRWui:
285 case AArch64::STRWpre:
286 case AArch64::STURWi:
287 case AArch64::STRXui:
288 case AArch64::STRXpre:
289 case AArch64::STURXi:
290 case AArch64::STR_ZXI:
291 case AArch64::LDRDui:
292 case AArch64::LDURDi:
293 case AArch64::LDRDpre:
294 case AArch64::LDRQui:
295 case AArch64::LDURQi:
296 case AArch64::LDRQpre:
297 case AArch64::LDRWui:
298 case AArch64::LDURWi:
299 case AArch64::LDRWpre:
300 case AArch64::LDRXui:
301 case AArch64::LDURXi:
302 case AArch64::LDRXpre:
303 case AArch64::STRSui:
304 case AArch64::STURSi:
305 case AArch64::STRSpre:
306 case AArch64::LDRSui:
307 case AArch64::LDURSi:
308 case AArch64::LDRSpre:
309 case AArch64::LDR_ZXI:
310 return Opc;
311 case AArch64::LDRSWui:
312 return AArch64::LDRWui;
313 case AArch64::LDURSWi:
314 return AArch64::LDURWi;
315 case AArch64::LDRSWpre:
316 return AArch64::LDRWpre;
317 }
318}
319
320static unsigned getMatchingWideOpcode(unsigned Opc) {
321 switch (Opc) {
322 default:
323 llvm_unreachable("Opcode has no wide equivalent!");
324 case AArch64::STRBBui:
325 return AArch64::STRHHui;
326 case AArch64::STRHHui:
327 return AArch64::STRWui;
328 case AArch64::STURBBi:
329 return AArch64::STURHHi;
330 case AArch64::STURHHi:
331 return AArch64::STURWi;
332 case AArch64::STURWi:
333 return AArch64::STURXi;
334 case AArch64::STRWui:
335 return AArch64::STRXui;
336 }
337}
338
339static unsigned getMatchingPairOpcode(unsigned Opc) {
340 switch (Opc) {
341 default:
342 llvm_unreachable("Opcode has no pairwise equivalent!");
343 case AArch64::STRSui:
344 case AArch64::STURSi:
345 return AArch64::STPSi;
346 case AArch64::STRSpre:
347 return AArch64::STPSpre;
348 case AArch64::STRDui:
349 case AArch64::STURDi:
350 return AArch64::STPDi;
351 case AArch64::STRDpre:
352 return AArch64::STPDpre;
353 case AArch64::STRQui:
354 case AArch64::STURQi:
355 case AArch64::STR_ZXI:
356 return AArch64::STPQi;
357 case AArch64::STRQpre:
358 return AArch64::STPQpre;
359 case AArch64::STRWui:
360 case AArch64::STURWi:
361 return AArch64::STPWi;
362 case AArch64::STRWpre:
363 return AArch64::STPWpre;
364 case AArch64::STRXui:
365 case AArch64::STURXi:
366 return AArch64::STPXi;
367 case AArch64::STRXpre:
368 return AArch64::STPXpre;
369 case AArch64::LDRSui:
370 case AArch64::LDURSi:
371 return AArch64::LDPSi;
372 case AArch64::LDRSpre:
373 return AArch64::LDPSpre;
374 case AArch64::LDRDui:
375 case AArch64::LDURDi:
376 return AArch64::LDPDi;
377 case AArch64::LDRDpre:
378 return AArch64::LDPDpre;
379 case AArch64::LDRQui:
380 case AArch64::LDURQi:
381 case AArch64::LDR_ZXI:
382 return AArch64::LDPQi;
383 case AArch64::LDRQpre:
384 return AArch64::LDPQpre;
385 case AArch64::LDRWui:
386 case AArch64::LDURWi:
387 return AArch64::LDPWi;
388 case AArch64::LDRWpre:
389 return AArch64::LDPWpre;
390 case AArch64::LDRXui:
391 case AArch64::LDURXi:
392 return AArch64::LDPXi;
393 case AArch64::LDRXpre:
394 return AArch64::LDPXpre;
395 case AArch64::LDRSWui:
396 case AArch64::LDURSWi:
397 return AArch64::LDPSWi;
398 case AArch64::LDRSWpre:
399 return AArch64::LDPSWpre;
400 }
401}
402
403static unsigned isMatchingStore(MachineInstr &LoadInst,
404 MachineInstr &StoreInst) {
405 unsigned LdOpc = LoadInst.getOpcode();
406 unsigned StOpc = StoreInst.getOpcode();
407 switch (LdOpc) {
408 default:
409 llvm_unreachable("Unsupported load instruction!");
410 case AArch64::LDRBBui:
411 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
412 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
413 case AArch64::LDURBBi:
414 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
415 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
416 case AArch64::LDRHHui:
417 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
418 StOpc == AArch64::STRXui;
419 case AArch64::LDURHHi:
420 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
421 StOpc == AArch64::STURXi;
422 case AArch64::LDRWui:
423 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
424 case AArch64::LDURWi:
425 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
426 case AArch64::LDRXui:
427 return StOpc == AArch64::STRXui;
428 case AArch64::LDURXi:
429 return StOpc == AArch64::STURXi;
430 }
431}
432
433static unsigned getPreIndexedOpcode(unsigned Opc) {
434 // FIXME: We don't currently support creating pre-indexed loads/stores when
435 // the load or store is the unscaled version. If we decide to perform such an
436 // optimization in the future the cases for the unscaled loads/stores will
437 // need to be added here.
438 switch (Opc) {
439 default:
440 llvm_unreachable("Opcode has no pre-indexed equivalent!");
441 case AArch64::STRBui:
442 return AArch64::STRBpre;
443 case AArch64::STRHui:
444 return AArch64::STRHpre;
445 case AArch64::STRSui:
446 return AArch64::STRSpre;
447 case AArch64::STRDui:
448 return AArch64::STRDpre;
449 case AArch64::STRQui:
450 return AArch64::STRQpre;
451 case AArch64::STRBBui:
452 return AArch64::STRBBpre;
453 case AArch64::STRHHui:
454 return AArch64::STRHHpre;
455 case AArch64::STRWui:
456 return AArch64::STRWpre;
457 case AArch64::STRXui:
458 return AArch64::STRXpre;
459 case AArch64::LDRBui:
460 return AArch64::LDRBpre;
461 case AArch64::LDRHui:
462 return AArch64::LDRHpre;
463 case AArch64::LDRSui:
464 return AArch64::LDRSpre;
465 case AArch64::LDRDui:
466 return AArch64::LDRDpre;
467 case AArch64::LDRQui:
468 return AArch64::LDRQpre;
469 case AArch64::LDRBBui:
470 return AArch64::LDRBBpre;
471 case AArch64::LDRHHui:
472 return AArch64::LDRHHpre;
473 case AArch64::LDRWui:
474 return AArch64::LDRWpre;
475 case AArch64::LDRXui:
476 return AArch64::LDRXpre;
477 case AArch64::LDRSWui:
478 return AArch64::LDRSWpre;
479 case AArch64::LDPSi:
480 return AArch64::LDPSpre;
481 case AArch64::LDPSWi:
482 return AArch64::LDPSWpre;
483 case AArch64::LDPDi:
484 return AArch64::LDPDpre;
485 case AArch64::LDPQi:
486 return AArch64::LDPQpre;
487 case AArch64::LDPWi:
488 return AArch64::LDPWpre;
489 case AArch64::LDPXi:
490 return AArch64::LDPXpre;
491 case AArch64::STPSi:
492 return AArch64::STPSpre;
493 case AArch64::STPDi:
494 return AArch64::STPDpre;
495 case AArch64::STPQi:
496 return AArch64::STPQpre;
497 case AArch64::STPWi:
498 return AArch64::STPWpre;
499 case AArch64::STPXi:
500 return AArch64::STPXpre;
501 case AArch64::STGi:
502 return AArch64::STGPreIndex;
503 case AArch64::STZGi:
504 return AArch64::STZGPreIndex;
505 case AArch64::ST2Gi:
506 return AArch64::ST2GPreIndex;
507 case AArch64::STZ2Gi:
508 return AArch64::STZ2GPreIndex;
509 case AArch64::STGPi:
510 return AArch64::STGPpre;
511 }
512}
513
514static unsigned getBaseAddressOpcode(unsigned Opc) {
515 // TODO: Add more index address stores.
516 switch (Opc) {
517 default:
518 llvm_unreachable("Opcode has no base address equivalent!");
519 case AArch64::LDRBroX:
520 return AArch64::LDRBui;
521 case AArch64::LDRBBroX:
522 return AArch64::LDRBBui;
523 case AArch64::LDRSBXroX:
524 return AArch64::LDRSBXui;
525 case AArch64::LDRSBWroX:
526 return AArch64::LDRSBWui;
527 case AArch64::LDRHroX:
528 return AArch64::LDRHui;
529 case AArch64::LDRHHroX:
530 return AArch64::LDRHHui;
531 case AArch64::LDRSHXroX:
532 return AArch64::LDRSHXui;
533 case AArch64::LDRSHWroX:
534 return AArch64::LDRSHWui;
535 case AArch64::LDRWroX:
536 return AArch64::LDRWui;
537 case AArch64::LDRSroX:
538 return AArch64::LDRSui;
539 case AArch64::LDRSWroX:
540 return AArch64::LDRSWui;
541 case AArch64::LDRDroX:
542 return AArch64::LDRDui;
543 case AArch64::LDRXroX:
544 return AArch64::LDRXui;
545 case AArch64::LDRQroX:
546 return AArch64::LDRQui;
547 }
548}
549
550static unsigned getPostIndexedOpcode(unsigned Opc) {
551 switch (Opc) {
552 default:
553 llvm_unreachable("Opcode has no post-indexed wise equivalent!");
554 case AArch64::STRBui:
555 return AArch64::STRBpost;
556 case AArch64::STRHui:
557 return AArch64::STRHpost;
558 case AArch64::STRSui:
559 case AArch64::STURSi:
560 return AArch64::STRSpost;
561 case AArch64::STRDui:
562 case AArch64::STURDi:
563 return AArch64::STRDpost;
564 case AArch64::STRQui:
565 case AArch64::STURQi:
566 return AArch64::STRQpost;
567 case AArch64::STRBBui:
568 return AArch64::STRBBpost;
569 case AArch64::STRHHui:
570 return AArch64::STRHHpost;
571 case AArch64::STRWui:
572 case AArch64::STURWi:
573 return AArch64::STRWpost;
574 case AArch64::STRXui:
575 case AArch64::STURXi:
576 return AArch64::STRXpost;
577 case AArch64::LDRBui:
578 return AArch64::LDRBpost;
579 case AArch64::LDRHui:
580 return AArch64::LDRHpost;
581 case AArch64::LDRSui:
582 case AArch64::LDURSi:
583 return AArch64::LDRSpost;
584 case AArch64::LDRDui:
585 case AArch64::LDURDi:
586 return AArch64::LDRDpost;
587 case AArch64::LDRQui:
588 case AArch64::LDURQi:
589 return AArch64::LDRQpost;
590 case AArch64::LDRBBui:
591 return AArch64::LDRBBpost;
592 case AArch64::LDRHHui:
593 return AArch64::LDRHHpost;
594 case AArch64::LDRWui:
595 case AArch64::LDURWi:
596 return AArch64::LDRWpost;
597 case AArch64::LDRXui:
598 case AArch64::LDURXi:
599 return AArch64::LDRXpost;
600 case AArch64::LDRSWui:
601 return AArch64::LDRSWpost;
602 case AArch64::LDPSi:
603 return AArch64::LDPSpost;
604 case AArch64::LDPSWi:
605 return AArch64::LDPSWpost;
606 case AArch64::LDPDi:
607 return AArch64::LDPDpost;
608 case AArch64::LDPQi:
609 return AArch64::LDPQpost;
610 case AArch64::LDPWi:
611 return AArch64::LDPWpost;
612 case AArch64::LDPXi:
613 return AArch64::LDPXpost;
614 case AArch64::STPSi:
615 return AArch64::STPSpost;
616 case AArch64::STPDi:
617 return AArch64::STPDpost;
618 case AArch64::STPQi:
619 return AArch64::STPQpost;
620 case AArch64::STPWi:
621 return AArch64::STPWpost;
622 case AArch64::STPXi:
623 return AArch64::STPXpost;
624 case AArch64::STGi:
625 return AArch64::STGPostIndex;
626 case AArch64::STZGi:
627 return AArch64::STZGPostIndex;
628 case AArch64::ST2Gi:
629 return AArch64::ST2GPostIndex;
630 case AArch64::STZ2Gi:
631 return AArch64::STZ2GPostIndex;
632 case AArch64::STGPi:
633 return AArch64::STGPpost;
634 }
635}
636
637static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI) {
638
639 unsigned OpcA = FirstMI.getOpcode();
640 unsigned OpcB = MI.getOpcode();
641
642 switch (OpcA) {
643 default:
644 return false;
645 case AArch64::STRSpre:
646 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
647 case AArch64::STRDpre:
648 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
649 case AArch64::STRQpre:
650 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
651 case AArch64::STRWpre:
652 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
653 case AArch64::STRXpre:
654 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
655 case AArch64::LDRSpre:
656 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
657 case AArch64::LDRDpre:
658 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
659 case AArch64::LDRQpre:
660 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
661 case AArch64::LDRWpre:
662 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
663 case AArch64::LDRXpre:
664 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
665 case AArch64::LDRSWpre:
666 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
667 }
668}
669
670// Returns the scale and offset range of pre/post indexed variants of MI.
671static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
672 int &MinOffset, int &MaxOffset) {
673 bool IsPaired = AArch64InstrInfo::isPairedLdSt(MI);
674 bool IsTagStore = isTagStore(MI);
675 // ST*G and all paired ldst have the same scale in pre/post-indexed variants
676 // as in the "unsigned offset" variant.
677 // All other pre/post indexed ldst instructions are unscaled.
678 Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
679
680 if (IsPaired) {
681 MinOffset = -64;
682 MaxOffset = 63;
683 } else {
684 MinOffset = -256;
685 MaxOffset = 255;
686 }
687}
688
689static MachineOperand &getLdStRegOp(MachineInstr &MI,
690 unsigned PairedRegOp = 0) {
691 assert(PairedRegOp < 2 && "Unexpected register operand idx.");
692 bool IsPreLdSt = AArch64InstrInfo::isPreLdSt(MI);
693 if (IsPreLdSt)
694 PairedRegOp += 1;
695 unsigned Idx =
696 AArch64InstrInfo::isPairedLdSt(MI) || IsPreLdSt ? PairedRegOp : 0;
697 return MI.getOperand(i: Idx);
698}
699
700static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst,
701 MachineInstr &StoreInst,
702 const AArch64InstrInfo *TII) {
703 assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
704 int LoadSize = TII->getMemScale(MI: LoadInst);
705 int StoreSize = TII->getMemScale(MI: StoreInst);
706 int UnscaledStOffset =
707 TII->hasUnscaledLdStOffset(MI&: StoreInst)
708 ? AArch64InstrInfo::getLdStOffsetOp(MI: StoreInst).getImm()
709 : AArch64InstrInfo::getLdStOffsetOp(MI: StoreInst).getImm() * StoreSize;
710 int UnscaledLdOffset =
711 TII->hasUnscaledLdStOffset(MI&: LoadInst)
712 ? AArch64InstrInfo::getLdStOffsetOp(MI: LoadInst).getImm()
713 : AArch64InstrInfo::getLdStOffsetOp(MI: LoadInst).getImm() * LoadSize;
714 return (UnscaledStOffset <= UnscaledLdOffset) &&
715 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
716}
717
718static bool isPromotableZeroStoreInst(MachineInstr &MI) {
719 unsigned Opc = MI.getOpcode();
720 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
721 isNarrowStore(Opc)) &&
722 getLdStRegOp(MI).getReg() == AArch64::WZR;
723}
724
725static bool isPromotableLoadFromStore(MachineInstr &MI) {
726 switch (MI.getOpcode()) {
727 default:
728 return false;
729 // Scaled instructions.
730 case AArch64::LDRBBui:
731 case AArch64::LDRHHui:
732 case AArch64::LDRWui:
733 case AArch64::LDRXui:
734 // Unscaled instructions.
735 case AArch64::LDURBBi:
736 case AArch64::LDURHHi:
737 case AArch64::LDURWi:
738 case AArch64::LDURXi:
739 return true;
740 }
741}
742
743static bool isMergeableLdStUpdate(MachineInstr &MI, AArch64FunctionInfo &AFI) {
744 unsigned Opc = MI.getOpcode();
745 switch (Opc) {
746 default:
747 return false;
748 // Scaled instructions.
749 case AArch64::STRBui:
750 case AArch64::STRHui:
751 case AArch64::STRSui:
752 case AArch64::STRDui:
753 case AArch64::STRQui:
754 case AArch64::STRXui:
755 case AArch64::STRWui:
756 case AArch64::STRHHui:
757 case AArch64::STRBBui:
758 case AArch64::LDRBui:
759 case AArch64::LDRHui:
760 case AArch64::LDRSui:
761 case AArch64::LDRDui:
762 case AArch64::LDRQui:
763 case AArch64::LDRXui:
764 case AArch64::LDRWui:
765 case AArch64::LDRHHui:
766 case AArch64::LDRBBui:
767 case AArch64::STGi:
768 case AArch64::STZGi:
769 case AArch64::ST2Gi:
770 case AArch64::STZ2Gi:
771 case AArch64::STGPi:
772 // Unscaled instructions.
773 case AArch64::STURSi:
774 case AArch64::STURDi:
775 case AArch64::STURQi:
776 case AArch64::STURWi:
777 case AArch64::STURXi:
778 case AArch64::LDURSi:
779 case AArch64::LDURDi:
780 case AArch64::LDURQi:
781 case AArch64::LDURWi:
782 case AArch64::LDURXi:
783 // Paired instructions.
784 case AArch64::LDPSi:
785 case AArch64::LDPSWi:
786 case AArch64::LDPDi:
787 case AArch64::LDPQi:
788 case AArch64::LDPWi:
789 case AArch64::LDPXi:
790 case AArch64::STPSi:
791 case AArch64::STPDi:
792 case AArch64::STPQi:
793 case AArch64::STPWi:
794 case AArch64::STPXi:
795 // Make sure this is a reg+imm (as opposed to an address reloc).
796 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
797 return false;
798
799 // When using stack tagging, simple sp+imm loads and stores are not
800 // tag-checked, but pre- and post-indexed versions of them are, so we can't
801 // replace the former with the latter. This transformation would be valid
802 // if the load/store accesses an untagged stack slot, but we don't have
803 // that information available after frame indices have been eliminated.
804 if (AFI.isMTETagged() &&
805 AArch64InstrInfo::getLdStBaseOp(MI).getReg() == AArch64::SP)
806 return false;
807
808 return true;
809 }
810}
811
812// Make sure this is a reg+reg Ld/St
813static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
814 unsigned Opc = MI.getOpcode();
815 switch (Opc) {
816 default:
817 return false;
818 // Scaled instructions.
819 // TODO: Add more index address stores.
820 case AArch64::LDRBroX:
821 case AArch64::LDRBBroX:
822 case AArch64::LDRSBXroX:
823 case AArch64::LDRSBWroX:
824 Scale = 1;
825 return true;
826 case AArch64::LDRHroX:
827 case AArch64::LDRHHroX:
828 case AArch64::LDRSHXroX:
829 case AArch64::LDRSHWroX:
830 Scale = 2;
831 return true;
832 case AArch64::LDRWroX:
833 case AArch64::LDRSroX:
834 case AArch64::LDRSWroX:
835 Scale = 4;
836 return true;
837 case AArch64::LDRDroX:
838 case AArch64::LDRXroX:
839 Scale = 8;
840 return true;
841 case AArch64::LDRQroX:
842 Scale = 16;
843 return true;
844 }
845}
846
847static bool isRewritableImplicitDef(const MachineInstr &MI,
848 const MachineOperand &MO) {
849 switch (MI.getOpcode()) {
850 default:
851 return MO.isRenamable();
852 case AArch64::ORRWrs:
853 case AArch64::ADDWri:
854 return true;
855 }
856}
857
858MachineBasicBlock::iterator
859AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
860 MachineBasicBlock::iterator MergeMI,
861 const LdStPairFlags &Flags) {
862 assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
863 "Expected promotable zero stores.");
864
865 MachineBasicBlock::iterator E = I->getParent()->end();
866 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
867 // If NextI is the second of the two instructions to be merged, we need
868 // to skip one further. Either way we merge will invalidate the iterator,
869 // and we don't need to scan the new instruction, as it's a pairwise
870 // instruction, which we're not considering for further action anyway.
871 if (NextI == MergeMI)
872 NextI = next_nodbg(It: NextI, End: E);
873
874 unsigned Opc = I->getOpcode();
875 unsigned MergeMIOpc = MergeMI->getOpcode();
876 bool IsScaled = !TII->hasUnscaledLdStOffset(Opc);
877 bool IsMergedMIScaled = !TII->hasUnscaledLdStOffset(Opc: MergeMIOpc);
878 int OffsetStride = IsScaled ? TII->getMemScale(MI: *I) : 1;
879 int MergeMIOffsetStride = IsMergedMIScaled ? TII->getMemScale(MI: *MergeMI) : 1;
880
881 bool MergeForward = Flags.getMergeForward();
882 // Insert our new paired instruction after whichever of the paired
883 // instructions MergeForward indicates.
884 MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
885 // Also based on MergeForward is from where we copy the base register operand
886 // so we get the flags compatible with the input code.
887 const MachineOperand &BaseRegOp =
888 MergeForward ? AArch64InstrInfo::getLdStBaseOp(MI: *MergeMI)
889 : AArch64InstrInfo::getLdStBaseOp(MI: *I);
890
891 // Which register is Rt and which is Rt2 depends on the offset order.
892 int64_t IOffsetInBytes =
893 AArch64InstrInfo::getLdStOffsetOp(MI: *I).getImm() * OffsetStride;
894 int64_t MIOffsetInBytes =
895 AArch64InstrInfo::getLdStOffsetOp(MI: *MergeMI).getImm() *
896 MergeMIOffsetStride;
897 // Select final offset based on the offset order.
898 int64_t OffsetImm;
899 if (IOffsetInBytes > MIOffsetInBytes)
900 OffsetImm = MIOffsetInBytes;
901 else
902 OffsetImm = IOffsetInBytes;
903
904 int NewOpcode = getMatchingWideOpcode(Opc);
905 // Adjust final offset on scaled stores because the new instruction
906 // has a different scale.
907 if (!TII->hasUnscaledLdStOffset(Opc: NewOpcode)) {
908 int NewOffsetStride = TII->getMemScale(Opc: NewOpcode);
909 assert(((OffsetImm % NewOffsetStride) == 0) &&
910 "Offset should be a multiple of the store memory scale");
911 OffsetImm = OffsetImm / NewOffsetStride;
912 }
913
914 // Construct the new instruction.
915 DebugLoc DL = I->getDebugLoc();
916 MachineBasicBlock *MBB = I->getParent();
917 MachineInstrBuilder MIB;
918 MIB = BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: NewOpcode))
919 .addReg(RegNo: isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
920 .add(MO: BaseRegOp)
921 .addImm(Val: OffsetImm)
922 .cloneMergedMemRefs(OtherMIs: {&*I, &*MergeMI})
923 .setMIFlags(I->mergeFlagsWith(Other: *MergeMI));
924 (void)MIB;
925
926 LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
927 LLVM_DEBUG(I->print(dbgs()));
928 LLVM_DEBUG(dbgs() << " ");
929 LLVM_DEBUG(MergeMI->print(dbgs()));
930 LLVM_DEBUG(dbgs() << " with instruction:\n ");
931 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
932 LLVM_DEBUG(dbgs() << "\n");
933
934 // Erase the old instructions.
935 I->eraseFromParent();
936 MergeMI->eraseFromParent();
937 return NextI;
938}
939
940// Apply Fn to all instructions between MI and the beginning of the block, until
941// a def for DefReg is reached. Returns true, iff Fn returns true for all
942// visited instructions. Stop after visiting Limit iterations.
943static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
944 const TargetRegisterInfo *TRI, unsigned Limit,
945 std::function<bool(MachineInstr &, bool)> &Fn) {
946 auto MBB = MI.getParent();
947 for (MachineInstr &I :
948 instructionsWithoutDebug(It: MI.getReverseIterator(), End: MBB->instr_rend())) {
949 if (!Limit)
950 return false;
951 --Limit;
952
953 bool isDef = any_of(Range: I.operands(), P: [DefReg, TRI](MachineOperand &MOP) {
954 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
955 TRI->regsOverlap(RegA: MOP.getReg(), RegB: DefReg);
956 });
957 if (!Fn(I, isDef))
958 return false;
959 if (isDef)
960 break;
961 }
962 return true;
963}
964
965static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units,
966 const TargetRegisterInfo *TRI) {
967
968 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
969 if (MOP.isReg() && MOP.isKill())
970 Units.removeReg(Reg: MOP.getReg());
971
972 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
973 if (MOP.isReg() && !MOP.isKill())
974 Units.addReg(Reg: MOP.getReg());
975}
976
977/// This function will add a new entry into the debugValueSubstitutions table
978/// when two instruction have been merged into a new one represented by \p
979/// MergedInstr.
980static void addDebugSubstitutionsToTable(MachineFunction *MF,
981 unsigned InstrNumToSet,
982 MachineInstr &OriginalInstr,
983 MachineInstr &MergedInstr) {
984
985 // Figure out the Operand Index of the destination register of the
986 // OriginalInstr in the new MergedInstr.
987 auto Reg = OriginalInstr.getOperand(i: 0).getReg();
988 unsigned OperandNo = 0;
989 bool RegFound = false;
990 for (const auto Op : MergedInstr.operands()) {
991 if (Op.getReg() == Reg) {
992 RegFound = true;
993 break;
994 }
995 OperandNo++;
996 }
997
998 if (RegFound)
999 MF->makeDebugValueSubstitution({OriginalInstr.peekDebugInstrNum(), 0},
1000 {InstrNumToSet, OperandNo});
1001}
1002
1003MachineBasicBlock::iterator
1004AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
1005 MachineBasicBlock::iterator Paired,
1006 const LdStPairFlags &Flags) {
1007 MachineBasicBlock::iterator E = I->getParent()->end();
1008 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
1009 // If NextI is the second of the two instructions to be merged, we need
1010 // to skip one further. Either way we merge will invalidate the iterator,
1011 // and we don't need to scan the new instruction, as it's a pairwise
1012 // instruction, which we're not considering for further action anyway.
1013 if (NextI == Paired)
1014 NextI = next_nodbg(It: NextI, End: E);
1015
1016 int SExtIdx = Flags.getSExtIdx();
1017 unsigned Opc =
1018 SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(Opc: I->getOpcode());
1019 bool IsUnscaled = TII->hasUnscaledLdStOffset(Opc);
1020 int OffsetStride = IsUnscaled ? TII->getMemScale(MI: *I) : 1;
1021
1022 bool MergeForward = Flags.getMergeForward();
1023
1024 std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
1025 if (RenameReg) {
1026 MCRegister RegToRename = getLdStRegOp(MI&: *I).getReg();
1027 DefinedInBB.addReg(Reg: *RenameReg);
1028
1029 // Return the sub/super register for RenameReg, matching the size of
1030 // OriginalReg.
1031 auto GetMatchingSubReg =
1032 [this, RenameReg](const TargetRegisterClass *C) -> MCPhysReg {
1033 for (MCPhysReg SubOrSuper :
1034 TRI->sub_and_superregs_inclusive(Reg: *RenameReg)) {
1035 if (C->contains(Reg: SubOrSuper))
1036 return SubOrSuper;
1037 }
1038 llvm_unreachable("Should have found matching sub or super register!");
1039 };
1040
1041 std::function<bool(MachineInstr &, bool)> UpdateMIs =
1042 [this, RegToRename, GetMatchingSubReg, MergeForward](MachineInstr &MI,
1043 bool IsDef) {
1044 if (IsDef) {
1045 bool SeenDef = false;
1046 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1047 MachineOperand &MOP = MI.getOperand(i: OpIdx);
1048 // Rename the first explicit definition and all implicit
1049 // definitions matching RegToRename.
1050 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1051 (!MergeForward || !SeenDef ||
1052 (MOP.isDef() && MOP.isImplicit())) &&
1053 TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename)) {
1054 assert((MOP.isImplicit() ||
1055 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1056 "Need renamable operands");
1057 Register MatchingReg;
1058 if (const TargetRegisterClass *RC =
1059 MI.getRegClassConstraint(OpIdx, TII, TRI))
1060 MatchingReg = GetMatchingSubReg(RC);
1061 else {
1062 if (!isRewritableImplicitDef(MI, MO: MOP))
1063 continue;
1064 MatchingReg = GetMatchingSubReg(
1065 TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1066 }
1067 MOP.setReg(MatchingReg);
1068 SeenDef = true;
1069 }
1070 }
1071 } else {
1072 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1073 MachineOperand &MOP = MI.getOperand(i: OpIdx);
1074 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1075 TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename)) {
1076 assert((MOP.isImplicit() ||
1077 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1078 "Need renamable operands");
1079 Register MatchingReg;
1080 if (const TargetRegisterClass *RC =
1081 MI.getRegClassConstraint(OpIdx, TII, TRI))
1082 MatchingReg = GetMatchingSubReg(RC);
1083 else
1084 MatchingReg = GetMatchingSubReg(
1085 TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1086 assert(MatchingReg.isValid() &&
1087 "Cannot find matching regs for renaming");
1088 MOP.setReg(MatchingReg);
1089 }
1090 }
1091 }
1092 LLVM_DEBUG(dbgs() << "Renamed " << MI);
1093 return true;
1094 };
1095 forAllMIsUntilDef(MI&: MergeForward ? *I : *Paired->getPrevNode(), DefReg: RegToRename,
1096 TRI, UINT32_MAX, Fn&: UpdateMIs);
1097
1098#if !defined(NDEBUG)
1099 // For forward merging store:
1100 // Make sure the register used for renaming is not used between the
1101 // paired instructions. That would trash the content before the new
1102 // paired instruction.
1103 MCPhysReg RegToCheck = *RenameReg;
1104 // For backward merging load:
1105 // Make sure the register being renamed is not used between the
1106 // paired instructions. That would trash the content after the new
1107 // paired instruction.
1108 if (!MergeForward)
1109 RegToCheck = RegToRename;
1110 for (auto &MI :
1111 iterator_range<MachineInstrBundleIterator<llvm::MachineInstr>>(
1112 MergeForward ? std::next(I) : I,
1113 MergeForward ? std::next(Paired) : Paired))
1114 assert(all_of(MI.operands(),
1115 [this, RegToCheck](const MachineOperand &MOP) {
1116 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1117 MOP.isUndef() ||
1118 !TRI->regsOverlap(MOP.getReg(), RegToCheck);
1119 }) &&
1120 "Rename register used between paired instruction, trashing the "
1121 "content");
1122#endif
1123 }
1124
1125 // Insert our new paired instruction after whichever of the paired
1126 // instructions MergeForward indicates.
1127 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
1128 // Also based on MergeForward is from where we copy the base register operand
1129 // so we get the flags compatible with the input code.
1130 const MachineOperand &BaseRegOp =
1131 MergeForward ? AArch64InstrInfo::getLdStBaseOp(MI: *Paired)
1132 : AArch64InstrInfo::getLdStBaseOp(MI: *I);
1133
1134 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: *I).getImm();
1135 int PairedOffset = AArch64InstrInfo::getLdStOffsetOp(MI: *Paired).getImm();
1136 bool PairedIsUnscaled = TII->hasUnscaledLdStOffset(Opc: Paired->getOpcode());
1137 if (IsUnscaled != PairedIsUnscaled) {
1138 // We're trying to pair instructions that differ in how they are scaled. If
1139 // I is scaled then scale the offset of Paired accordingly. Otherwise, do
1140 // the opposite (i.e., make Paired's offset unscaled).
1141 int MemSize = TII->getMemScale(MI: *Paired);
1142 if (PairedIsUnscaled) {
1143 // If the unscaled offset isn't a multiple of the MemSize, we can't
1144 // pair the operations together.
1145 assert(!(PairedOffset % TII->getMemScale(*Paired)) &&
1146 "Offset should be a multiple of the stride!");
1147 PairedOffset /= MemSize;
1148 } else {
1149 PairedOffset *= MemSize;
1150 }
1151 }
1152
1153 // Which register is Rt and which is Rt2 depends on the offset order.
1154 // However, for pre load/stores the Rt should be the one of the pre
1155 // load/store.
1156 MachineInstr *RtMI, *Rt2MI;
1157 if (Offset == PairedOffset + OffsetStride &&
1158 !AArch64InstrInfo::isPreLdSt(MI: *I)) {
1159 RtMI = &*Paired;
1160 Rt2MI = &*I;
1161 // Here we swapped the assumption made for SExtIdx.
1162 // I.e., we turn ldp I, Paired into ldp Paired, I.
1163 // Update the index accordingly.
1164 if (SExtIdx != -1)
1165 SExtIdx = (SExtIdx + 1) % 2;
1166 } else {
1167 RtMI = &*I;
1168 Rt2MI = &*Paired;
1169 }
1170 int OffsetImm = AArch64InstrInfo::getLdStOffsetOp(MI: *RtMI).getImm();
1171 // Scale the immediate offset, if necessary.
1172 if (TII->hasUnscaledLdStOffset(Opc: RtMI->getOpcode())) {
1173 assert(!(OffsetImm % TII->getMemScale(*RtMI)) &&
1174 "Unscaled offset cannot be scaled.");
1175 OffsetImm /= TII->getMemScale(MI: *RtMI);
1176 }
1177
1178 // Construct the new instruction.
1179 MachineInstrBuilder MIB;
1180 DebugLoc DL = I->getDebugLoc();
1181 MachineBasicBlock *MBB = I->getParent();
1182 MachineOperand RegOp0 = getLdStRegOp(MI&: *RtMI);
1183 MachineOperand RegOp1 = getLdStRegOp(MI&: *Rt2MI);
1184 MachineOperand &PairedRegOp = RtMI == &*Paired ? RegOp0 : RegOp1;
1185 // Kill flags may become invalid when moving stores for pairing.
1186 if (RegOp0.isUse()) {
1187 if (!MergeForward) {
1188 // Clear kill flags on store if moving upwards. Example:
1189 // STRWui kill %w0, ...
1190 // USE %w1
1191 // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards
1192 // We are about to move the store of w1, so its kill flag may become
1193 // invalid; not the case for w0.
1194 // Since w1 is used between the stores, the kill flag on w1 is cleared
1195 // after merging.
1196 // STPWi kill %w0, %w1, ...
1197 // USE %w1
1198 for (auto It = std::next(x: I); It != Paired && PairedRegOp.isKill(); ++It)
1199 if (It->readsRegister(Reg: PairedRegOp.getReg(), TRI))
1200 PairedRegOp.setIsKill(false);
1201 } else {
1202 // Clear kill flags of the first stores register. Example:
1203 // STRWui %w1, ...
1204 // USE kill %w1 ; need to clear kill flag when moving STRWui downwards
1205 // STRW %w0
1206 Register Reg = getLdStRegOp(MI&: *I).getReg();
1207 for (MachineInstr &MI :
1208 make_range(x: std::next(x: I->getIterator()), y: Paired->getIterator()))
1209 MI.clearRegisterKills(Reg, RegInfo: TRI);
1210 }
1211 }
1212
1213 unsigned int MatchPairOpcode = getMatchingPairOpcode(Opc);
1214 MIB = BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: MatchPairOpcode));
1215
1216 // Adds the pre-index operand for pre-indexed ld/st pairs.
1217 if (AArch64InstrInfo::isPreLdSt(MI: *RtMI))
1218 MIB.addReg(RegNo: BaseRegOp.getReg(), Flags: RegState::Define);
1219
1220 MIB.add(MO: RegOp0)
1221 .add(MO: RegOp1)
1222 .add(MO: BaseRegOp)
1223 .addImm(Val: OffsetImm)
1224 .cloneMergedMemRefs(OtherMIs: {&*I, &*Paired})
1225 .setMIFlags(I->mergeFlagsWith(Other: *Paired));
1226
1227 (void)MIB;
1228
1229 LLVM_DEBUG(
1230 dbgs() << "Creating pair load/store. Replacing instructions:\n ");
1231 LLVM_DEBUG(I->print(dbgs()));
1232 LLVM_DEBUG(dbgs() << " ");
1233 LLVM_DEBUG(Paired->print(dbgs()));
1234 LLVM_DEBUG(dbgs() << " with instruction:\n ");
1235 if (SExtIdx != -1) {
1236 // Generate the sign extension for the proper result of the ldp.
1237 // I.e., with X1, that would be:
1238 // %w1 = KILL %w1, implicit-def %x1
1239 // %x1 = SBFMXri killed %x1, 0, 31
1240 MachineOperand &DstMO = MIB->getOperand(i: SExtIdx);
1241 // Right now, DstMO has the extended register, since it comes from an
1242 // extended opcode.
1243 Register DstRegX = DstMO.getReg();
1244 // Get the W variant of that register.
1245 Register DstRegW = TRI->getSubReg(Reg: DstRegX, Idx: AArch64::sub_32);
1246 // Update the result of LDP to use the W instead of the X variant.
1247 DstMO.setReg(DstRegW);
1248 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1249 LLVM_DEBUG(dbgs() << "\n");
1250 // Make the machine verifier happy by providing a definition for
1251 // the X register.
1252 // Insert this definition right after the generated LDP, i.e., before
1253 // InsertionPoint.
1254 MachineInstrBuilder MIBKill =
1255 BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::KILL), DestReg: DstRegW)
1256 .addReg(RegNo: DstRegW)
1257 .addReg(RegNo: DstRegX, Flags: RegState::Define);
1258 MIBKill->getOperand(i: 2).setImplicit();
1259 // Create the sign extension.
1260 MachineInstrBuilder MIBSXTW =
1261 BuildMI(BB&: *MBB, I: InsertionPoint, MIMD: DL, MCID: TII->get(Opcode: AArch64::SBFMXri), DestReg: DstRegX)
1262 .addReg(RegNo: DstRegX)
1263 .addImm(Val: 0)
1264 .addImm(Val: 31);
1265 (void)MIBSXTW;
1266
1267 // In the case of a sign-extend, where we have something like:
1268 // debugValueSubstitutions:[]
1269 // $w1 = LDRWui $x0, 1, debug-instr-number 1
1270 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1271 // $x0 = LDRSWui $x0, 0, debug-instr-number 2
1272 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1273
1274 // It will be converted to:
1275 // debugValueSubstitutions:[]
1276 // $w0, $w1 = LDPWi $x0, 0
1277 // $w0 = KILL $w0, implicit-def $x0
1278 // $x0 = SBFMXri $x0, 0, 31
1279 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1280 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1281
1282 // We want the final result to look like:
1283 // debugValueSubstitutions:
1284 // - { srcinst: 1, srcop: 0, dstinst: 4, dstop: 1, subreg: 0 }
1285 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1286 // $w0, $w1 = LDPWi $x0, 0, debug-instr-number 4
1287 // $w0 = KILL $w0, implicit-def $x0
1288 // $x0 = SBFMXri $x0, 0, 31, debug-instr-number 3
1289 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1290 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1291
1292 // $x0 is where the final value is stored, so the sign extend (SBFMXri)
1293 // instruction contains the final value we care about we give it a new
1294 // debug-instr-number 3. Whereas, $w1 contains the final value that we care
1295 // about, therefore the LDP instruction is also given a new
1296 // debug-instr-number 4. We have to add these substitutions to the
1297 // debugValueSubstitutions table. However, we also have to ensure that the
1298 // OpIndex that pointed to debug-instr-number 1 gets updated to 1, because
1299 // $w1 is the second operand of the LDP instruction.
1300
1301 if (I->peekDebugInstrNum()) {
1302 // If I is the instruction which got sign extended and has a
1303 // debug-instr-number, give the SBFMXri instruction a new
1304 // debug-instr-number, and update the debugValueSubstitutions table with
1305 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1306 // instruction a new debug-instr-number, and update the
1307 // debugValueSubstitutions table with the new debug-instr-number and
1308 // OpIndex pair.
1309 unsigned NewInstrNum;
1310 if (DstRegX == I->getOperand(i: 0).getReg()) {
1311 NewInstrNum = MIBSXTW->getDebugInstrNum();
1312 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *I,
1313 MergedInstr&: *MIBSXTW);
1314 } else {
1315 NewInstrNum = MIB->getDebugInstrNum();
1316 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *I, MergedInstr&: *MIB);
1317 }
1318 }
1319 if (Paired->peekDebugInstrNum()) {
1320 // If Paired is the instruction which got sign extended and has a
1321 // debug-instr-number, give the SBFMXri instruction a new
1322 // debug-instr-number, and update the debugValueSubstitutions table with
1323 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1324 // instruction a new debug-instr-number, and update the
1325 // debugValueSubstitutions table with the new debug-instr-number and
1326 // OpIndex pair.
1327 unsigned NewInstrNum;
1328 if (DstRegX == Paired->getOperand(i: 0).getReg()) {
1329 NewInstrNum = MIBSXTW->getDebugInstrNum();
1330 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *Paired,
1331 MergedInstr&: *MIBSXTW);
1332 } else {
1333 NewInstrNum = MIB->getDebugInstrNum();
1334 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewInstrNum, OriginalInstr&: *Paired,
1335 MergedInstr&: *MIB);
1336 }
1337 }
1338
1339 LLVM_DEBUG(dbgs() << " Extend operand:\n ");
1340 LLVM_DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs()));
1341 } else if (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI) {
1342 // We are combining SVE fill/spill to LDP/STP, so we need to use the Q
1343 // variant of the registers.
1344 MachineOperand &MOp0 = MIB->getOperand(i: 0);
1345 MachineOperand &MOp1 = MIB->getOperand(i: 1);
1346 assert(AArch64::ZPRRegClass.contains(MOp0.getReg()) &&
1347 AArch64::ZPRRegClass.contains(MOp1.getReg()) && "Invalid register.");
1348 MOp0.setReg(AArch64::Q0 + (MOp0.getReg() - AArch64::Z0));
1349 MOp1.setReg(AArch64::Q0 + (MOp1.getReg() - AArch64::Z0));
1350 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1351 } else {
1352
1353 // In the case that the merge doesn't result in a sign-extend, if we have
1354 // something like:
1355 // debugValueSubstitutions:[]
1356 // $x1 = LDRXui $x0, 1, debug-instr-number 1
1357 // DBG_INSTR_REF !13, dbg-instr-ref(1, 0), debug-location !11
1358 // $x0 = LDRXui killed $x0, 0, debug-instr-number 2
1359 // DBG_INSTR_REF !14, dbg-instr-ref(2, 0), debug-location !11
1360
1361 // It will be converted to:
1362 // debugValueSubstitutions: []
1363 // $x0, $x1 = LDPXi $x0, 0
1364 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1365 // DBG_INSTR_REF !13, dbg-instr-ref(2, 0), debug-location !14
1366
1367 // We want the final result to look like:
1368 // debugValueSubstitutions:
1369 // - { srcinst: 1, srcop: 0, dstinst: 3, dstop: 1, subreg: 0 }
1370 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1371 // $x0, $x1 = LDPXi $x0, 0, debug-instr-number 3
1372 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1373 // DBG_INSTR_REF !12, dbg-instr-ref(2, 0), debug-location !14
1374
1375 // Here all that needs to be done is, that the LDP instruction needs to be
1376 // updated with a new debug-instr-number, we then need to add entries into
1377 // the debugSubstitutions table to map the old instr-refs to the new ones.
1378
1379 // Assign new DebugInstrNum to the Paired instruction.
1380 if (I->peekDebugInstrNum()) {
1381 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1382 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewDebugInstrNum, OriginalInstr&: *I,
1383 MergedInstr&: *MIB);
1384 }
1385 if (Paired->peekDebugInstrNum()) {
1386 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1387 addDebugSubstitutionsToTable(MF: MBB->getParent(), InstrNumToSet: NewDebugInstrNum, OriginalInstr&: *Paired,
1388 MergedInstr&: *MIB);
1389 }
1390
1391 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1392 }
1393 LLVM_DEBUG(dbgs() << "\n");
1394
1395 if (MergeForward)
1396 for (const MachineOperand &MOP : phys_regs_and_masks(MI: *I))
1397 if (MOP.isReg() && MOP.isKill())
1398 DefinedInBB.addReg(Reg: MOP.getReg());
1399
1400 // Copy over any implicit-def operands. This is like MI.copyImplicitOps, but
1401 // only copies implicit defs and makes sure that each operand is only added
1402 // once in case of duplicates.
1403 auto CopyImplicitOps = [&](MachineBasicBlock::iterator MI1,
1404 MachineBasicBlock::iterator MI2) {
1405 SmallSetVector<Register, 4> Ops;
1406 for (const MachineOperand &MO :
1407 llvm::drop_begin(RangeOrContainer: MI1->operands(), N: MI1->getDesc().getNumOperands()))
1408 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1409 Ops.insert(X: MO.getReg());
1410 for (const MachineOperand &MO :
1411 llvm::drop_begin(RangeOrContainer: MI2->operands(), N: MI2->getDesc().getNumOperands()))
1412 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1413 Ops.insert(X: MO.getReg());
1414 for (auto Op : Ops)
1415 MIB.addDef(RegNo: Op, Flags: RegState::Implicit);
1416 };
1417 CopyImplicitOps(I, Paired);
1418
1419 // Erase the old instructions.
1420 I->eraseFromParent();
1421 Paired->eraseFromParent();
1422
1423 return NextI;
1424}
1425
1426MachineBasicBlock::iterator
1427AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
1428 MachineBasicBlock::iterator StoreI) {
1429 MachineBasicBlock::iterator NextI =
1430 next_nodbg(It: LoadI, End: LoadI->getParent()->end());
1431
1432 int LoadSize = TII->getMemScale(MI: *LoadI);
1433 int StoreSize = TII->getMemScale(MI: *StoreI);
1434 Register LdRt = getLdStRegOp(MI&: *LoadI).getReg();
1435 const MachineOperand &StMO = getLdStRegOp(MI&: *StoreI);
1436 Register StRt = getLdStRegOp(MI&: *StoreI).getReg();
1437 bool IsStoreXReg = TRI->getRegClass(i: AArch64::GPR64RegClassID)->contains(Reg: StRt);
1438
1439 assert((IsStoreXReg ||
1440 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1441 "Unexpected RegClass");
1442
1443 MachineInstr *BitExtMI;
1444 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1445 // Remove the load, if the destination register of the loads is the same
1446 // register for stored value.
1447 if (StRt == LdRt && LoadSize == 8) {
1448 for (MachineInstr &MI : make_range(x: StoreI->getIterator(),
1449 y: LoadI->getIterator())) {
1450 if (MI.killsRegister(Reg: StRt, TRI)) {
1451 MI.clearRegisterKills(Reg: StRt, RegInfo: TRI);
1452 break;
1453 }
1454 }
1455 LLVM_DEBUG(dbgs() << "Remove load instruction:\n ");
1456 LLVM_DEBUG(LoadI->print(dbgs()));
1457 LLVM_DEBUG(dbgs() << "\n");
1458 LoadI->eraseFromParent();
1459 return NextI;
1460 }
1461 // Replace the load with a mov if the load and store are in the same size.
1462 BitExtMI =
1463 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1464 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), DestReg: LdRt)
1465 .addReg(RegNo: IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1466 .add(MO: StMO)
1467 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
1468 .setMIFlags(LoadI->getFlags());
1469 } else {
1470 // FIXME: Currently we disable this transformation in big-endian targets as
1471 // performance and correctness are verified only in little-endian.
1472 if (!Subtarget->isLittleEndian())
1473 return NextI;
1474 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI&: *LoadI);
1475 assert(IsUnscaled == TII->hasUnscaledLdStOffset(*StoreI) &&
1476 "Unsupported ld/st match");
1477 assert(LoadSize <= StoreSize && "Invalid load size");
1478 int UnscaledLdOffset =
1479 IsUnscaled
1480 ? AArch64InstrInfo::getLdStOffsetOp(MI: *LoadI).getImm()
1481 : AArch64InstrInfo::getLdStOffsetOp(MI: *LoadI).getImm() * LoadSize;
1482 int UnscaledStOffset =
1483 IsUnscaled
1484 ? AArch64InstrInfo::getLdStOffsetOp(MI: *StoreI).getImm()
1485 : AArch64InstrInfo::getLdStOffsetOp(MI: *StoreI).getImm() * StoreSize;
1486 int Width = LoadSize * 8;
1487 Register DestReg =
1488 IsStoreXReg ? Register(TRI->getMatchingSuperReg(
1489 Reg: LdRt, SubIdx: AArch64::sub_32, RC: &AArch64::GPR64RegClass))
1490 : LdRt;
1491
1492 assert((UnscaledLdOffset >= UnscaledStOffset &&
1493 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1494 "Invalid offset");
1495
1496 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1497 int Imms = Immr + Width - 1;
1498 if (UnscaledLdOffset == UnscaledStOffset) {
1499 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N
1500 | ((Immr) << 6) // immr
1501 | ((Imms) << 0) // imms
1502 ;
1503
1504 BitExtMI =
1505 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1506 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1507 DestReg)
1508 .add(MO: StMO)
1509 .addImm(Val: AndMaskEncoded)
1510 .setMIFlags(LoadI->getFlags());
1511 } else if (IsStoreXReg && Imms == 31) {
1512 // Use the 32 bit variant of UBFM if it's the LSR alias of the
1513 // instruction.
1514 assert(Immr <= Imms && "Expected LSR alias of UBFM");
1515 BitExtMI = BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1516 MCID: TII->get(Opcode: AArch64::UBFMWri),
1517 DestReg: TRI->getSubReg(Reg: DestReg, Idx: AArch64::sub_32))
1518 .addReg(RegNo: TRI->getSubReg(Reg: StRt, Idx: AArch64::sub_32))
1519 .addImm(Val: Immr)
1520 .addImm(Val: Imms)
1521 .setMIFlags(LoadI->getFlags());
1522 } else {
1523 BitExtMI =
1524 BuildMI(BB&: *LoadI->getParent(), I: LoadI, MIMD: LoadI->getDebugLoc(),
1525 MCID: TII->get(Opcode: IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1526 DestReg)
1527 .add(MO: StMO)
1528 .addImm(Val: Immr)
1529 .addImm(Val: Imms)
1530 .setMIFlags(LoadI->getFlags());
1531 }
1532 }
1533
1534 // Clear kill flags between store and load.
1535 for (MachineInstr &MI : make_range(x: StoreI->getIterator(),
1536 y: BitExtMI->getIterator()))
1537 if (MI.killsRegister(Reg: StRt, TRI)) {
1538 MI.clearRegisterKills(Reg: StRt, RegInfo: TRI);
1539 break;
1540 }
1541
1542 LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
1543 LLVM_DEBUG(StoreI->print(dbgs()));
1544 LLVM_DEBUG(dbgs() << " ");
1545 LLVM_DEBUG(LoadI->print(dbgs()));
1546 LLVM_DEBUG(dbgs() << " with instructions:\n ");
1547 LLVM_DEBUG(StoreI->print(dbgs()));
1548 LLVM_DEBUG(dbgs() << " ");
1549 LLVM_DEBUG((BitExtMI)->print(dbgs()));
1550 LLVM_DEBUG(dbgs() << "\n");
1551
1552 // Erase the old instructions.
1553 LoadI->eraseFromParent();
1554 return NextI;
1555}
1556
1557static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
1558 // Convert the byte-offset used by unscaled into an "element" offset used
1559 // by the scaled pair load/store instructions.
1560 if (IsUnscaled) {
1561 // If the byte-offset isn't a multiple of the stride, there's no point
1562 // trying to match it.
1563 if (Offset % OffsetStride)
1564 return false;
1565 Offset /= OffsetStride;
1566 }
1567 return Offset <= 63 && Offset >= -64;
1568}
1569
1570// Do alignment, specialized to power of 2 and for signed ints,
1571// avoiding having to do a C-style cast from uint_64t to int when
1572// using alignTo from include/llvm/Support/MathExtras.h.
1573// FIXME: Move this function to include/MathExtras.h?
1574static int alignTo(int Num, int PowOf2) {
1575 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1576}
1577
1578static bool mayAlias(MachineInstr &MIa,
1579 SmallVectorImpl<MachineInstr *> &MemInsns,
1580 AliasAnalysis *AA) {
1581 for (MachineInstr *MIb : MemInsns) {
1582 if (MIa.mayAlias(AA, Other: *MIb, /*UseTBAA*/ false)) {
1583 LLVM_DEBUG(dbgs() << "Aliasing with: "; MIb->dump());
1584 return true;
1585 }
1586 }
1587
1588 LLVM_DEBUG(dbgs() << "No aliases found\n");
1589 return false;
1590}
1591
1592bool AArch64LoadStoreOpt::findMatchingStore(
1593 MachineBasicBlock::iterator I, unsigned Limit,
1594 MachineBasicBlock::iterator &StoreI) {
1595 MachineBasicBlock::iterator B = I->getParent()->begin();
1596 MachineBasicBlock::iterator MBBI = I;
1597 MachineInstr &LoadMI = *I;
1598 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: LoadMI).getReg();
1599
1600 // If the load is the first instruction in the block, there's obviously
1601 // not any matching store.
1602 if (MBBI == B)
1603 return false;
1604
1605 // Track which register units have been modified and used between the first
1606 // insn and the second insn.
1607 ModifiedRegUnits.clear();
1608 UsedRegUnits.clear();
1609
1610 unsigned Count = 0;
1611 do {
1612 MBBI = prev_nodbg(It: MBBI, Begin: B);
1613 MachineInstr &MI = *MBBI;
1614
1615 // Don't count transient instructions towards the search limit since there
1616 // may be different numbers of them if e.g. debug information is present.
1617 if (!MI.isTransient())
1618 ++Count;
1619
1620 // If the load instruction reads directly from the address to which the
1621 // store instruction writes and the stored value is not modified, we can
1622 // promote the load. Since we do not handle stores with pre-/post-index,
1623 // it's unnecessary to check if BaseReg is modified by the store itself.
1624 // Also we can't handle stores without an immediate offset operand,
1625 // while the operand might be the address for a global variable.
1626 if (MI.mayStore() && isMatchingStore(LoadInst&: LoadMI, StoreInst&: MI) &&
1627 BaseReg == AArch64InstrInfo::getLdStBaseOp(MI).getReg() &&
1628 AArch64InstrInfo::getLdStOffsetOp(MI).isImm() &&
1629 isLdOffsetInRangeOfSt(LoadInst&: LoadMI, StoreInst&: MI, TII) &&
1630 ModifiedRegUnits.available(Reg: getLdStRegOp(MI).getReg())) {
1631 StoreI = MBBI;
1632 return true;
1633 }
1634
1635 if (MI.isCall())
1636 return false;
1637
1638 // Update modified / uses register units.
1639 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1640
1641 // Otherwise, if the base register is modified, we have no match, so
1642 // return early.
1643 if (!ModifiedRegUnits.available(Reg: BaseReg))
1644 return false;
1645
1646 // If we encounter a store aliased with the load, return early.
1647 if (MI.mayStore() && LoadMI.mayAlias(AA, Other: MI, /*UseTBAA*/ false))
1648 return false;
1649 } while (MBBI != B && Count < Limit);
1650 return false;
1651}
1652
1653static bool needsWinCFI(const MachineFunction *MF) {
1654 return MF->getTarget().getMCAsmInfo().usesWindowsCFI() &&
1655 MF->getFunction().needsUnwindTableEntry();
1656}
1657
1658// Returns true if FirstMI and MI are candidates for merging or pairing.
1659// Otherwise, returns false.
1660static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
1661 LdStPairFlags &Flags,
1662 const AArch64InstrInfo *TII) {
1663 // If this is volatile or if pairing is suppressed, not a candidate.
1664 if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
1665 return false;
1666
1667 // We should have already checked FirstMI for pair suppression and volatility.
1668 assert(!FirstMI.hasOrderedMemoryRef() &&
1669 !TII->isLdStPairSuppressed(FirstMI) &&
1670 "FirstMI shouldn't get here if either of these checks are true.");
1671
1672 if (needsWinCFI(MF: MI.getMF()) && (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
1673 MI.getFlag(Flag: MachineInstr::FrameDestroy)))
1674 return false;
1675
1676 unsigned OpcA = FirstMI.getOpcode();
1677 unsigned OpcB = MI.getOpcode();
1678
1679 // Opcodes match: If the opcodes are pre ld/st there is nothing more to check.
1680 if (OpcA == OpcB)
1681 return !AArch64InstrInfo::isPreLdSt(MI: FirstMI);
1682
1683 // Bail out if one of the opcodes is SVE fill/spill, as we currently don't
1684 // allow pairing them with other instructions.
1685 if (OpcA == AArch64::LDR_ZXI || OpcA == AArch64::STR_ZXI ||
1686 OpcB == AArch64::LDR_ZXI || OpcB == AArch64::STR_ZXI)
1687 return false;
1688
1689 // Two pre ld/st of different opcodes cannot be merged either
1690 if (AArch64InstrInfo::isPreLdSt(MI: FirstMI) && AArch64InstrInfo::isPreLdSt(MI))
1691 return false;
1692
1693 // Try to match a sign-extended load/store with a zero-extended load/store.
1694 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1695 unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc: OpcA, IsValidLdStrOpc: &IsValidLdStrOpc);
1696 assert(IsValidLdStrOpc &&
1697 "Given Opc should be a Load or Store with an immediate");
1698 // OpcA will be the first instruction in the pair.
1699 if (NonSExtOpc == getMatchingNonSExtOpcode(Opc: OpcB, IsValidLdStrOpc: &PairIsValidLdStrOpc)) {
1700 Flags.setSExtIdx(NonSExtOpc == OpcA ? 1 : 0);
1701 return true;
1702 }
1703
1704 // If the second instruction isn't even a mergable/pairable load/store, bail
1705 // out.
1706 if (!PairIsValidLdStrOpc)
1707 return false;
1708
1709 // Narrow stores do not have a matching pair opcodes, so constrain their
1710 // merging to zero stores.
1711 if (isNarrowStore(Opc: OpcA) || isNarrowStore(Opc: OpcB))
1712 return getLdStRegOp(MI&: FirstMI).getReg() == AArch64::WZR &&
1713 getLdStRegOp(MI).getReg() == AArch64::WZR &&
1714 TII->getMemScale(MI: FirstMI) == TII->getMemScale(MI);
1715
1716 // The STR<S,D,Q,W,X>pre - STR<S,D,Q,W,X>ui and
1717 // LDR<S,D,Q,W,X,SW>pre-LDR<S,D,Q,W,X,SW>ui
1718 // are candidate pairs that can be merged.
1719 if (isPreLdStPairCandidate(FirstMI, MI))
1720 return true;
1721
1722 // Try to match an unscaled load/store with a scaled load/store.
1723 return TII->hasUnscaledLdStOffset(Opc: OpcA) != TII->hasUnscaledLdStOffset(Opc: OpcB) &&
1724 getMatchingPairOpcode(Opc: OpcA) == getMatchingPairOpcode(Opc: OpcB);
1725
1726 // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
1727}
1728
1729static bool canRenameMOP(const MachineInstr &MI, const MachineOperand &MOP,
1730 const TargetRegisterInfo *TRI) {
1731 if (MOP.isReg()) {
1732 auto *RegClass = TRI->getMinimalPhysRegClass(Reg: MOP.getReg());
1733 // Renaming registers with multiple disjunct sub-registers (e.g. the
1734 // result of a LD3) means that all sub-registers are renamed, potentially
1735 // impacting other instructions we did not check. Bail out.
1736 // Note that this relies on the structure of the AArch64 register file. In
1737 // particular, a subregister cannot be written without overwriting the
1738 // whole register.
1739 if (RegClass->HasDisjunctSubRegs && RegClass->CoveredBySubRegs &&
1740 (TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::dsub0) ||
1741 TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::qsub0) ||
1742 TRI->getSubRegisterClass(SuperRC: RegClass, SubRegIdx: AArch64::zsub0))) {
1743 LLVM_DEBUG(
1744 dbgs()
1745 << " Cannot rename operands with multiple disjunct subregisters ("
1746 << MOP << ")\n");
1747 return false;
1748 }
1749
1750 // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
1751 // them must be known. For example, in ORRWrs the implicit-def
1752 // corresponds to the result register.
1753 if (MOP.isImplicit() && MOP.isDef()) {
1754 if (!isRewritableImplicitDef(MI, MO: MOP))
1755 return false;
1756 return TRI->isSuperOrSubRegisterEq(RegA: MI.getOperand(i: 0).getReg(),
1757 RegB: MOP.getReg());
1758 }
1759 }
1760 return MOP.isImplicit() ||
1761 (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
1762}
1763
1764static bool
1765canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
1766 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1767 const TargetRegisterInfo *TRI) {
1768 if (!FirstMI.mayStore())
1769 return false;
1770
1771 // Check if we can find an unused register which we can use to rename
1772 // the register used by the first load/store.
1773
1774 auto RegToRename = getLdStRegOp(MI&: FirstMI).getReg();
1775 // For now, we only rename if the store operand gets killed at the store.
1776 if (!getLdStRegOp(MI&: FirstMI).isKill() &&
1777 !any_of(Range: FirstMI.operands(),
1778 P: [TRI, RegToRename](const MachineOperand &MOP) {
1779 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1780 MOP.isImplicit() && MOP.isKill() &&
1781 TRI->regsOverlap(RegA: RegToRename, RegB: MOP.getReg());
1782 })) {
1783 LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI);
1784 return false;
1785 }
1786
1787 bool FoundDef = false;
1788
1789 // For each instruction between FirstMI and the previous def for RegToRename,
1790 // we
1791 // * check if we can rename RegToRename in this instruction
1792 // * collect the registers used and required register classes for RegToRename.
1793 std::function<bool(MachineInstr &, bool)> CheckMIs = [&](MachineInstr &MI,
1794 bool IsDef) {
1795 LLVM_DEBUG(dbgs() << "Checking " << MI);
1796 // Currently we do not try to rename across frame-setup instructions.
1797 if (MI.getFlag(Flag: MachineInstr::FrameSetup)) {
1798 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1799 << "currently\n");
1800 return false;
1801 }
1802
1803 UsedInBetween.accumulate(MI);
1804
1805 // For a definition, check that we can rename the definition and exit the
1806 // loop.
1807 FoundDef = IsDef;
1808
1809 // For defs, check if we can rename the first def of RegToRename.
1810 if (FoundDef) {
1811 // For some pseudo instructions, we might not generate code in the end
1812 // (e.g. KILL) and we would end up without a correct def for the rename
1813 // register.
1814 // TODO: This might be overly conservative and we could handle those cases
1815 // in multiple ways:
1816 // 1. Insert an extra copy, to materialize the def.
1817 // 2. Skip pseudo-defs until we find an non-pseudo def.
1818 if (MI.isPseudo()) {
1819 LLVM_DEBUG(dbgs() << " Cannot rename pseudo/bundle instruction\n");
1820 return false;
1821 }
1822
1823 for (auto &MOP : MI.operands()) {
1824 if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
1825 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1826 continue;
1827 if (!canRenameMOP(MI, MOP, TRI)) {
1828 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1829 return false;
1830 }
1831 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1832 }
1833 return true;
1834 } else {
1835 for (auto &MOP : MI.operands()) {
1836 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1837 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1838 continue;
1839
1840 if (!canRenameMOP(MI, MOP, TRI)) {
1841 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1842 return false;
1843 }
1844 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1845 }
1846 }
1847 return true;
1848 };
1849
1850 const AArch64Options &CLOpts =
1851 FirstMI.getMF()->getSubtarget<AArch64Subtarget>().getCLOpts();
1852 if (!forAllMIsUntilDef(MI&: FirstMI, DefReg: RegToRename, TRI,
1853 Limit: CLOpts.load_store_scan_limit, Fn&: CheckMIs))
1854 return false;
1855
1856 if (!FoundDef) {
1857 LLVM_DEBUG(dbgs() << " Did not find definition for register in BB\n");
1858 return false;
1859 }
1860 return true;
1861}
1862
1863// We want to merge the second load into the first by rewriting the usages of
1864// the same reg between first (incl.) and second (excl.). We don't need to care
1865// about any insns before FirstLoad or after SecondLoad.
1866// 1. The second load writes new value into the same reg.
1867// - The renaming is impossible to impact later use of the reg.
1868// - The second load always trash the value written by the first load which
1869// means the reg must be killed before the second load.
1870// 2. The first load must be a def for the same reg so we don't need to look
1871// into anything before it.
1872static bool canRenameUntilSecondLoad(
1873 MachineInstr &FirstLoad, MachineInstr &SecondLoad,
1874 LiveRegUnits &UsedInBetween,
1875 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1876 const TargetRegisterInfo *TRI) {
1877 if (FirstLoad.isPseudo())
1878 return false;
1879
1880 UsedInBetween.accumulate(MI: FirstLoad);
1881 auto RegToRename = getLdStRegOp(MI&: FirstLoad).getReg();
1882 bool Success = std::all_of(
1883 first: FirstLoad.getIterator(), last: SecondLoad.getIterator(),
1884 pred: [&](MachineInstr &MI) {
1885 LLVM_DEBUG(dbgs() << "Checking " << MI);
1886 // Currently we do not try to rename across frame-setup instructions.
1887 if (MI.getFlag(Flag: MachineInstr::FrameSetup)) {
1888 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1889 << "currently\n");
1890 return false;
1891 }
1892
1893 for (auto &MOP : MI.operands()) {
1894 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1895 !TRI->regsOverlap(RegA: MOP.getReg(), RegB: RegToRename))
1896 continue;
1897 if (!canRenameMOP(MI, MOP, TRI)) {
1898 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1899 return false;
1900 }
1901 RequiredClasses.insert(Ptr: TRI->getMinimalPhysRegClass(Reg: MOP.getReg()));
1902 }
1903
1904 return true;
1905 });
1906 return Success;
1907}
1908
1909// Check if we can find a physical register for renaming \p Reg. This register
1910// must:
1911// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
1912// defined registers up to the point where the renamed register will be used,
1913// * not used in \p UsedInBetween; UsedInBetween must contain all accessed
1914// registers in the range the rename register will be used,
1915// * is available in all used register classes (checked using RequiredClasses).
1916static std::optional<MCPhysReg> tryToFindRegisterToRename(
1917 const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB,
1918 LiveRegUnits &UsedInBetween,
1919 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1920 const TargetRegisterInfo *TRI) {
1921 const MachineRegisterInfo &RegInfo = MF.getRegInfo();
1922
1923 // Checks if any sub- or super-register of PR is callee saved.
1924 auto AnySubOrSuperRegCalleePreserved = [&MF, TRI](MCPhysReg PR) {
1925 return any_of(Range: TRI->sub_and_superregs_inclusive(Reg: PR),
1926 P: [&MF, TRI](MCPhysReg SubOrSuper) {
1927 return TRI->isCalleeSavedPhysReg(PhysReg: SubOrSuper, MF);
1928 });
1929 };
1930
1931 // Check if PR or one of its sub- or super-registers can be used for all
1932 // required register classes.
1933 auto CanBeUsedForAllClasses = [&RequiredClasses, TRI](MCPhysReg PR) {
1934 return all_of(Range&: RequiredClasses, P: [PR, TRI](const TargetRegisterClass *C) {
1935 return any_of(
1936 Range: TRI->sub_and_superregs_inclusive(Reg: PR),
1937 P: [C](MCPhysReg SubOrSuper) { return C->contains(Reg: SubOrSuper); });
1938 });
1939 };
1940
1941 auto *RegClass = TRI->getMinimalPhysRegClass(Reg);
1942 for (const MCPhysReg &PR : *RegClass) {
1943 if (DefinedInBB.available(Reg: PR) && UsedInBetween.available(Reg: PR) &&
1944 !RegInfo.isReserved(PhysReg: PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1945 CanBeUsedForAllClasses(PR)) {
1946 DefinedInBB.addReg(Reg: PR);
1947 LLVM_DEBUG(dbgs() << "Found rename register " << printReg(PR, TRI)
1948 << "\n");
1949 return {PR};
1950 }
1951 }
1952 LLVM_DEBUG(dbgs() << "No rename register found from "
1953 << TRI->getRegClassName(RegClass) << "\n");
1954 return std::nullopt;
1955}
1956
1957// For store pairs: returns a register from FirstMI to the beginning of the
1958// block that can be renamed.
1959// For load pairs: returns a register from FirstMI to MI that can be renamed.
1960static std::optional<MCPhysReg> findRenameRegForSameLdStRegPair(
1961 std::optional<bool> MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI,
1962 Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween,
1963 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1964 const TargetRegisterInfo *TRI) {
1965 std::optional<MCPhysReg> RenameReg;
1966 if (!DebugCounter::shouldExecute(Counter&: RegRenamingCounter))
1967 return RenameReg;
1968
1969 auto *RegClass = TRI->getMinimalPhysRegClass(Reg: getLdStRegOp(MI&: FirstMI).getReg());
1970 MachineFunction &MF = *FirstMI.getParent()->getParent();
1971 if (!RegClass || !MF.getRegInfo().tracksLiveness())
1972 return RenameReg;
1973
1974 const bool IsLoad = FirstMI.mayLoad();
1975
1976 if (!MaybeCanRename) {
1977 if (IsLoad)
1978 MaybeCanRename = {canRenameUntilSecondLoad(FirstLoad&: FirstMI, SecondLoad&: MI, UsedInBetween,
1979 RequiredClasses, TRI)};
1980 else
1981 MaybeCanRename = {
1982 canRenameUpToDef(FirstMI, UsedInBetween, RequiredClasses, TRI)};
1983 }
1984
1985 if (*MaybeCanRename) {
1986 RenameReg = tryToFindRegisterToRename(MF, Reg, DefinedInBB, UsedInBetween,
1987 RequiredClasses, TRI);
1988 }
1989 return RenameReg;
1990}
1991
1992/// Scan the instructions looking for a load/store that can be combined with the
1993/// current instruction into a wider equivalent or a load/store pair.
1994MachineBasicBlock::iterator
1995AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
1996 LdStPairFlags &Flags, unsigned Limit,
1997 bool FindNarrowMerge) {
1998 MachineBasicBlock::iterator E = I->getParent()->end();
1999 MachineBasicBlock::iterator MBBI = I;
2000 MachineInstr &FirstMI = *I;
2001 MBBI = next_nodbg(It: MBBI, End: E);
2002
2003 bool MayLoad = FirstMI.mayLoad();
2004 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI&: FirstMI);
2005 Register Reg = getLdStRegOp(MI&: FirstMI).getReg();
2006 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: FirstMI).getReg();
2007 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: FirstMI).getImm();
2008 int OffsetStride = IsUnscaled ? TII->getMemScale(MI: FirstMI) : 1;
2009 bool IsPromotableZeroStore = isPromotableZeroStoreInst(MI&: FirstMI);
2010
2011 std::optional<bool> MaybeCanRename;
2012 if (!Subtarget->getCLOpts().load_store_renaming)
2013 MaybeCanRename = {false};
2014
2015 SmallPtrSet<const TargetRegisterClass *, 5> RequiredClasses;
2016 LiveRegUnits UsedInBetween;
2017 UsedInBetween.init(TRI: *TRI);
2018
2019 Flags.clearRenameReg();
2020
2021 // Track which register units have been modified and used between the first
2022 // insn (inclusive) and the second insn.
2023 ModifiedRegUnits.clear();
2024 UsedRegUnits.clear();
2025
2026 // Remember any instructions that read/write memory between FirstMI and MI.
2027 SmallVector<MachineInstr *, 4> MemInsns;
2028
2029 LLVM_DEBUG(dbgs() << "Find match for: "; FirstMI.dump());
2030 for (unsigned Count = 0; MBBI != E && Count < Limit;
2031 MBBI = next_nodbg(It: MBBI, End: E)) {
2032 MachineInstr &MI = *MBBI;
2033 LLVM_DEBUG(dbgs() << "Analysing 2nd insn: "; MI.dump());
2034
2035 UsedInBetween.accumulate(MI);
2036
2037 // Don't count transient instructions towards the search limit since there
2038 // may be different numbers of them if e.g. debug information is present.
2039 if (!MI.isTransient())
2040 ++Count;
2041
2042 Flags.setSExtIdx(-1);
2043 if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
2044 AArch64InstrInfo::getLdStOffsetOp(MI).isImm()) {
2045 assert(MI.mayLoadOrStore() && "Expected memory operation.");
2046 // If we've found another instruction with the same opcode, check to see
2047 // if the base and offset are compatible with our starting instruction.
2048 // These instructions all have scaled immediate operands, so we just
2049 // check for +1/-1. Make sure to check the new instruction offset is
2050 // actually an immediate and not a symbolic reference destined for
2051 // a relocation.
2052 Register MIBaseReg = AArch64InstrInfo::getLdStBaseOp(MI).getReg();
2053 int MIOffset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2054 bool MIIsUnscaled = TII->hasUnscaledLdStOffset(MI);
2055 if (IsUnscaled != MIIsUnscaled) {
2056 // We're trying to pair instructions that differ in how they are scaled.
2057 // If FirstMI is scaled then scale the offset of MI accordingly.
2058 // Otherwise, do the opposite (i.e., make MI's offset unscaled).
2059 int MemSize = TII->getMemScale(MI);
2060 if (MIIsUnscaled) {
2061 // If the unscaled offset isn't a multiple of the MemSize, we can't
2062 // pair the operations together: bail and keep looking.
2063 if (MIOffset % MemSize) {
2064 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2065 UsedRegUnits, TRI);
2066 MemInsns.push_back(Elt: &MI);
2067 continue;
2068 }
2069 MIOffset /= MemSize;
2070 } else {
2071 MIOffset *= MemSize;
2072 }
2073 }
2074
2075 bool IsPreLdSt = isPreLdStPairCandidate(FirstMI, MI);
2076
2077 if (BaseReg == MIBaseReg) {
2078 // If the offset of the second ld/st is not equal to the size of the
2079 // destination register it can’t be paired with a pre-index ld/st
2080 // pair. Additionally if the base reg is used or modified the operations
2081 // can't be paired: bail and keep looking.
2082 if (IsPreLdSt) {
2083 bool IsOutOfBounds = MIOffset != TII->getMemScale(MI);
2084 bool IsBaseRegUsed = !UsedRegUnits.available(
2085 Reg: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2086 bool IsBaseRegModified = !ModifiedRegUnits.available(
2087 Reg: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2088 // If the stored value and the address of the second instruction is
2089 // the same, it needs to be using the updated register and therefore
2090 // it must not be folded.
2091 bool IsMIRegTheSame =
2092 TRI->regsOverlap(RegA: getLdStRegOp(MI).getReg(),
2093 RegB: AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2094 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
2095 IsMIRegTheSame) {
2096 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2097 UsedRegUnits, TRI);
2098 MemInsns.push_back(Elt: &MI);
2099 continue;
2100 }
2101 } else {
2102 if ((Offset != MIOffset + OffsetStride) &&
2103 (Offset + OffsetStride != MIOffset)) {
2104 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2105 UsedRegUnits, TRI);
2106 MemInsns.push_back(Elt: &MI);
2107 continue;
2108 }
2109 }
2110
2111 int MinOffset = Offset < MIOffset ? Offset : MIOffset;
2112 if (FindNarrowMerge) {
2113 // If the alignment requirements of the scaled wide load/store
2114 // instruction can't express the offset of the scaled narrow input,
2115 // bail and keep looking. For promotable zero stores, allow only when
2116 // the stored value is the same (i.e., WZR).
2117 if ((!IsUnscaled && alignTo(Num: MinOffset, PowOf2: 2) != MinOffset) ||
2118 (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) {
2119 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2120 UsedRegUnits, TRI);
2121 MemInsns.push_back(Elt: &MI);
2122 continue;
2123 }
2124 } else {
2125 // Pairwise instructions have a 7-bit signed offset field. Single
2126 // insns have a 12-bit unsigned offset field. If the resultant
2127 // immediate offset of merging these instructions is out of range for
2128 // a pairwise instruction, bail and keep looking.
2129 if (!inBoundsForPair(IsUnscaled, Offset: MinOffset, OffsetStride)) {
2130 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2131 UsedRegUnits, TRI);
2132 MemInsns.push_back(Elt: &MI);
2133 LLVM_DEBUG(dbgs() << "Offset doesn't fit in immediate, "
2134 << "keep looking.\n");
2135 continue;
2136 }
2137 // If the alignment requirements of the paired (scaled) instruction
2138 // can't express the offset of the unscaled input, bail and keep
2139 // looking.
2140 if (IsUnscaled && (alignTo(Num: MinOffset, PowOf2: OffsetStride) != MinOffset)) {
2141 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2142 UsedRegUnits, TRI);
2143 MemInsns.push_back(Elt: &MI);
2144 LLVM_DEBUG(dbgs()
2145 << "Offset doesn't fit due to alignment requirements, "
2146 << "keep looking.\n");
2147 continue;
2148 }
2149 }
2150
2151 // If the BaseReg has been modified, then we cannot do the optimization.
2152 // For example, in the following pattern
2153 // ldr x1 [x2]
2154 // ldr x2 [x3]
2155 // ldr x4 [x2, #8],
2156 // the first and third ldr cannot be converted to ldp x1, x4, [x2]
2157 if (!ModifiedRegUnits.available(Reg: BaseReg))
2158 return E;
2159
2160 const bool SameLoadReg = MayLoad && TRI->isSuperOrSubRegisterEq(
2161 RegA: Reg, RegB: getLdStRegOp(MI).getReg());
2162
2163 // If the Rt of the second instruction (destination register of the
2164 // load) was not modified or used between the two instructions and none
2165 // of the instructions between the second and first alias with the
2166 // second, we can combine the second into the first.
2167 bool RtNotModified =
2168 ModifiedRegUnits.available(Reg: getLdStRegOp(MI).getReg());
2169 bool RtNotUsed = !(MI.mayLoad() && !SameLoadReg &&
2170 !UsedRegUnits.available(Reg: getLdStRegOp(MI).getReg()));
2171
2172 LLVM_DEBUG(dbgs() << "Checking, can combine 2nd into 1st insn:\n"
2173 << "Reg '" << getLdStRegOp(MI) << "' not modified: "
2174 << (RtNotModified ? "true" : "false") << "\n"
2175 << "Reg '" << getLdStRegOp(MI) << "' not used: "
2176 << (RtNotUsed ? "true" : "false") << "\n");
2177
2178 if (RtNotModified && RtNotUsed && !mayAlias(MIa&: MI, MemInsns, AA)) {
2179 // For pairs loading into the same reg, try to find a renaming
2180 // opportunity to allow the renaming of Reg between FirstMI and MI
2181 // and combine MI into FirstMI; otherwise bail and keep looking.
2182 if (SameLoadReg) {
2183 std::optional<MCPhysReg> RenameReg =
2184 findRenameRegForSameLdStRegPair(MaybeCanRename, FirstMI, MI,
2185 Reg, DefinedInBB, UsedInBetween,
2186 RequiredClasses, TRI);
2187 if (!RenameReg) {
2188 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2189 UsedRegUnits, TRI);
2190 MemInsns.push_back(Elt: &MI);
2191 LLVM_DEBUG(dbgs() << "Can't find reg for renaming, "
2192 << "keep looking.\n");
2193 continue;
2194 }
2195 Flags.setRenameReg(*RenameReg);
2196 }
2197
2198 Flags.setMergeForward(false);
2199 if (!SameLoadReg)
2200 Flags.clearRenameReg();
2201 return MBBI;
2202 }
2203
2204 // Likewise, if the Rt of the first instruction is not modified or used
2205 // between the two instructions and none of the instructions between the
2206 // first and the second alias with the first, we can combine the first
2207 // into the second.
2208 RtNotModified = !(
2209 MayLoad && !UsedRegUnits.available(Reg: getLdStRegOp(MI&: FirstMI).getReg()));
2210
2211 LLVM_DEBUG(dbgs() << "Checking, can combine 1st into 2nd insn:\n"
2212 << "Reg '" << getLdStRegOp(FirstMI)
2213 << "' not modified: "
2214 << (RtNotModified ? "true" : "false") << "\n");
2215
2216 if (RtNotModified && !mayAlias(MIa&: FirstMI, MemInsns, AA)) {
2217 if (ModifiedRegUnits.available(Reg: getLdStRegOp(MI&: FirstMI).getReg())) {
2218 Flags.setMergeForward(true);
2219 Flags.clearRenameReg();
2220 return MBBI;
2221 }
2222
2223 std::optional<MCPhysReg> RenameReg = findRenameRegForSameLdStRegPair(
2224 MaybeCanRename, FirstMI, MI, Reg, DefinedInBB, UsedInBetween,
2225 RequiredClasses, TRI);
2226 if (RenameReg) {
2227 Flags.setMergeForward(true);
2228 Flags.setRenameReg(*RenameReg);
2229 return MBBI;
2230 }
2231 }
2232 LLVM_DEBUG(dbgs() << "Unable to combine these instructions due to "
2233 << "interference in between, keep looking.\n");
2234 }
2235 }
2236
2237 // If the instruction wasn't a matching load or store. Stop searching if we
2238 // encounter a call instruction that might modify memory.
2239 if (MI.isCall()) {
2240 LLVM_DEBUG(dbgs() << "Found a call, stop looking.\n");
2241 return E;
2242 }
2243
2244 // Update modified / uses register units.
2245 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2246
2247 // Otherwise, if the base register is modified, we have no match, so
2248 // return early.
2249 if (!ModifiedRegUnits.available(Reg: BaseReg)) {
2250 LLVM_DEBUG(dbgs() << "Base reg is modified, stop looking.\n");
2251 return E;
2252 }
2253
2254 // Update list of instructions that read/write memory.
2255 if (MI.mayLoadOrStore())
2256 MemInsns.push_back(Elt: &MI);
2257 }
2258 return E;
2259}
2260
2261static MachineBasicBlock::iterator
2262maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI) {
2263 assert((MI.getOpcode() == AArch64::SUBXri ||
2264 MI.getOpcode() == AArch64::ADDXri) &&
2265 "Expected a register update instruction");
2266 auto End = MI.getParent()->end();
2267 if (MaybeCFI == End ||
2268 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
2269 !(MI.getFlag(Flag: MachineInstr::FrameSetup) ||
2270 MI.getFlag(Flag: MachineInstr::FrameDestroy)) ||
2271 MI.getOperand(i: 0).getReg() != AArch64::SP)
2272 return End;
2273
2274 const MachineFunction &MF = *MI.getParent()->getParent();
2275 unsigned CFIIndex = MaybeCFI->getOperand(i: 0).getCFIIndex();
2276 const MCCFIInstruction &CFI = MF.getFrameInstructions()[CFIIndex];
2277 switch (CFI.getOperation()) {
2278 case MCCFIInstruction::OpDefCfa:
2279 case MCCFIInstruction::OpDefCfaOffset:
2280 return MaybeCFI;
2281 default:
2282 return End;
2283 }
2284}
2285
2286std::optional<MachineBasicBlock::iterator> AArch64LoadStoreOpt::mergeUpdateInsn(
2287 MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update,
2288 bool IsForward, bool IsPreIdx, bool MergeEither) {
2289 assert((Update->getOpcode() == AArch64::ADDXri ||
2290 Update->getOpcode() == AArch64::SUBXri) &&
2291 "Unexpected base register update instruction to merge!");
2292 MachineBasicBlock::iterator E = I->getParent()->end();
2293 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
2294
2295 // If updating the SP and the following instruction is CFA offset related CFI,
2296 // make sure the CFI follows the SP update either by merging at the location
2297 // of the update or by moving the CFI after the merged instruction. If unable
2298 // to do so, bail.
2299 MachineBasicBlock::iterator InsertPt = I;
2300 if (IsForward) {
2301 assert(IsPreIdx);
2302 if (auto CFI = maybeMoveCFI(MI&: *Update, MaybeCFI: next_nodbg(It: Update, End: E)); CFI != E) {
2303 if (MergeEither) {
2304 InsertPt = Update;
2305 } else {
2306 // Take care not to reorder CFIs.
2307 if (std::any_of(first: std::next(x: CFI), last: I, pred: [](const auto &Insn) {
2308 return Insn.getOpcode() == TargetOpcode::CFI_INSTRUCTION;
2309 }))
2310 return std::nullopt;
2311
2312 MachineBasicBlock *MBB = InsertPt->getParent();
2313 MBB->splice(Where: std::next(x: InsertPt), Other: MBB, From: CFI);
2314 }
2315 }
2316 }
2317
2318 // Return the instruction following the merged instruction, which is
2319 // the instruction following our unmerged load. Unless that's the add/sub
2320 // instruction we're merging, in which case it's the one after that.
2321 if (NextI == Update)
2322 NextI = next_nodbg(It: NextI, End: E);
2323
2324 int Value = Update->getOperand(i: 2).getImm();
2325 assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 &&
2326 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
2327 if (Update->getOpcode() == AArch64::SUBXri)
2328 Value = -Value;
2329
2330 unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(Opc: I->getOpcode())
2331 : getPostIndexedOpcode(Opc: I->getOpcode());
2332 MachineInstrBuilder MIB;
2333 int Scale, MinOffset, MaxOffset;
2334 getPrePostIndexedMemOpInfo(MI: *I, Scale, MinOffset, MaxOffset);
2335 if (!AArch64InstrInfo::isPairedLdSt(MI: *I)) {
2336 // Non-paired instruction.
2337 MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
2338 MCID: TII->get(Opcode: NewOpc))
2339 .add(MO: Update->getOperand(i: 0))
2340 .add(MO: getLdStRegOp(MI&: *I))
2341 .add(MO: AArch64InstrInfo::getLdStBaseOp(MI: *I))
2342 .addImm(Val: Value / Scale)
2343 .setMemRefs(I->memoperands())
2344 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2345 } else {
2346 // Paired instruction.
2347 MIB = BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
2348 MCID: TII->get(Opcode: NewOpc))
2349 .add(MO: Update->getOperand(i: 0))
2350 .add(MO: getLdStRegOp(MI&: *I, PairedRegOp: 0))
2351 .add(MO: getLdStRegOp(MI&: *I, PairedRegOp: 1))
2352 .add(MO: AArch64InstrInfo::getLdStBaseOp(MI: *I))
2353 .addImm(Val: Value / Scale)
2354 .setMemRefs(I->memoperands())
2355 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2356 }
2357
2358 if (IsPreIdx) {
2359 ++NumPreFolded;
2360 LLVM_DEBUG(dbgs() << "Creating pre-indexed load/store.");
2361 } else {
2362 ++NumPostFolded;
2363 LLVM_DEBUG(dbgs() << "Creating post-indexed load/store.");
2364 }
2365 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2366 LLVM_DEBUG(I->print(dbgs()));
2367 LLVM_DEBUG(dbgs() << " ");
2368 LLVM_DEBUG(Update->print(dbgs()));
2369 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2370 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
2371 LLVM_DEBUG(dbgs() << "\n");
2372
2373 // Erase the old instructions for the block.
2374 I->eraseFromParent();
2375 Update->eraseFromParent();
2376
2377 return NextI;
2378}
2379
2380MachineBasicBlock::iterator
2381AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I,
2382 MachineBasicBlock::iterator Update,
2383 unsigned Offset, int Scale) {
2384 assert((Update->getOpcode() == AArch64::MOVKWi) &&
2385 "Unexpected const mov instruction to merge!");
2386 MachineBasicBlock::iterator E = I->getParent()->end();
2387 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
2388 MachineBasicBlock::iterator PrevI = prev_nodbg(It: Update, Begin: E);
2389 MachineInstr &MemMI = *I;
2390 unsigned Mask = (1 << 12) * Scale - 1;
2391 unsigned Low = Offset & Mask;
2392 unsigned High = Offset - Low;
2393 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2394 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getReg();
2395 MachineInstrBuilder AddMIB, MemMIB;
2396
2397 // Add IndexReg, BaseReg, High (the BaseReg may be SP)
2398 AddMIB =
2399 BuildMI(BB&: *I->getParent(), I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
2400 .addDef(RegNo: IndexReg)
2401 .addUse(RegNo: BaseReg)
2402 .addImm(Val: High >> 12) // shifted value
2403 .addImm(Val: 12); // shift 12
2404 (void)AddMIB;
2405 // Ld/St DestReg, IndexReg, Imm12
2406 unsigned NewOpc = getBaseAddressOpcode(Opc: I->getOpcode());
2407 MemMIB = BuildMI(BB&: *I->getParent(), I, MIMD: I->getDebugLoc(), MCID: TII->get(Opcode: NewOpc))
2408 .add(MO: getLdStRegOp(MI&: MemMI))
2409 .add(MO: AArch64InstrInfo::getLdStOffsetOp(MI: MemMI))
2410 .addImm(Val: Low / Scale)
2411 .setMemRefs(I->memoperands())
2412 .setMIFlags(I->mergeFlagsWith(Other: *Update));
2413 (void)MemMIB;
2414
2415 ++NumConstOffsetFolded;
2416 LLVM_DEBUG(dbgs() << "Creating base address load/store.\n");
2417 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2418 LLVM_DEBUG(PrevI->print(dbgs()));
2419 LLVM_DEBUG(dbgs() << " ");
2420 LLVM_DEBUG(Update->print(dbgs()));
2421 LLVM_DEBUG(dbgs() << " ");
2422 LLVM_DEBUG(I->print(dbgs()));
2423 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2424 LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs()));
2425 LLVM_DEBUG(dbgs() << " ");
2426 LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs()));
2427 LLVM_DEBUG(dbgs() << "\n");
2428
2429 // Erase the old instructions for the block.
2430 I->eraseFromParent();
2431 PrevI->eraseFromParent();
2432 Update->eraseFromParent();
2433
2434 return NextI;
2435}
2436
2437bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
2438 MachineInstr &MI,
2439 unsigned BaseReg, int Offset) {
2440 switch (MI.getOpcode()) {
2441 default:
2442 break;
2443 case AArch64::SUBXri:
2444 case AArch64::ADDXri:
2445 // Make sure it's a vanilla immediate operand, not a relocation or
2446 // anything else we can't handle.
2447 if (!MI.getOperand(i: 2).isImm())
2448 break;
2449 // Watch out for 1 << 12 shifted value.
2450 if (AArch64_AM::getShiftValue(Imm: MI.getOperand(i: 3).getImm()))
2451 break;
2452
2453 // The update instruction source and destination register must be the
2454 // same as the load/store base register.
2455 if (MI.getOperand(i: 0).getReg() != BaseReg ||
2456 MI.getOperand(i: 1).getReg() != BaseReg)
2457 break;
2458
2459 int UpdateOffset = MI.getOperand(i: 2).getImm();
2460 if (MI.getOpcode() == AArch64::SUBXri)
2461 UpdateOffset = -UpdateOffset;
2462
2463 // The immediate must be a multiple of the scaling factor of the pre/post
2464 // indexed instruction.
2465 int Scale, MinOffset, MaxOffset;
2466 getPrePostIndexedMemOpInfo(MI: MemMI, Scale, MinOffset, MaxOffset);
2467 if (UpdateOffset % Scale != 0)
2468 break;
2469
2470 // Scaled offset must fit in the instruction immediate.
2471 int ScaledOffset = UpdateOffset / Scale;
2472 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
2473 break;
2474
2475 // If we have a non-zero Offset, we check that it matches the amount
2476 // we're adding to the register.
2477 if (!Offset || Offset == UpdateOffset)
2478 return true;
2479 break;
2480 }
2481 return false;
2482}
2483
2484bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI,
2485 MachineInstr &MI,
2486 unsigned IndexReg,
2487 unsigned &Offset) {
2488 // The update instruction source and destination register must be the
2489 // same as the load/store index register.
2490 if (MI.getOpcode() == AArch64::MOVKWi &&
2491 TRI->isSuperOrSubRegisterEq(RegA: IndexReg, RegB: MI.getOperand(i: 1).getReg())) {
2492
2493 // movz + movk hold a large offset of a Ld/St instruction.
2494 MachineBasicBlock::iterator B = MI.getParent()->begin();
2495 MachineBasicBlock::iterator MBBI = &MI;
2496 // Skip the scene when the MI is the first instruction of a block.
2497 if (MBBI == B)
2498 return false;
2499 MBBI = prev_nodbg(It: MBBI, Begin: B);
2500 MachineInstr &MovzMI = *MBBI;
2501 // Make sure the MOVKWi and MOVZWi set the same register.
2502 if (MovzMI.getOpcode() == AArch64::MOVZWi &&
2503 MovzMI.getOperand(i: 0).getReg() == MI.getOperand(i: 0).getReg()) {
2504 unsigned Low = MovzMI.getOperand(i: 1).getImm();
2505 unsigned High = MI.getOperand(i: 2).getImm() << MI.getOperand(i: 3).getImm();
2506 Offset = High + Low;
2507 // 12-bit optionally shifted immediates are legal for adds.
2508 return Offset >> 24 == 0;
2509 }
2510 }
2511 return false;
2512}
2513
2514MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
2515 MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
2516 MachineBasicBlock::iterator E = I->getParent()->end();
2517 MachineInstr &MemMI = *I;
2518 MachineBasicBlock::iterator MBBI = I;
2519
2520 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2521 int MIUnscaledOffset = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getImm() *
2522 TII->getMemScale(MI: MemMI);
2523
2524 // Scan forward looking for post-index opportunities. Updating instructions
2525 // can't be formed if the memory instruction doesn't have the offset we're
2526 // looking for.
2527 if (MIUnscaledOffset != UnscaledOffset)
2528 return E;
2529
2530 // If the base register overlaps a source/destination register, we can't
2531 // merge the update. This does not apply to tag store instructions which
2532 // ignore the address part of the source register.
2533 // This does not apply to STGPi as well, which does not have unpredictable
2534 // behavior in this case unlike normal stores, and always performs writeback
2535 // after reading the source register value.
2536 if (!isTagStore(MI: MemMI) && MemMI.getOpcode() != AArch64::STGPi) {
2537 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MI: MemMI);
2538 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
2539 Register DestReg = getLdStRegOp(MI&: MemMI, PairedRegOp: i).getReg();
2540 if (DestReg == BaseReg || TRI->isSubRegister(RegA: BaseReg, RegB: DestReg))
2541 return E;
2542 }
2543 }
2544
2545 // Track which register units have been modified and used between the first
2546 // insn (inclusive) and the second insn.
2547 ModifiedRegUnits.clear();
2548 UsedRegUnits.clear();
2549 MBBI = next_nodbg(It: MBBI, End: E);
2550
2551 // We can't post-increment the stack pointer if any instruction between
2552 // the memory access (I) and the increment (MBBI) can access the memory
2553 // region defined by [SP, MBBI].
2554 const bool BaseRegSP = BaseReg == AArch64::SP;
2555 if (BaseRegSP && needsWinCFI(MF: I->getMF())) {
2556 // FIXME: For now, we always block the optimization over SP in windows
2557 // targets as it requires to adjust the unwind/debug info, messing up
2558 // the unwind info can actually cause a miscompile.
2559 return E;
2560 }
2561
2562 unsigned Count = 0;
2563 MachineBasicBlock *CurMBB = I->getParent();
2564 // choice of next block to visit is liveins-based
2565 bool VisitSucc = CurMBB->getParent()->getRegInfo().tracksLiveness();
2566
2567 while (true) {
2568 for (MachineBasicBlock::iterator CurEnd = CurMBB->end();
2569 MBBI != CurEnd && Count < Limit; MBBI = next_nodbg(It: MBBI, End: CurEnd)) {
2570 MachineInstr &MI = *MBBI;
2571
2572 // Don't count transient instructions towards the search limit since there
2573 // may be different numbers of them if e.g. debug information is present.
2574 if (!MI.isTransient())
2575 ++Count;
2576
2577 // If we found a match, return it.
2578 if (isMatchingUpdateInsn(MemMI&: *I, MI, BaseReg, Offset: UnscaledOffset))
2579 return MBBI;
2580
2581 // Update the status of what the instruction clobbered and used.
2582 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
2583 TRI);
2584
2585 // Otherwise, if the base register is used or modified, we have no match,
2586 // so return early. If we are optimizing SP, do not allow instructions
2587 // that may load or store in between the load and the optimized value
2588 // update.
2589 if (!ModifiedRegUnits.available(Reg: BaseReg) ||
2590 !UsedRegUnits.available(Reg: BaseReg) ||
2591 (BaseRegSP && MBBI->mayLoadOrStore()))
2592 return E;
2593 }
2594
2595 if (!VisitSucc || Limit <= Count)
2596 break;
2597
2598 // Try to go downward to successors along a CF path w/o side enters
2599 // such that BaseReg is alive along it but not at its exits
2600 MachineBasicBlock *SuccToVisit = nullptr;
2601 unsigned LiveSuccCount = 0;
2602 for (MachineBasicBlock *Succ : CurMBB->successors()) {
2603 for (MCRegAliasIterator AI(BaseReg, TRI, true); AI.isValid(); ++AI) {
2604 if (Succ->isLiveIn(Reg: *AI)) {
2605 if (LiveSuccCount++)
2606 return E;
2607 if (Succ->pred_size() == 1)
2608 SuccToVisit = Succ;
2609 break;
2610 }
2611 }
2612 }
2613 if (!SuccToVisit)
2614 break;
2615 CurMBB = SuccToVisit;
2616 MBBI = CurMBB->begin();
2617 }
2618
2619 return E;
2620}
2621
2622MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
2623 MachineBasicBlock::iterator I, unsigned Limit, bool &MergeEither) {
2624 MachineBasicBlock::iterator B = I->getParent()->begin();
2625 MachineBasicBlock::iterator E = I->getParent()->end();
2626 MachineInstr &MemMI = *I;
2627 MachineBasicBlock::iterator MBBI = I;
2628 MachineFunction &MF = *MemMI.getMF();
2629
2630 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MI: MemMI).getReg();
2631 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getImm();
2632
2633 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MI: MemMI);
2634 Register DestReg[] = {getLdStRegOp(MI&: MemMI, PairedRegOp: 0).getReg(),
2635 IsPairedInsn ? getLdStRegOp(MI&: MemMI, PairedRegOp: 1).getReg()
2636 : Register()};
2637
2638 // If the load/store is the first instruction in the block, there's obviously
2639 // not any matching update. Ditto if the memory offset isn't zero.
2640 if (MBBI == B || Offset != 0)
2641 return E;
2642 // If the base register overlaps a destination register, we can't
2643 // merge the update.
2644 if (!isTagStore(MI: MemMI)) {
2645 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i)
2646 if (DestReg[i] == BaseReg || TRI->isSubRegister(RegA: BaseReg, RegB: DestReg[i]))
2647 return E;
2648 }
2649
2650 const bool BaseRegSP = BaseReg == AArch64::SP;
2651 if (BaseRegSP && needsWinCFI(MF: I->getMF())) {
2652 // FIXME: For now, we always block the optimization over SP in windows
2653 // targets as it requires to adjust the unwind/debug info, messing up
2654 // the unwind info can actually cause a miscompile.
2655 return E;
2656 }
2657
2658 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2659 unsigned RedZoneSize =
2660 Subtarget.getTargetLowering()->getRedZoneSize(F: MF.getFunction());
2661
2662 // Track which register units have been modified and used between the first
2663 // insn (inclusive) and the second insn.
2664 ModifiedRegUnits.clear();
2665 UsedRegUnits.clear();
2666 unsigned Count = 0;
2667 bool MemAccessBeforeSPPreInc = false;
2668 MergeEither = true;
2669 do {
2670 MBBI = prev_nodbg(It: MBBI, Begin: B);
2671 MachineInstr &MI = *MBBI;
2672
2673 // Don't count transient instructions towards the search limit since there
2674 // may be different numbers of them if e.g. debug information is present.
2675 if (!MI.isTransient())
2676 ++Count;
2677
2678 // If we found a match, return it.
2679 if (isMatchingUpdateInsn(MemMI&: *I, MI, BaseReg, Offset)) {
2680 // Check that the update value is within our red zone limit (which may be
2681 // zero).
2682 if (MemAccessBeforeSPPreInc && MBBI->getOperand(i: 2).getImm() > RedZoneSize)
2683 return E;
2684 return MBBI;
2685 }
2686
2687 // Update the status of what the instruction clobbered and used.
2688 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2689
2690 // Otherwise, if the base register is used or modified, we have no match, so
2691 // return early.
2692 if (!ModifiedRegUnits.available(Reg: BaseReg) ||
2693 !UsedRegUnits.available(Reg: BaseReg))
2694 return E;
2695
2696 // If we have a destination register (i.e. a load instruction) and a
2697 // destination register is used or modified, then we can only merge forward,
2698 // i.e. the combined instruction is put in the place of the memory
2699 // instruction. Same applies if we see a memory access or side effects.
2700 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects() ||
2701 (DestReg[0].isValid() && !(ModifiedRegUnits.available(Reg: DestReg[0]) &&
2702 UsedRegUnits.available(Reg: DestReg[0]))) ||
2703 (DestReg[1].isValid() && !(ModifiedRegUnits.available(Reg: DestReg[1]) &&
2704 UsedRegUnits.available(Reg: DestReg[1]))))
2705 MergeEither = false;
2706
2707 // Keep track if we have a memory access before an SP pre-increment, in this
2708 // case we need to validate later that the update amount respects the red
2709 // zone.
2710 if (BaseRegSP && MBBI->mayLoadOrStore())
2711 MemAccessBeforeSPPreInc = true;
2712 } while (MBBI != B && Count < Limit);
2713 return E;
2714}
2715
2716MachineBasicBlock::iterator
2717AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
2718 MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) {
2719 MachineBasicBlock::iterator B = I->getParent()->begin();
2720 MachineBasicBlock::iterator E = I->getParent()->end();
2721 MachineInstr &MemMI = *I;
2722 MachineBasicBlock::iterator MBBI = I;
2723
2724 // If the load is the first instruction in the block, there's obviously
2725 // not any matching load or store.
2726 if (MBBI == B)
2727 return E;
2728
2729 // Make sure the IndexReg is killed and the shift amount is zero.
2730 // TODO: Relex this restriction to extend, simplify processing now.
2731 if (!AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).isKill() ||
2732 !AArch64InstrInfo::getLdStAmountOp(MI: MemMI).isImm() ||
2733 (AArch64InstrInfo::getLdStAmountOp(MI: MemMI).getImm() != 0))
2734 return E;
2735
2736 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MI: MemMI).getReg();
2737
2738 // Track which register units have been modified and used between the first
2739 // insn (inclusive) and the second insn.
2740 ModifiedRegUnits.clear();
2741 UsedRegUnits.clear();
2742 unsigned Count = 0;
2743 do {
2744 MBBI = prev_nodbg(It: MBBI, Begin: B);
2745 MachineInstr &MI = *MBBI;
2746
2747 // Don't count transient instructions towards the search limit since there
2748 // may be different numbers of them if e.g. debug information is present.
2749 if (!MI.isTransient())
2750 ++Count;
2751
2752 // If we found a match, return it.
2753 if (isMatchingMovConstInsn(MemMI&: *I, MI, IndexReg, Offset)) {
2754 return MBBI;
2755 }
2756
2757 // Update the status of what the instruction clobbered and used.
2758 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2759
2760 // Otherwise, if the index register is used or modified, we have no match,
2761 // so return early.
2762 if (!ModifiedRegUnits.available(Reg: IndexReg) ||
2763 !UsedRegUnits.available(Reg: IndexReg))
2764 return E;
2765
2766 } while (MBBI != B && Count < Limit);
2767 return E;
2768}
2769
2770bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2771 MachineBasicBlock::iterator &MBBI) {
2772 MachineInstr &MI = *MBBI;
2773 // If this is a volatile load, don't mess with it.
2774 if (MI.hasOrderedMemoryRef())
2775 return false;
2776
2777 if (needsWinCFI(MF: MI.getMF()) && MI.getFlag(Flag: MachineInstr::FrameDestroy))
2778 return false;
2779
2780 // Make sure this is a reg+imm.
2781 // FIXME: It is possible to extend it to handle reg+reg cases.
2782 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
2783 return false;
2784
2785 // Look backward up to the scan limit.
2786 MachineBasicBlock::iterator StoreI;
2787 if (findMatchingStore(I: MBBI, Limit: Subtarget->getCLOpts().load_store_scan_limit,
2788 StoreI)) {
2789 ++NumLoadsFromStoresPromoted;
2790 // Promote the load. Keeping the iterator straight is a
2791 // pain, so we let the merge routine tell us what the next instruction
2792 // is after it's done mucking about.
2793 MBBI = promoteLoadFromStore(LoadI: MBBI, StoreI);
2794 return true;
2795 }
2796 return false;
2797}
2798
2799// Merge adjacent zero stores into a wider store.
2800bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2801 MachineBasicBlock::iterator &MBBI) {
2802 assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store.");
2803 MachineInstr &MI = *MBBI;
2804 MachineBasicBlock::iterator E = MI.getParent()->end();
2805
2806 if (!TII->isCandidateToMergeOrPair(MI))
2807 return false;
2808
2809 // Look ahead up to the scan limit for a mergeable instruction.
2810 LdStPairFlags Flags;
2811 MachineBasicBlock::iterator MergeMI = findMatchingInsn(
2812 I: MBBI, Flags, Limit: Subtarget->getCLOpts().load_store_scan_limit,
2813 /*FindNarrowMerge=*/true);
2814 if (MergeMI != E) {
2815 ++NumZeroStoresPromoted;
2816
2817 // Keeping the iterator straight is a pain, so we let the merge routine tell
2818 // us what the next instruction is after it's done mucking about.
2819 MBBI = mergeNarrowZeroStores(I: MBBI, MergeMI, Flags);
2820 return true;
2821 }
2822 return false;
2823}
2824
2825// Find loads and stores that can be merged into a single load or store pair
2826// instruction.
2827bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
2828 MachineInstr &MI = *MBBI;
2829 MachineBasicBlock::iterator E = MI.getParent()->end();
2830
2831 if (!TII->isCandidateToMergeOrPair(MI))
2832 return false;
2833
2834 // If disable-ldp feature is opted, do not emit ldp.
2835 if (MI.mayLoad() && Subtarget->hasDisableLdp())
2836 return false;
2837
2838 // If disable-stp feature is opted, do not emit stp.
2839 if (MI.mayStore() && Subtarget->hasDisableStp())
2840 return false;
2841
2842 // Early exit if the offset is not possible to match. (6 bits of positive
2843 // range, plus allow an extra one in case we find a later insn that matches
2844 // with Offset-1)
2845 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI);
2846 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2847 int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
2848 // Allow one more for offset.
2849 if (Offset > 0)
2850 Offset -= OffsetStride;
2851 if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
2852 return false;
2853
2854 // Look ahead up to the scan limit for a pairable instruction.
2855 LdStPairFlags Flags;
2856 MachineBasicBlock::iterator Paired = findMatchingInsn(
2857 I: MBBI, Flags, Limit: Subtarget->getCLOpts().load_store_scan_limit,
2858 /*FindNarrowMerge=*/false);
2859
2860 if (Paired == E)
2861 return false;
2862
2863 // Keeping the iterator straight is a pain, so we let the merge routine tell
2864 // us what the next instruction is after it's done mucking about.
2865 auto Prev = std::prev(x: MBBI);
2866
2867 // Fetch the memoperand of the load/store that is a candidate for combination.
2868 MachineMemOperand *MemOp =
2869 MI.memoperands_empty() ? nullptr : MI.memoperands().front();
2870
2871 // If a load/store arrives and ldp/stp-aligned-only feature is opted, check
2872 // that the alignment of the source pointer is at least double the alignment
2873 // of the type.
2874 if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) ||
2875 (MI.mayStore() && Subtarget->hasStpAlignedOnly())) {
2876 // If there is no size/align information, cancel the transformation.
2877 if (!MemOp || !MemOp->getMemoryType().isValid()) {
2878 NumFailedAlignmentCheck++;
2879 return false;
2880 }
2881
2882 // Get the needed alignments to check them if
2883 // ldp-aligned-only/stp-aligned-only features are opted.
2884 uint64_t MemAlignment = MemOp->getAlign().value();
2885 uint64_t TypeAlignment =
2886 Align(MemOp->getSize().getValue().getKnownMinValue()).value();
2887
2888 if (MemAlignment < 2 * TypeAlignment) {
2889 NumFailedAlignmentCheck++;
2890 return false;
2891 }
2892 }
2893
2894 ++NumPairCreated;
2895 if (TII->hasUnscaledLdStOffset(MI))
2896 ++NumUnscaledPairCreated;
2897
2898 MBBI = mergePairedInsns(I: MBBI, Paired, Flags);
2899 // Collect liveness info for instructions between Prev and the new position
2900 // MBBI.
2901 for (auto I = std::next(x: Prev); I != MBBI; I++)
2902 updateDefinedRegisters(MI&: *I, Units&: DefinedInBB, TRI);
2903
2904 return true;
2905}
2906
2907bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2908 (MachineBasicBlock::iterator &MBBI) {
2909 MachineInstr &MI = *MBBI;
2910 MachineBasicBlock::iterator E = MI.getParent()->end();
2911 MachineBasicBlock::iterator Update;
2912
2913 // Do not form post-inc addressing mode for volatile accesses. Instructions
2914 // performing register writeback do not set a valid instruction syndrome,
2915 // making it impossible to handle MMIO in protected hypervisors.
2916 // Exclude accesses based on the stack pointer, as these can't be MMIO.
2917 // Also exclude MTE tag store instructions.
2918 if (MBBI->hasOrderedMemoryRef() &&
2919 AArch64InstrInfo::getLdStBaseOp(MI).getReg() != AArch64::SP &&
2920 !isTagStore(MI) && MI.getOpcode() != AArch64::STGPi)
2921 return false;
2922
2923 // Look forward to try to form a post-index instruction. For example,
2924 // ldr x0, [x20]
2925 // add x20, x20, #32
2926 // merged into:
2927 // ldr x0, [x20], #32
2928 Update = findMatchingUpdateInsnForward(
2929 I: MBBI, UnscaledOffset: 0, Limit: Subtarget->getCLOpts().update_scan_limit);
2930 if (Update != E) {
2931 // Merge the update into the ld/st.
2932 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/false,
2933 /*IsPreIdx=*/false,
2934 /*MergeEither=*/false)) {
2935 MBBI = *NextI;
2936 return true;
2937 }
2938 }
2939
2940 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2941 if (TII->hasUnscaledLdStOffset(Opc: MI.getOpcode()))
2942 return false;
2943
2944 // Look back to try to find a pre-index instruction. For example,
2945 // add x0, x0, #8
2946 // ldr x1, [x0]
2947 // merged into:
2948 // ldr x1, [x0, #8]!
2949 bool MergeEither;
2950 Update = findMatchingUpdateInsnBackward(
2951 I: MBBI, Limit: Subtarget->getCLOpts().update_scan_limit, MergeEither);
2952 if (Update != E) {
2953 // Merge the update into the ld/st.
2954 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/true,
2955 /*IsPreIdx=*/true, MergeEither)) {
2956 MBBI = *NextI;
2957 return true;
2958 }
2959 }
2960
2961 // The immediate in the load/store is scaled by the size of the memory
2962 // operation. The immediate in the add we're looking for,
2963 // however, is not, so adjust here.
2964 int UnscaledOffset =
2965 AArch64InstrInfo::getLdStOffsetOp(MI).getImm() * TII->getMemScale(MI);
2966
2967 // Look forward to try to find a pre-index instruction. For example,
2968 // ldr x1, [x0, #64]
2969 // add x0, x0, #64
2970 // merged into:
2971 // ldr x1, [x0, #64]!
2972 Update = findMatchingUpdateInsnForward(
2973 I: MBBI, UnscaledOffset, Limit: Subtarget->getCLOpts().update_scan_limit);
2974 if (Update != E) {
2975 // Merge the update into the ld/st.
2976 if (auto NextI = mergeUpdateInsn(I: MBBI, Update, /*IsForward=*/false,
2977 /*IsPreIdx=*/true,
2978 /*MergeEither=*/false)) {
2979 MBBI = *NextI;
2980 return true;
2981 }
2982 }
2983
2984 return false;
2985}
2986
2987bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
2988 int Scale) {
2989 MachineInstr &MI = *MBBI;
2990 MachineBasicBlock::iterator E = MI.getParent()->end();
2991 MachineBasicBlock::iterator Update;
2992
2993 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2994 if (TII->hasUnscaledLdStOffset(Opc: MI.getOpcode()))
2995 return false;
2996
2997 // Look back to try to find a const offset for index LdSt instruction. For
2998 // example,
2999 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3000 // ldr x1, [x0, x8]
3001 // merged into:
3002 // add x8, x0, a * (1<<12)
3003 // ldr x1, [x8, imm12]
3004 unsigned Offset;
3005 Update = findMatchingConstOffsetBackward(
3006 I: MBBI, Limit: Subtarget->getCLOpts().load_store_const_scan_limit, Offset);
3007 if (Update != E && (Offset & (Scale - 1)) == 0) {
3008 // Merge the imm12 into the ld/st.
3009 MBBI = mergeConstOffsetInsn(I: MBBI, Update, Offset, Scale);
3010 return true;
3011 }
3012
3013 return false;
3014}
3015
3016// Map a GPR store opcode to its FPR equivalent at the same data width.
3017// Returns 0 if no mapping exists.
3018static unsigned getGPRToFPRStoreOpcode(unsigned GPRStoreOpc) {
3019 switch (GPRStoreOpc) {
3020 // Unsigned immediate.
3021 case AArch64::STRBBui:
3022 return AArch64::STRBui;
3023 case AArch64::STRHHui:
3024 return AArch64::STRHui;
3025 case AArch64::STRWui:
3026 return AArch64::STRSui;
3027 case AArch64::STRXui:
3028 return AArch64::STRDui;
3029 // Unscaled immediate.
3030 case AArch64::STURBBi:
3031 return AArch64::STURBi;
3032 case AArch64::STURHHi:
3033 return AArch64::STURHi;
3034 case AArch64::STURWi:
3035 return AArch64::STURSi;
3036 case AArch64::STURXi:
3037 return AArch64::STURDi;
3038 // Register offset.
3039 case AArch64::STRBBroW:
3040 return AArch64::STRBroW;
3041 case AArch64::STRBBroX:
3042 return AArch64::STRBroX;
3043 case AArch64::STRHHroW:
3044 return AArch64::STRHroW;
3045 case AArch64::STRHHroX:
3046 return AArch64::STRHroX;
3047 case AArch64::STRWroW:
3048 return AArch64::STRSroW;
3049 case AArch64::STRWroX:
3050 return AArch64::STRSroX;
3051 case AArch64::STRXroW:
3052 return AArch64::STRDroW;
3053 case AArch64::STRXroX:
3054 return AArch64::STRDroX;
3055 default:
3056 return 0;
3057 }
3058}
3059
3060// Given a UMOV-lane-0 opcode, return the sub-register index to extract from
3061// the vector register, or 0 if the opcode is not a supported UMOV.
3062static unsigned getUMOVSubRegIdx(unsigned UMOVOpc) {
3063 switch (UMOVOpc) {
3064 case AArch64::UMOVvi8_idx0:
3065 return AArch64::bsub;
3066 case AArch64::UMOVvi16_idx0:
3067 return AArch64::hsub;
3068 case AArch64::UMOVvi32_idx0:
3069 return AArch64::ssub;
3070 case AArch64::UMOVvi64_idx0:
3071 return AArch64::dsub;
3072 default:
3073 return 0;
3074 }
3075}
3076
3077bool AArch64LoadStoreOpt::tryToReplaceUMOVStore(
3078 MachineBasicBlock::iterator &MBBI) {
3079 MachineInstr &StoreMI = *MBBI;
3080
3081 unsigned FPRStoreOpc = getGPRToFPRStoreOpcode(GPRStoreOpc: StoreMI.getOpcode());
3082 if (!FPRStoreOpc)
3083 return false;
3084
3085 if (StoreMI.hasOrderedMemoryRef() || StoreMI.memoperands().size() != 1)
3086 return false;
3087
3088 MachineBasicBlock *MBB = StoreMI.getParent();
3089 MCPhysReg StoreValReg = StoreMI.getOperand(i: 0).getReg();
3090
3091 if (!StoreMI.getOperand(i: 0).isKill())
3092 return false;
3093
3094 // Bail out if the store uses the value register elsewhere (e.g., as the base
3095 // address in `str w8, [x8, #0]`).
3096 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3097 if (StoreMI.getOperand(i: I).isReg() &&
3098 TRI->regsOverlap(RegA: StoreMI.getOperand(i: I).getReg(), RegB: StoreValReg))
3099 return false;
3100
3101 // Scan backward to find the UMOV that defines the store's value register.
3102 MachineInstr *UMOVMI = nullptr;
3103 MachineBasicBlock::iterator B = MBB->begin();
3104 unsigned SubRegIdx = 0;
3105 unsigned Count = 0;
3106 for (auto It = MBBI; It != B;) {
3107 MachineInstr &MI = *--It;
3108 if (MI.isDebugInstr())
3109 continue;
3110 if (++Count > Subtarget->getCLOpts().umov_fold_scan_limit)
3111 return false;
3112 if (MI.readsRegister(Reg: StoreValReg, TRI))
3113 return false;
3114 if (MI.modifiesRegister(Reg: StoreValReg, TRI)) {
3115 SubRegIdx = getUMOVSubRegIdx(UMOVOpc: MI.getOpcode());
3116 if (!SubRegIdx)
3117 return false;
3118 UMOVMI = &MI;
3119 break;
3120 }
3121 }
3122 if (!UMOVMI)
3123 return false;
3124 MCPhysReg VecReg = UMOVMI->getOperand(i: 1).getReg();
3125 MCPhysReg FPRReg = TRI->getSubReg(Reg: VecReg, Idx: SubRegIdx);
3126 if ((*StoreMI.memoperands_begin())->getSizeInBits() !=
3127 TRI->getRegSizeInBits(RC: *TRI->getMinimalPhysRegClass(Reg: FPRReg)))
3128 return false;
3129
3130 // Check that no instruction between UMOV and store clobbers the vector
3131 // register. Also track whether VecReg is killed anywhere from the UMOV
3132 // (inclusive) through the intervening instructions -- we need this to decide
3133 // whether the FPR sub-register can be marked killed on the new store.
3134 bool VecRegKilled = UMOVMI->killsRegister(Reg: VecReg, TRI);
3135 for (auto It = std::next(x: UMOVMI->getIterator()); It != MBBI; ++It) {
3136 if (It->modifiesRegister(Reg: VecReg, TRI))
3137 return false;
3138 if (!VecRegKilled && It->killsRegister(Reg: VecReg, TRI))
3139 VecRegKilled = true;
3140 }
3141
3142 // Safe to proceed. Clear kill flags on the vector register between UMOV and
3143 // the new store so the FPR sub-register stays live.
3144 UMOVMI->clearRegisterKills(Reg: VecReg, RegInfo: TRI);
3145 for (auto It = std::next(x: UMOVMI->getIterator()); It != MBBI; ++It)
3146 It->clearRegisterKills(Reg: VecReg, RegInfo: TRI);
3147
3148 LLVM_DEBUG(dbgs() << "Folding UMOV + store: " << *UMOVMI << " + "
3149 << StoreMI);
3150
3151 auto MIB = BuildMI(BB&: *MBB, I: MBBI, MIMD: StoreMI.getDebugLoc(), MCID: TII->get(Opcode: FPRStoreOpc))
3152 .addReg(RegNo: FPRReg, Flags: getKillRegState(B: VecRegKilled));
3153 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3154 MIB.add(MO: StoreMI.getOperand(i: I));
3155 MIB.setMemRefs(StoreMI.memoperands());
3156
3157 MBBI = MBB->erase(I: MBBI);
3158 UMOVMI->eraseFromParent();
3159
3160 ++NumUMOVFoldedToFPRStore;
3161 return true;
3162}
3163
3164bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
3165 bool EnableNarrowZeroStOpt) {
3166 AArch64FunctionInfo &AFI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
3167
3168 bool Modified = false;
3169 // Six transformations to do here:
3170 // 1) Find loads that directly read from stores and promote them by
3171 // replacing with mov instructions. If the store is wider than the load,
3172 // the load will be replaced with a bitfield extract.
3173 // e.g.,
3174 // str w1, [x0, #4]
3175 // ldrh w2, [x0, #6]
3176 // ; becomes
3177 // str w1, [x0, #4]
3178 // lsr w2, w1, #16
3179 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3180 MBBI != E;) {
3181 if (isPromotableLoadFromStore(MI&: *MBBI) && tryToPromoteLoadFromStore(MBBI))
3182 Modified = true;
3183 else
3184 ++MBBI;
3185 }
3186 // 2) Merge adjacent zero stores into a wider store.
3187 // e.g.,
3188 // strh wzr, [x0]
3189 // strh wzr, [x0, #2]
3190 // ; becomes
3191 // str wzr, [x0]
3192 // e.g.,
3193 // str wzr, [x0]
3194 // str wzr, [x0, #4]
3195 // ; becomes
3196 // str xzr, [x0]
3197 if (EnableNarrowZeroStOpt)
3198 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3199 MBBI != E;) {
3200 if (isPromotableZeroStoreInst(MI&: *MBBI) && tryToMergeZeroStInst(MBBI))
3201 Modified = true;
3202 else
3203 ++MBBI;
3204 }
3205 // 3) Find loads and stores that can be merged into a single load or store
3206 // pair instruction.
3207 // When compiling for SVE 128, also try to combine SVE fill/spill
3208 // instructions into LDP/STP.
3209 // e.g.,
3210 // ldr x0, [x2]
3211 // ldr x1, [x2, #8]
3212 // ; becomes
3213 // ldp x0, x1, [x2]
3214 // e.g.,
3215 // ldr z0, [x2]
3216 // ldr z1, [x2, #1, mul vl]
3217 // ; becomes
3218 // ldp q0, q1, [x2]
3219
3220 if (MBB.getParent()->getRegInfo().tracksLiveness()) {
3221 DefinedInBB.clear();
3222 DefinedInBB.addLiveIns(MBB);
3223 }
3224
3225 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3226 MBBI != E;) {
3227 // Track currently live registers up to this point, to help with
3228 // searching for a rename register on demand.
3229 updateDefinedRegisters(MI&: *MBBI, Units&: DefinedInBB, TRI);
3230 if (TII->isPairableLdStInst(MI: *MBBI) && tryToPairLdStInst(MBBI))
3231 Modified = true;
3232 else
3233 ++MBBI;
3234 }
3235 // 4) Find base register updates that can be merged into the load or store
3236 // as a base-reg writeback.
3237 // e.g.,
3238 // ldr x0, [x2]
3239 // add x2, x2, #4
3240 // ; becomes
3241 // ldr x0, [x2], #4
3242 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3243 MBBI != E;) {
3244 if (isMergeableLdStUpdate(MI&: *MBBI, AFI) && tryToMergeLdStUpdate(MBBI))
3245 Modified = true;
3246 else
3247 ++MBBI;
3248 }
3249
3250 // 5) Find a register assigned with a const value that can be combined with
3251 // into the load or store. e.g.,
3252 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3253 // ldr x1, [x0, x8]
3254 // ; becomes
3255 // add x8, x0, a * (1<<12)
3256 // ldr x1, [x8, imm12]
3257 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3258 MBBI != E;) {
3259 int Scale;
3260 if (isMergeableIndexLdSt(MI&: *MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale))
3261 Modified = true;
3262 else
3263 ++MBBI;
3264 }
3265
3266 // 6) Replace UMOV (lane 0) + GPR store with a direct FPR sub-register store.
3267 // e.g.,
3268 // umov w8, v0.h[0]
3269 // strh w8, [x0]
3270 // ; becomes
3271 // str h0, [x0]
3272 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3273 MBBI != E;) {
3274 if (tryToReplaceUMOVStore(MBBI))
3275 Modified = true;
3276 else
3277 ++MBBI;
3278 }
3279
3280 return Modified;
3281}
3282
3283bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
3284 Subtarget = &Fn.getSubtarget<AArch64Subtarget>();
3285 TII = Subtarget->getInstrInfo();
3286 TRI = Subtarget->getRegisterInfo();
3287
3288 // Resize the modified and used register unit trackers. We do this once
3289 // per function and then clear the register units each time we optimize a load
3290 // or store.
3291 ModifiedRegUnits.init(TRI: *TRI);
3292 UsedRegUnits.init(TRI: *TRI);
3293 DefinedInBB.init(TRI: *TRI);
3294
3295 bool Modified = false;
3296 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
3297 for (auto &MBB : Fn) {
3298 auto M = optimizeBlock(MBB, EnableNarrowZeroStOpt: enableNarrowZeroStOpt);
3299 Modified |= M;
3300 }
3301
3302 return Modified;
3303}
3304
3305// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and
3306// stores near one another? Note: The pre-RA instruction scheduler already has
3307// hooks to try and schedule pairable loads/stores together to improve pairing
3308// opportunities. Thus, pre-RA pairing pass may not be worth the effort.
3309
3310// FIXME: When pairing store instructions it's very possible for this pass to
3311// hoist a store with a KILL marker above another use (without a KILL marker).
3312// The resulting IR is invalid, but nothing uses the KILL markers after this
3313// pass, so it's never caused a problem in practice.
3314
3315bool AArch64LoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
3316 if (skipFunction(F: MF.getFunction()))
3317 return false;
3318 AArch64LoadStoreOpt Impl;
3319 Impl.AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3320 return Impl.runOnMachineFunction(Fn&: MF);
3321}
3322
3323/// createAArch64LoadStoreOptimizationPass - returns an instance of the
3324/// load / store optimization pass.
3325FunctionPass *llvm::createAArch64LoadStoreOptLegacyPass() {
3326 return new AArch64LoadStoreOptLegacy();
3327}
3328
3329PreservedAnalyses
3330AArch64LoadStoreOptPass::run(MachineFunction &MF,
3331 MachineFunctionAnalysisManager &MFAM) {
3332 AArch64LoadStoreOpt Impl;
3333 Impl.AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3334 .getManager()
3335 .getResult<AAManager>(IR&: MF.getFunction());
3336 bool Changed = Impl.runOnMachineFunction(Fn&: MF);
3337 if (!Changed)
3338 return PreservedAnalyses::all();
3339 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
3340 PA.preserveSet<CFGAnalyses>();
3341 return PA;
3342}
3343