1//===- SILoadStoreOptimizer.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass tries to fuse DS instructions with close by immediate offsets.
10// This will fuse operations such as
11// ds_read_b32 v0, v2 offset:16
12// ds_read_b32 v1, v2 offset:32
13// ==>
14// ds_read2_b32 v[0:1], v2, offset0:4 offset1:8
15//
16// The same is done for certain SMEM and VMEM opcodes, e.g.:
17// s_buffer_load_dword s4, s[0:3], 4
18// s_buffer_load_dword s5, s[0:3], 8
19// ==>
20// s_buffer_load_dwordx2 s[4:5], s[0:3], 4
21//
22// This pass also tries to promote constant offset to the immediate by
23// adjusting the base. It tries to use a base from the nearby instructions that
24// allows it to have a 13bit constant offset and then promotes the 13bit offset
25// to the immediate.
26// E.g.
27// s_movk_i32 s0, 0x1800
28// v_add_co_u32_e32 v0, vcc, s0, v2
29// v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
30//
31// s_movk_i32 s0, 0x1000
32// v_add_co_u32_e32 v5, vcc, s0, v2
33// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
34// global_load_dwordx2 v[5:6], v[5:6], off
35// global_load_dwordx2 v[0:1], v[0:1], off
36// =>
37// s_movk_i32 s0, 0x1000
38// v_add_co_u32_e32 v5, vcc, s0, v2
39// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
40// global_load_dwordx2 v[5:6], v[5:6], off
41// global_load_dwordx2 v[0:1], v[5:6], off offset:2048
42//
43// Future improvements:
44//
45// - This is currently missing stores of constants because loading
46// the constant into the data register is placed between the stores, although
47// this is arguably a scheduling problem.
48//
49// - Live interval recomputing seems inefficient. This currently only matches
50// one pair, and recomputes live intervals and moves on to the next pair. It
51// would be better to compute a list of all merges that need to occur.
52//
53// - With a list of instructions to process, we can also merge more. If a
54// cluster of loads have offsets that are too large to fit in the 8-bit
55// offsets, but are close enough to fit in the 8 bits, we can add to the base
56// pointer and use the new reduced offsets.
57//
58//===----------------------------------------------------------------------===//
59
60#include "SILoadStoreOptimizer.h"
61#include "AMDGPU.h"
62#include "GCNSubtarget.h"
63#include "SIDefines.h"
64#include "llvm/Analysis/AliasAnalysis.h"
65#include "llvm/CodeGen/MachineFunctionPass.h"
66#include "llvm/InitializePasses.h"
67
68using namespace llvm;
69
70#define DEBUG_TYPE "si-load-store-opt"
71
72namespace {
73enum InstClassEnum {
74 UNKNOWN,
75 DS_READ,
76 DS_WRITE,
77 S_BUFFER_LOAD_IMM,
78 S_BUFFER_LOAD_SGPR_IMM,
79 S_LOAD_IMM,
80 BUFFER_LOAD,
81 BUFFER_STORE,
82 MIMG,
83 TBUFFER_LOAD,
84 TBUFFER_STORE,
85 GLOBAL_LOAD_SADDR,
86 GLOBAL_STORE_SADDR,
87 FLAT_LOAD,
88 FLAT_STORE,
89 FLAT_LOAD_SADDR,
90 FLAT_STORE_SADDR,
91 GLOBAL_LOAD, // GLOBAL_LOAD/GLOBAL_STORE are never used as the InstClass of
92 GLOBAL_STORE // any CombineInfo, they are only ever returned by
93 // getCommonInstClass.
94};
95
96struct AddressRegs {
97 unsigned char NumVAddrs = 0;
98 bool SBase = false;
99 bool SRsrc = false;
100 bool SOffset = false;
101 bool SAddr = false;
102 bool VAddr = false;
103 bool Addr = false;
104 bool SSamp = false;
105};
106
107// GFX10 image_sample instructions can have 12 vaddrs + srsrc + ssamp.
108const unsigned MaxAddressRegs = 12 + 1 + 1;
109
110class SILoadStoreOptimizer {
111 struct CombineInfo {
112 MachineBasicBlock::iterator I;
113 unsigned EltSize;
114 unsigned Offset;
115 unsigned Width;
116 unsigned Format;
117 unsigned BaseOff;
118 unsigned DMask;
119 InstClassEnum InstClass;
120 unsigned CPol = 0;
121 const TargetRegisterClass *DataRC;
122 bool UseST64;
123 int AddrIdx[MaxAddressRegs];
124 const MachineOperand *AddrReg[MaxAddressRegs];
125 unsigned NumAddresses;
126 unsigned Order;
127
128 bool hasSameBaseAddress(const CombineInfo &CI) {
129 if (NumAddresses != CI.NumAddresses)
130 return false;
131
132 const MachineInstr &MI = *CI.I;
133 for (unsigned i = 0; i < NumAddresses; i++) {
134 const MachineOperand &AddrRegNext = MI.getOperand(i: AddrIdx[i]);
135
136 if (AddrReg[i]->isImm() || AddrRegNext.isImm()) {
137 if (AddrReg[i]->isImm() != AddrRegNext.isImm() ||
138 AddrReg[i]->getImm() != AddrRegNext.getImm()) {
139 return false;
140 }
141 continue;
142 }
143
144 // Check same base pointer. Be careful of subregisters, which can occur
145 // with vectors of pointers.
146 if (AddrReg[i]->getReg() != AddrRegNext.getReg() ||
147 AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) {
148 return false;
149 }
150 }
151 return true;
152 }
153
154 bool hasMergeableAddress(const MachineRegisterInfo &MRI) {
155 for (unsigned i = 0; i < NumAddresses; ++i) {
156 const MachineOperand *AddrOp = AddrReg[i];
157 // Immediates are always OK.
158 if (AddrOp->isImm())
159 continue;
160
161 // Don't try to merge addresses that aren't either immediates or registers.
162 // TODO: Should be possible to merge FrameIndexes and maybe some other
163 // non-register
164 if (!AddrOp->isReg())
165 return false;
166
167 // TODO: We should be able to merge instructions with other physical reg
168 // addresses too.
169 if (AddrOp->getReg().isPhysical() &&
170 AddrOp->getReg() != AMDGPU::SGPR_NULL)
171 return false;
172
173 // If an address has only one use then there will be no other
174 // instructions with the same address, so we can't merge this one.
175 if (MRI.hasOneNonDBGUse(RegNo: AddrOp->getReg()))
176 return false;
177 }
178 return true;
179 }
180
181 void setMI(MachineBasicBlock::iterator MI, const SILoadStoreOptimizer &LSO);
182
183 // Compare by pointer order.
184 bool operator<(const CombineInfo& Other) const {
185 return (InstClass == MIMG) ? DMask < Other.DMask : Offset < Other.Offset;
186 }
187 };
188
189 struct BaseRegisters {
190 Register LoReg;
191 Register HiReg;
192
193 unsigned LoSubReg = 0;
194 unsigned HiSubReg = 0;
195 // True when using V_ADD_U64_e64 pattern
196 bool UseV64Pattern = false;
197 };
198
199 struct MemAddress {
200 BaseRegisters Base;
201 int64_t Offset = 0;
202 };
203
204 using MemInfoMap = DenseMap<MachineInstr *, MemAddress>;
205
206private:
207 MachineFunction *MF = nullptr;
208 const GCNSubtarget *STM = nullptr;
209 const SIInstrInfo *TII = nullptr;
210 const SIRegisterInfo *TRI = nullptr;
211 MachineRegisterInfo *MRI = nullptr;
212 AliasAnalysis *AA = nullptr;
213 bool OptimizeAgain;
214
215 bool canSwapInstructions(const DenseSet<Register> &ARegDefs,
216 const DenseSet<Register> &ARegUses,
217 const MachineInstr &A, const MachineInstr &B) const;
218 static bool dmasksCanBeCombined(const CombineInfo &CI,
219 const SIInstrInfo &TII,
220 const CombineInfo &Paired);
221 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
222 CombineInfo &Paired, bool Modify = false);
223 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
224 const CombineInfo &Paired);
225 unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
226 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
227 const CombineInfo &Paired);
228 const TargetRegisterClass *
229 getTargetRegisterClass(const CombineInfo &CI,
230 const CombineInfo &Paired) const;
231 const TargetRegisterClass *getDataRegClass(const MachineInstr &MI) const;
232
233 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
234
235 void copyToDestRegs(CombineInfo &CI, CombineInfo &Paired,
236 MachineBasicBlock::iterator InsertBefore,
237 const DebugLoc &DL, AMDGPU::OpName OpName,
238 Register DestReg) const;
239 Register copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
240 MachineBasicBlock::iterator InsertBefore,
241 const DebugLoc &DL, AMDGPU::OpName OpName) const;
242
243 unsigned read2Opcode(unsigned EltSize) const;
244 unsigned read2ST64Opcode(unsigned EltSize) const;
245 MachineBasicBlock::iterator
246 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
247 MachineBasicBlock::iterator InsertBefore);
248
249 unsigned write2Opcode(unsigned EltSize) const;
250 unsigned write2ST64Opcode(unsigned EltSize) const;
251 unsigned getWrite2Opcode(const CombineInfo &CI) const;
252
253 MachineBasicBlock::iterator
254 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
255 MachineBasicBlock::iterator InsertBefore);
256 MachineBasicBlock::iterator
257 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
258 MachineBasicBlock::iterator InsertBefore);
259 MachineBasicBlock::iterator
260 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
261 MachineBasicBlock::iterator InsertBefore);
262 MachineBasicBlock::iterator
263 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
264 MachineBasicBlock::iterator InsertBefore);
265 MachineBasicBlock::iterator
266 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
267 MachineBasicBlock::iterator InsertBefore);
268 MachineBasicBlock::iterator
269 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
270 MachineBasicBlock::iterator InsertBefore);
271 MachineBasicBlock::iterator
272 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
273 MachineBasicBlock::iterator InsertBefore);
274 MachineBasicBlock::iterator
275 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
276 MachineBasicBlock::iterator InsertBefore);
277 MachineBasicBlock::iterator
278 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
279 MachineBasicBlock::iterator InsertBefore);
280
281 void updateBaseAndOffset(MachineInstr &I, Register NewBase,
282 int32_t NewOffset) const;
283 void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
284 Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
285 MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
286 bool processBaseWithConstOffset64(MachineInstr *AddDef,
287 const MachineOperand &Base,
288 MemAddress &Addr) const;
289 void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
290 /// Promotes constant offset to the immediate by adjusting the base. It
291 /// tries to use a base from the nearby instructions that allows it to have
292 /// a 13bit constant offset which gets promoted to the immediate.
293 bool promoteConstantOffsetToImm(MachineInstr &CI,
294 MemInfoMap &Visited,
295 SmallPtrSet<MachineInstr *, 4> &Promoted) const;
296 void addInstToMergeableList(const CombineInfo &CI,
297 std::list<std::list<CombineInfo> > &MergeableInsts) const;
298
299 std::pair<MachineBasicBlock::iterator, bool> collectMergeableInsts(
300 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
301 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
302 std::list<std::list<CombineInfo>> &MergeableInsts) const;
303
304 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
305 const CombineInfo &Paired);
306
307 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
308 const CombineInfo &Paired);
309
310 bool optimizeInstsWithSameBaseAddr(std::list<CombineInfo> &MergeList,
311 bool &OptimizeListAgain);
312 bool optimizeBlock(std::list<std::list<CombineInfo> > &MergeableInsts);
313
314public:
315 SILoadStoreOptimizer(AliasAnalysis *AA) : AA(AA) {}
316 bool run(MachineFunction &MF);
317};
318
319class SILoadStoreOptimizerLegacy : public MachineFunctionPass {
320public:
321 static char ID;
322
323 SILoadStoreOptimizerLegacy() : MachineFunctionPass(ID) {}
324
325 bool runOnMachineFunction(MachineFunction &MF) override;
326
327 StringRef getPassName() const override { return "SI Load Store Optimizer"; }
328
329 void getAnalysisUsage(AnalysisUsage &AU) const override {
330 AU.setPreservesCFG();
331 AU.addRequired<AAResultsWrapperPass>();
332
333 MachineFunctionPass::getAnalysisUsage(AU);
334 }
335
336 MachineFunctionProperties getRequiredProperties() const override {
337 return MachineFunctionProperties().setIsSSA();
338 }
339};
340
341static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
342 const unsigned Opc = MI.getOpcode();
343
344 if (TII.isMUBUF(Opcode: Opc)) {
345 // FIXME: Handle d16 correctly
346 return AMDGPU::getMUBUFElements(Opc);
347 }
348 if (TII.isImage(MI)) {
349 uint64_t DMaskImm =
350 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask)->getImm();
351 return llvm::popcount(Value: DMaskImm);
352 }
353 if (TII.isMTBUF(Opcode: Opc)) {
354 return AMDGPU::getMTBUFElements(Opc);
355 }
356
357 switch (Opc) {
358 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
359 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
360 case AMDGPU::S_LOAD_DWORD_IMM:
361 case AMDGPU::GLOBAL_LOAD_DWORD:
362 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
363 case AMDGPU::GLOBAL_STORE_DWORD:
364 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
365 case AMDGPU::FLAT_LOAD_DWORD:
366 case AMDGPU::FLAT_STORE_DWORD:
367 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
368 case AMDGPU::FLAT_STORE_DWORD_SADDR:
369 return 1;
370 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
371 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
372 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
373 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
374 case AMDGPU::S_LOAD_DWORDX2_IMM:
375 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
376 case AMDGPU::GLOBAL_LOAD_DWORDX2:
377 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
378 case AMDGPU::GLOBAL_STORE_DWORDX2:
379 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
380 case AMDGPU::FLAT_LOAD_DWORDX2:
381 case AMDGPU::FLAT_STORE_DWORDX2:
382 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
383 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
384 return 2;
385 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
386 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
387 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
388 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
389 case AMDGPU::S_LOAD_DWORDX3_IMM:
390 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
391 case AMDGPU::GLOBAL_LOAD_DWORDX3:
392 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
393 case AMDGPU::GLOBAL_STORE_DWORDX3:
394 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
395 case AMDGPU::FLAT_LOAD_DWORDX3:
396 case AMDGPU::FLAT_STORE_DWORDX3:
397 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
398 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
399 return 3;
400 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
401 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
402 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
403 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
404 case AMDGPU::S_LOAD_DWORDX4_IMM:
405 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
406 case AMDGPU::GLOBAL_LOAD_DWORDX4:
407 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
408 case AMDGPU::GLOBAL_STORE_DWORDX4:
409 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
410 case AMDGPU::FLAT_LOAD_DWORDX4:
411 case AMDGPU::FLAT_STORE_DWORDX4:
412 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
413 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
414 return 4;
415 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
416 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
417 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
418 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
419 case AMDGPU::S_LOAD_DWORDX8_IMM:
420 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
421 return 8;
422 case AMDGPU::DS_READ_B32:
423 case AMDGPU::DS_READ_B32_gfx9:
424 case AMDGPU::DS_WRITE_B32:
425 case AMDGPU::DS_WRITE_B32_gfx9:
426 return 1;
427 case AMDGPU::DS_READ_B64:
428 case AMDGPU::DS_READ_B64_gfx9:
429 case AMDGPU::DS_WRITE_B64:
430 case AMDGPU::DS_WRITE_B64_gfx9:
431 return 2;
432 default:
433 return 0;
434 }
435}
436
437/// Maps instruction opcode to enum InstClassEnum.
438static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
439 switch (Opc) {
440 default:
441 if (TII.isMUBUF(Opcode: Opc)) {
442 switch (AMDGPU::getMUBUFBaseOpcode(Opc)) {
443 default:
444 return UNKNOWN;
445 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN:
446 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN_exact:
447 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN:
448 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN_exact:
449 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
450 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
451 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
452 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
453 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:
454 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact:
455 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN:
456 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact:
457 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN:
458 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN_exact:
459 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET:
460 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET_exact:
461 return BUFFER_LOAD;
462 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN:
463 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN_exact:
464 case AMDGPU::BUFFER_STORE_DWORD_IDXEN:
465 case AMDGPU::BUFFER_STORE_DWORD_IDXEN_exact:
466 case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
467 case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
468 case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
469 case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
470 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN:
471 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact:
472 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN:
473 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN_exact:
474 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN:
475 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact:
476 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET:
477 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact:
478 return BUFFER_STORE;
479 }
480 }
481 if (TII.isImage(Opcode: Opc)) {
482 // Ignore instructions encoded without vaddr.
483 if (!AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
484 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr0))
485 return UNKNOWN;
486 // Ignore BVH instructions
487 if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH)
488 return UNKNOWN;
489 // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD.
490 if (TII.get(Opcode: Opc).mayStore() || !TII.get(Opcode: Opc).mayLoad() ||
491 TII.isGather4(Opcode: Opc))
492 return UNKNOWN;
493 return MIMG;
494 }
495 if (TII.isMTBUF(Opcode: Opc)) {
496 switch (AMDGPU::getMTBUFBaseOpcode(Opc)) {
497 default:
498 return UNKNOWN;
499 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN:
500 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN_exact:
501 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN:
502 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN_exact:
503 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN:
504 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN_exact:
505 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET:
506 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET_exact:
507 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN:
508 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN_exact:
509 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN:
510 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN_exact:
511 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN:
512 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN_exact:
513 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET:
514 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET_exact:
515 return TBUFFER_LOAD;
516 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN:
517 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN_exact:
518 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET:
519 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET_exact:
520 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN:
521 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact:
522 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET:
523 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET_exact:
524 return TBUFFER_STORE;
525 }
526 }
527 return UNKNOWN;
528 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
529 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
530 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
531 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
532 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
533 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
534 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
535 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
536 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
537 return S_BUFFER_LOAD_IMM;
538 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
539 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
540 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
541 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
542 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
543 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
544 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
545 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
546 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
547 return S_BUFFER_LOAD_SGPR_IMM;
548 case AMDGPU::S_LOAD_DWORD_IMM:
549 case AMDGPU::S_LOAD_DWORDX2_IMM:
550 case AMDGPU::S_LOAD_DWORDX3_IMM:
551 case AMDGPU::S_LOAD_DWORDX4_IMM:
552 case AMDGPU::S_LOAD_DWORDX8_IMM:
553 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
554 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
555 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
556 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
557 return S_LOAD_IMM;
558 case AMDGPU::DS_READ_B32:
559 case AMDGPU::DS_READ_B32_gfx9:
560 case AMDGPU::DS_READ_B64:
561 case AMDGPU::DS_READ_B64_gfx9:
562 return DS_READ;
563 case AMDGPU::DS_WRITE_B32:
564 case AMDGPU::DS_WRITE_B32_gfx9:
565 case AMDGPU::DS_WRITE_B64:
566 case AMDGPU::DS_WRITE_B64_gfx9:
567 return DS_WRITE;
568 case AMDGPU::GLOBAL_LOAD_DWORD:
569 case AMDGPU::GLOBAL_LOAD_DWORDX2:
570 case AMDGPU::GLOBAL_LOAD_DWORDX3:
571 case AMDGPU::GLOBAL_LOAD_DWORDX4:
572 case AMDGPU::FLAT_LOAD_DWORD:
573 case AMDGPU::FLAT_LOAD_DWORDX2:
574 case AMDGPU::FLAT_LOAD_DWORDX3:
575 case AMDGPU::FLAT_LOAD_DWORDX4:
576 return FLAT_LOAD;
577 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
578 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
579 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
580 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
581 return GLOBAL_LOAD_SADDR;
582 case AMDGPU::GLOBAL_STORE_DWORD:
583 case AMDGPU::GLOBAL_STORE_DWORDX2:
584 case AMDGPU::GLOBAL_STORE_DWORDX3:
585 case AMDGPU::GLOBAL_STORE_DWORDX4:
586 case AMDGPU::FLAT_STORE_DWORD:
587 case AMDGPU::FLAT_STORE_DWORDX2:
588 case AMDGPU::FLAT_STORE_DWORDX3:
589 case AMDGPU::FLAT_STORE_DWORDX4:
590 return FLAT_STORE;
591 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
592 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
593 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
594 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
595 return GLOBAL_STORE_SADDR;
596 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
597 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
598 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
599 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
600 return FLAT_LOAD_SADDR;
601 case AMDGPU::FLAT_STORE_DWORD_SADDR:
602 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
603 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
604 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
605 return FLAT_STORE_SADDR;
606 }
607}
608
609/// Determines instruction subclass from opcode. Only instructions
610/// of the same subclass can be merged together. The merged instruction may have
611/// a different subclass but must have the same class.
612static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
613 switch (Opc) {
614 default:
615 if (TII.isMUBUF(Opcode: Opc))
616 return AMDGPU::getMUBUFBaseOpcode(Opc);
617 if (TII.isImage(Opcode: Opc)) {
618 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
619 assert(Info);
620 return Info->BaseOpcode;
621 }
622 if (TII.isMTBUF(Opcode: Opc))
623 return AMDGPU::getMTBUFBaseOpcode(Opc);
624 return -1;
625 case AMDGPU::DS_READ_B32:
626 case AMDGPU::DS_READ_B32_gfx9:
627 case AMDGPU::DS_READ_B64:
628 case AMDGPU::DS_READ_B64_gfx9:
629 case AMDGPU::DS_WRITE_B32:
630 case AMDGPU::DS_WRITE_B32_gfx9:
631 case AMDGPU::DS_WRITE_B64:
632 case AMDGPU::DS_WRITE_B64_gfx9:
633 return Opc;
634 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
635 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
636 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
637 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
638 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
639 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
640 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
641 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
642 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
643 return AMDGPU::S_BUFFER_LOAD_DWORD_IMM;
644 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
645 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
646 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
647 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
648 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
649 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
650 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
651 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
652 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
653 return AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM;
654 case AMDGPU::S_LOAD_DWORD_IMM:
655 case AMDGPU::S_LOAD_DWORDX2_IMM:
656 case AMDGPU::S_LOAD_DWORDX3_IMM:
657 case AMDGPU::S_LOAD_DWORDX4_IMM:
658 case AMDGPU::S_LOAD_DWORDX8_IMM:
659 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
660 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
661 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
662 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
663 return AMDGPU::S_LOAD_DWORD_IMM;
664 case AMDGPU::GLOBAL_LOAD_DWORD:
665 case AMDGPU::GLOBAL_LOAD_DWORDX2:
666 case AMDGPU::GLOBAL_LOAD_DWORDX3:
667 case AMDGPU::GLOBAL_LOAD_DWORDX4:
668 case AMDGPU::FLAT_LOAD_DWORD:
669 case AMDGPU::FLAT_LOAD_DWORDX2:
670 case AMDGPU::FLAT_LOAD_DWORDX3:
671 case AMDGPU::FLAT_LOAD_DWORDX4:
672 return AMDGPU::FLAT_LOAD_DWORD;
673 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
674 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
675 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
676 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
677 return AMDGPU::GLOBAL_LOAD_DWORD_SADDR;
678 case AMDGPU::GLOBAL_STORE_DWORD:
679 case AMDGPU::GLOBAL_STORE_DWORDX2:
680 case AMDGPU::GLOBAL_STORE_DWORDX3:
681 case AMDGPU::GLOBAL_STORE_DWORDX4:
682 case AMDGPU::FLAT_STORE_DWORD:
683 case AMDGPU::FLAT_STORE_DWORDX2:
684 case AMDGPU::FLAT_STORE_DWORDX3:
685 case AMDGPU::FLAT_STORE_DWORDX4:
686 return AMDGPU::FLAT_STORE_DWORD;
687 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
688 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
689 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
690 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
691 return AMDGPU::GLOBAL_STORE_DWORD_SADDR;
692 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
693 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
694 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
695 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
696 return AMDGPU::FLAT_LOAD_DWORD_SADDR;
697 case AMDGPU::FLAT_STORE_DWORD_SADDR:
698 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
699 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
700 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
701 return AMDGPU::FLAT_STORE_DWORD_SADDR;
702 }
703}
704
705// GLOBAL loads and stores are classified as FLAT initially. If both combined
706// instructions are FLAT GLOBAL adjust the class to GLOBAL_LOAD or GLOBAL_STORE.
707// If either or both instructions are non segment specific FLAT the resulting
708// combined operation will be FLAT, potentially promoting one of the GLOBAL
709// operations to FLAT.
710// For other instructions return the original unmodified class.
711InstClassEnum
712SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI,
713 const CombineInfo &Paired) {
714 assert(CI.InstClass == Paired.InstClass);
715
716 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) &&
717 SIInstrInfo::isFLATGlobal(MI: *CI.I) && SIInstrInfo::isFLATGlobal(MI: *Paired.I))
718 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD;
719
720 return CI.InstClass;
721}
722
723static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
724 AddressRegs Result;
725
726 if (TII.isMUBUF(Opcode: Opc)) {
727 if (AMDGPU::getMUBUFHasVAddr(Opc))
728 Result.VAddr = true;
729 if (AMDGPU::getMUBUFHasSrsrc(Opc))
730 Result.SRsrc = true;
731 if (AMDGPU::getMUBUFHasSoffset(Opc))
732 Result.SOffset = true;
733
734 return Result;
735 }
736
737 if (TII.isImage(Opcode: Opc)) {
738 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
739 if (VAddr0Idx >= 0) {
740 AMDGPU::OpName RsrcName =
741 TII.isMIMG(Opcode: Opc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
742 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcName);
743 Result.NumVAddrs = RsrcIdx - VAddr0Idx;
744 } else {
745 Result.VAddr = true;
746 }
747 Result.SRsrc = true;
748 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
749 if (Info && AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode)->Sampler)
750 Result.SSamp = true;
751
752 return Result;
753 }
754 if (TII.isMTBUF(Opcode: Opc)) {
755 if (AMDGPU::getMTBUFHasVAddr(Opc))
756 Result.VAddr = true;
757 if (AMDGPU::getMTBUFHasSrsrc(Opc))
758 Result.SRsrc = true;
759 if (AMDGPU::getMTBUFHasSoffset(Opc))
760 Result.SOffset = true;
761
762 return Result;
763 }
764
765 switch (Opc) {
766 default:
767 return Result;
768 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
769 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
770 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
771 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
772 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
773 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
774 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
775 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
776 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
777 Result.SOffset = true;
778 [[fallthrough]];
779 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
780 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
781 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
782 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
783 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
784 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
785 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
786 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
787 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
788 case AMDGPU::S_LOAD_DWORD_IMM:
789 case AMDGPU::S_LOAD_DWORDX2_IMM:
790 case AMDGPU::S_LOAD_DWORDX3_IMM:
791 case AMDGPU::S_LOAD_DWORDX4_IMM:
792 case AMDGPU::S_LOAD_DWORDX8_IMM:
793 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
794 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
795 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
796 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
797 Result.SBase = true;
798 return Result;
799 case AMDGPU::DS_READ_B32:
800 case AMDGPU::DS_READ_B64:
801 case AMDGPU::DS_READ_B32_gfx9:
802 case AMDGPU::DS_READ_B64_gfx9:
803 case AMDGPU::DS_WRITE_B32:
804 case AMDGPU::DS_WRITE_B64:
805 case AMDGPU::DS_WRITE_B32_gfx9:
806 case AMDGPU::DS_WRITE_B64_gfx9:
807 Result.Addr = true;
808 return Result;
809 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
810 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
811 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
812 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
813 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
814 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
815 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
816 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
817 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
818 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
819 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
820 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
821 case AMDGPU::FLAT_STORE_DWORD_SADDR:
822 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
823 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
824 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
825 Result.SAddr = true;
826 [[fallthrough]];
827 case AMDGPU::GLOBAL_LOAD_DWORD:
828 case AMDGPU::GLOBAL_LOAD_DWORDX2:
829 case AMDGPU::GLOBAL_LOAD_DWORDX3:
830 case AMDGPU::GLOBAL_LOAD_DWORDX4:
831 case AMDGPU::GLOBAL_STORE_DWORD:
832 case AMDGPU::GLOBAL_STORE_DWORDX2:
833 case AMDGPU::GLOBAL_STORE_DWORDX3:
834 case AMDGPU::GLOBAL_STORE_DWORDX4:
835 case AMDGPU::FLAT_LOAD_DWORD:
836 case AMDGPU::FLAT_LOAD_DWORDX2:
837 case AMDGPU::FLAT_LOAD_DWORDX3:
838 case AMDGPU::FLAT_LOAD_DWORDX4:
839 case AMDGPU::FLAT_STORE_DWORD:
840 case AMDGPU::FLAT_STORE_DWORDX2:
841 case AMDGPU::FLAT_STORE_DWORDX3:
842 case AMDGPU::FLAT_STORE_DWORDX4:
843 Result.VAddr = true;
844 return Result;
845 }
846}
847
848void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
849 const SILoadStoreOptimizer &LSO) {
850 I = MI;
851 unsigned Opc = MI->getOpcode();
852 InstClass = getInstClass(Opc, TII: *LSO.TII);
853
854 if (InstClass == UNKNOWN)
855 return;
856
857 DataRC = LSO.getDataRegClass(MI: *MI);
858
859 switch (InstClass) {
860 case DS_READ:
861 EltSize =
862 (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8
863 : 4;
864 break;
865 case DS_WRITE:
866 EltSize =
867 (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8
868 : 4;
869 break;
870 case S_BUFFER_LOAD_IMM:
871 case S_BUFFER_LOAD_SGPR_IMM:
872 case S_LOAD_IMM:
873 EltSize = AMDGPU::convertSMRDOffsetUnits(ST: *LSO.STM, ByteOffset: 4);
874 break;
875 default:
876 EltSize = 4;
877 break;
878 }
879
880 if (InstClass == MIMG) {
881 DMask = LSO.TII->getNamedOperand(MI&: *I, OperandName: AMDGPU::OpName::dmask)->getImm();
882 // Offset is not considered for MIMG instructions.
883 Offset = 0;
884 } else {
885 int OffsetIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::offset);
886 Offset = I->getOperand(i: OffsetIdx).getImm();
887 }
888
889 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
890 Format = LSO.TII->getNamedOperand(MI&: *I, OperandName: AMDGPU::OpName::format)->getImm();
891 const AMDGPU::GcnBufferFormatInfo *Info =
892 AMDGPU::getGcnBufferFormatInfo(Format, STI: *LSO.STM);
893 EltSize = Info->BitsPerComp / 8;
894 }
895
896 Width = getOpcodeWidth(MI: *I, TII: *LSO.TII);
897
898 if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
899 Offset &= 0xffff;
900 } else if (InstClass != MIMG) {
901 CPol = LSO.TII->getNamedOperand(MI&: *I, OperandName: AMDGPU::OpName::cpol)->getImm();
902 }
903
904 AddressRegs Regs = getRegs(Opc, TII: *LSO.TII);
905 bool isVIMAGEorVSAMPLE = LSO.TII->isVIMAGE(MI: *I) || LSO.TII->isVSAMPLE(MI: *I);
906
907 NumAddresses = 0;
908 for (unsigned J = 0; J < Regs.NumVAddrs; J++)
909 AddrIdx[NumAddresses++] =
910 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0) + J;
911 if (Regs.Addr)
912 AddrIdx[NumAddresses++] =
913 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::addr);
914 if (Regs.SBase)
915 AddrIdx[NumAddresses++] =
916 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sbase);
917 if (Regs.SRsrc)
918 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
919 Opcode: Opc, Name: isVIMAGEorVSAMPLE ? AMDGPU::OpName::rsrc : AMDGPU::OpName::srsrc);
920 if (Regs.SOffset)
921 AddrIdx[NumAddresses++] =
922 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::soffset);
923 if (Regs.SAddr)
924 AddrIdx[NumAddresses++] =
925 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
926 if (Regs.VAddr)
927 AddrIdx[NumAddresses++] =
928 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
929 if (Regs.SSamp)
930 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
931 Opcode: Opc, Name: isVIMAGEorVSAMPLE ? AMDGPU::OpName::samp : AMDGPU::OpName::ssamp);
932 assert(NumAddresses <= MaxAddressRegs);
933
934 for (unsigned J = 0; J < NumAddresses; J++)
935 AddrReg[J] = &I->getOperand(i: AddrIdx[J]);
936}
937
938} // end anonymous namespace.
939
940INITIALIZE_PASS_BEGIN(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
941 "SI Load Store Optimizer", false, false)
942INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
943INITIALIZE_PASS_END(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
944 "SI Load Store Optimizer", false, false)
945
946char SILoadStoreOptimizerLegacy::ID = 0;
947
948char &llvm::SILoadStoreOptimizerLegacyID = SILoadStoreOptimizerLegacy::ID;
949
950FunctionPass *llvm::createSILoadStoreOptimizerLegacyPass() {
951 return new SILoadStoreOptimizerLegacy();
952}
953
954static void addDefsUsesToList(const MachineInstr &MI,
955 DenseSet<Register> &RegDefs,
956 DenseSet<Register> &RegUses) {
957 for (const auto &Op : MI.operands()) {
958 if (!Op.isReg())
959 continue;
960 if (Op.isDef())
961 RegDefs.insert(V: Op.getReg());
962 if (Op.readsReg())
963 RegUses.insert(V: Op.getReg());
964 }
965}
966
967bool SILoadStoreOptimizer::canSwapInstructions(
968 const DenseSet<Register> &ARegDefs, const DenseSet<Register> &ARegUses,
969 const MachineInstr &A, const MachineInstr &B) const {
970 if (A.mayLoadOrStore() && B.mayLoadOrStore() &&
971 (A.mayStore() || B.mayStore()) && A.mayAlias(AA, Other: B, UseTBAA: true))
972 return false;
973 for (const auto &BOp : B.operands()) {
974 if (!BOp.isReg())
975 continue;
976 if ((BOp.isDef() || BOp.readsReg()) && ARegDefs.contains(V: BOp.getReg()))
977 return false;
978 if (BOp.isDef() && ARegUses.contains(V: BOp.getReg()))
979 return false;
980 }
981 return true;
982}
983
984// Given that \p CI and \p Paired are adjacent memory operations produce a new
985// MMO for the combined operation with a new access size.
986MachineMemOperand *
987SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI,
988 const CombineInfo &Paired) {
989 const MachineMemOperand *MMOa = *CI.I->memoperands_begin();
990 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin();
991
992 unsigned Size = MMOa->getSize().getValue() + MMOb->getSize().getValue();
993
994 // A base pointer for the combined operation is the same as the leading
995 // operation's pointer.
996 if (Paired < CI)
997 std::swap(a&: MMOa, b&: MMOb);
998
999 MachinePointerInfo PtrInfo(MMOa->getPointerInfo());
1000 // If merging FLAT and GLOBAL set address space to FLAT.
1001 if (MMOb->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
1002 PtrInfo.AddrSpace = AMDGPUAS::FLAT_ADDRESS;
1003
1004 MachineFunction *MF = CI.I->getMF();
1005 return MF->getMachineMemOperand(MMO: MMOa, PtrInfo, Size);
1006}
1007
1008bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI,
1009 const SIInstrInfo &TII,
1010 const CombineInfo &Paired) {
1011 assert(CI.InstClass == MIMG);
1012
1013 // Ignore instructions with tfe/lwe set.
1014 const auto *TFEOp = TII.getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::tfe);
1015 const auto *LWEOp = TII.getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::lwe);
1016
1017 if ((TFEOp && TFEOp->getImm()) || (LWEOp && LWEOp->getImm()))
1018 return false;
1019
1020 // Check other optional immediate operands for equality.
1021 AMDGPU::OpName OperandsToMatch[] = {
1022 AMDGPU::OpName::cpol, AMDGPU::OpName::d16, AMDGPU::OpName::unorm,
1023 AMDGPU::OpName::da, AMDGPU::OpName::r128, AMDGPU::OpName::a16,
1024 AMDGPU::OpName::dim};
1025
1026 for (AMDGPU::OpName op : OperandsToMatch) {
1027 int Idx = AMDGPU::getNamedOperandIdx(Opcode: CI.I->getOpcode(), Name: op);
1028 if (AMDGPU::getNamedOperandIdx(Opcode: Paired.I->getOpcode(), Name: op) != Idx)
1029 return false;
1030 if (Idx != -1 &&
1031 CI.I->getOperand(i: Idx).getImm() != Paired.I->getOperand(i: Idx).getImm())
1032 return false;
1033 }
1034
1035 // Check DMask for overlaps.
1036 unsigned MaxMask = std::max(a: CI.DMask, b: Paired.DMask);
1037 unsigned MinMask = std::min(a: CI.DMask, b: Paired.DMask);
1038
1039 if (!MaxMask)
1040 return false;
1041
1042 unsigned AllowedBitsForMin = llvm::countr_zero(Val: MaxMask);
1043 if ((1u << AllowedBitsForMin) <= MinMask)
1044 return false;
1045
1046 return true;
1047}
1048
1049static unsigned getBufferFormatWithCompCount(unsigned OldFormat,
1050 unsigned ComponentCount,
1051 const GCNSubtarget &STI) {
1052 if (ComponentCount > 4)
1053 return 0;
1054
1055 const llvm::AMDGPU::GcnBufferFormatInfo *OldFormatInfo =
1056 llvm::AMDGPU::getGcnBufferFormatInfo(Format: OldFormat, STI);
1057 if (!OldFormatInfo)
1058 return 0;
1059
1060 const llvm::AMDGPU::GcnBufferFormatInfo *NewFormatInfo =
1061 llvm::AMDGPU::getGcnBufferFormatInfo(BitsPerComp: OldFormatInfo->BitsPerComp,
1062 NumComponents: ComponentCount,
1063 NumFormat: OldFormatInfo->NumFormat, STI);
1064
1065 if (!NewFormatInfo)
1066 return 0;
1067
1068 assert(NewFormatInfo->NumFormat == OldFormatInfo->NumFormat &&
1069 NewFormatInfo->BitsPerComp == OldFormatInfo->BitsPerComp);
1070
1071 return NewFormatInfo->Format;
1072}
1073
1074// Return the value in the inclusive range [Lo,Hi] that is aligned to the
1075// highest power of two. Note that the result is well defined for all inputs
1076// including corner cases like:
1077// - if Lo == Hi, return that value
1078// - if Lo == 0, return 0 (even though the "- 1" below underflows
1079// - if Lo > Hi, return 0 (as if the range wrapped around)
1080static uint32_t mostAlignedValueInRange(uint32_t Lo, uint32_t Hi) {
1081 return Hi & maskLeadingOnes<uint32_t>(N: llvm::countl_zero(Val: (Lo - 1) ^ Hi) + 1);
1082}
1083
1084bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI,
1085 const GCNSubtarget &STI,
1086 CombineInfo &Paired,
1087 bool Modify) {
1088 assert(CI.InstClass != MIMG);
1089
1090 // XXX - Would the same offset be OK? Is there any reason this would happen or
1091 // be useful?
1092 if (CI.Offset == Paired.Offset)
1093 return false;
1094
1095 // This won't be valid if the offset isn't aligned.
1096 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0))
1097 return false;
1098
1099 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) {
1100
1101 const llvm::AMDGPU::GcnBufferFormatInfo *Info0 =
1102 llvm::AMDGPU::getGcnBufferFormatInfo(Format: CI.Format, STI);
1103 const llvm::AMDGPU::GcnBufferFormatInfo *Info1 =
1104 llvm::AMDGPU::getGcnBufferFormatInfo(Format: Paired.Format, STI);
1105
1106 if (Info0->BitsPerComp != Info1->BitsPerComp ||
1107 Info0->NumFormat != Info1->NumFormat)
1108 return false;
1109
1110 // For 8-bit or 16-bit formats there is no 3-component variant.
1111 // If NumCombinedComponents is 3, try the 4-component format and use XYZ.
1112 // Example:
1113 // tbuffer_load_format_x + tbuffer_load_format_x + tbuffer_load_format_x
1114 // ==> tbuffer_load_format_xyz with format:[BUF_FMT_16_16_16_16_SNORM]
1115 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1116 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1117 NumCombinedComponents = 4;
1118
1119 if (getBufferFormatWithCompCount(OldFormat: CI.Format, ComponentCount: NumCombinedComponents, STI) ==
1120 0)
1121 return false;
1122
1123 // Merge only when the two access ranges are strictly back-to-back,
1124 // any gap or overlap can over-write data or leave holes.
1125 unsigned ElemIndex0 = CI.Offset / CI.EltSize;
1126 unsigned ElemIndex1 = Paired.Offset / Paired.EltSize;
1127 if (ElemIndex0 + CI.Width != ElemIndex1 &&
1128 ElemIndex1 + Paired.Width != ElemIndex0)
1129 return false;
1130
1131 // 1-byte formats require 1-byte alignment.
1132 // 2-byte formats require 2-byte alignment.
1133 // 4-byte and larger formats require 4-byte alignment.
1134 unsigned MergedBytes = CI.EltSize * NumCombinedComponents;
1135 unsigned RequiredAlign = std::min(a: MergedBytes, b: 4u);
1136 unsigned MinOff = std::min(a: CI.Offset, b: Paired.Offset);
1137 if (MinOff % RequiredAlign != 0)
1138 return false;
1139
1140 return true;
1141 }
1142
1143 uint32_t EltOffset0 = CI.Offset / CI.EltSize;
1144 uint32_t EltOffset1 = Paired.Offset / CI.EltSize;
1145 CI.UseST64 = false;
1146 CI.BaseOff = 0;
1147
1148 // Handle all non-DS instructions.
1149 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) {
1150 if (EltOffset0 + CI.Width != EltOffset1 &&
1151 EltOffset1 + Paired.Width != EltOffset0)
1152 return false;
1153 // Instructions with scale_offset modifier cannot be combined unless we
1154 // also generate a code to scale the offset and reset that bit.
1155 if (CI.CPol != Paired.CPol || (CI.CPol & AMDGPU::CPol::SCAL))
1156 return false;
1157 if (CI.InstClass == S_LOAD_IMM || CI.InstClass == S_BUFFER_LOAD_IMM ||
1158 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) {
1159 // Reject cases like:
1160 // dword + dwordx2 -> dwordx3
1161 // dword + dwordx3 -> dwordx4
1162 // If we tried to combine these cases, we would fail to extract a subreg
1163 // for the result of the second load due to SGPR alignment requirements.
1164 if (CI.Width != Paired.Width &&
1165 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset))
1166 return false;
1167 }
1168 return true;
1169 }
1170
1171 // If the offset in elements doesn't fit in 8-bits, we might be able to use
1172 // the stride 64 versions.
1173 if ((EltOffset0 % 64 == 0) && (EltOffset1 % 64) == 0 &&
1174 isUInt<8>(x: EltOffset0 / 64) && isUInt<8>(x: EltOffset1 / 64)) {
1175 if (Modify) {
1176 CI.Offset = EltOffset0 / 64;
1177 Paired.Offset = EltOffset1 / 64;
1178 CI.UseST64 = true;
1179 }
1180 return true;
1181 }
1182
1183 // Check if the new offsets fit in the reduced 8-bit range.
1184 if (isUInt<8>(x: EltOffset0) && isUInt<8>(x: EltOffset1)) {
1185 if (Modify) {
1186 CI.Offset = EltOffset0;
1187 Paired.Offset = EltOffset1;
1188 }
1189 return true;
1190 }
1191
1192 // Try to shift base address to decrease offsets.
1193 uint32_t Min = std::min(a: EltOffset0, b: EltOffset1);
1194 uint32_t Max = std::max(a: EltOffset0, b: EltOffset1);
1195
1196 const uint32_t Mask = maskTrailingOnes<uint32_t>(N: 8) * 64;
1197 if (((Max - Min) & ~Mask) == 0) {
1198 if (Modify) {
1199 // From the range of values we could use for BaseOff, choose the one that
1200 // is aligned to the highest power of two, to maximise the chance that
1201 // the same offset can be reused for other load/store pairs.
1202 uint32_t BaseOff = mostAlignedValueInRange(Lo: Max - 0xff * 64, Hi: Min);
1203 // Copy the low bits of the offsets, so that when we adjust them by
1204 // subtracting BaseOff they will be multiples of 64.
1205 BaseOff |= Min & maskTrailingOnes<uint32_t>(N: 6);
1206 CI.BaseOff = BaseOff * CI.EltSize;
1207 CI.Offset = (EltOffset0 - BaseOff) / 64;
1208 Paired.Offset = (EltOffset1 - BaseOff) / 64;
1209 CI.UseST64 = true;
1210 }
1211 return true;
1212 }
1213
1214 if (isUInt<8>(x: Max - Min)) {
1215 if (Modify) {
1216 // From the range of values we could use for BaseOff, choose the one that
1217 // is aligned to the highest power of two, to maximise the chance that
1218 // the same offset can be reused for other load/store pairs.
1219 uint32_t BaseOff = mostAlignedValueInRange(Lo: Max - 0xff, Hi: Min);
1220 CI.BaseOff = BaseOff * CI.EltSize;
1221 CI.Offset = EltOffset0 - BaseOff;
1222 Paired.Offset = EltOffset1 - BaseOff;
1223 }
1224 return true;
1225 }
1226
1227 return false;
1228}
1229
1230bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM,
1231 const CombineInfo &CI,
1232 const CombineInfo &Paired) {
1233 const unsigned Width = (CI.Width + Paired.Width);
1234 switch (CI.InstClass) {
1235 default:
1236 return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3));
1237 case S_BUFFER_LOAD_IMM:
1238 case S_BUFFER_LOAD_SGPR_IMM:
1239 case S_LOAD_IMM:
1240 switch (Width) {
1241 default:
1242 return false;
1243 case 2:
1244 case 4:
1245 case 8:
1246 return true;
1247 case 3:
1248 return STM.hasScalarDwordx3Loads();
1249 }
1250 }
1251}
1252
1253const TargetRegisterClass *
1254SILoadStoreOptimizer::getDataRegClass(const MachineInstr &MI) const {
1255 if (const auto *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)) {
1256 return TRI->getRegClassForReg(MRI: *MRI, Reg: Dst->getReg());
1257 }
1258 if (const auto *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata)) {
1259 return TRI->getRegClassForReg(MRI: *MRI, Reg: Src->getReg());
1260 }
1261 if (const auto *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0)) {
1262 return TRI->getRegClassForReg(MRI: *MRI, Reg: Src->getReg());
1263 }
1264 if (const auto *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
1265 return TRI->getRegClassForReg(MRI: *MRI, Reg: Dst->getReg());
1266 }
1267 if (const auto *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdata)) {
1268 return TRI->getRegClassForReg(MRI: *MRI, Reg: Src->getReg());
1269 }
1270 return nullptr;
1271}
1272
1273/// This function assumes that CI comes before Paired in a basic block. Return
1274/// an insertion point for the merged instruction or nullptr on failure.
1275SILoadStoreOptimizer::CombineInfo *
1276SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI,
1277 CombineInfo &Paired) {
1278 // If another instruction has already been merged into CI, it may now be a
1279 // type that we can't do any further merging into.
1280 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN)
1281 return nullptr;
1282 assert(CI.InstClass == Paired.InstClass);
1283
1284 if (getInstSubclass(Opc: CI.I->getOpcode(), TII: *TII) !=
1285 getInstSubclass(Opc: Paired.I->getOpcode(), TII: *TII))
1286 return nullptr;
1287
1288 // Check both offsets (or masks for MIMG) can be combined and fit in the
1289 // reduced range.
1290 if (CI.InstClass == MIMG) {
1291 if (!dmasksCanBeCombined(CI, TII: *TII, Paired))
1292 return nullptr;
1293 } else {
1294 if (!widthsFit(STM: *STM, CI, Paired) || !offsetsCanBeCombined(CI, STI: *STM, Paired))
1295 return nullptr;
1296 }
1297
1298 DenseSet<Register> RegDefs;
1299 DenseSet<Register> RegUses;
1300 CombineInfo *Where;
1301 if (CI.I->mayLoad()) {
1302 // Try to hoist Paired up to CI.
1303 addDefsUsesToList(MI: *Paired.I, RegDefs, RegUses);
1304 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) {
1305 if (!canSwapInstructions(ARegDefs: RegDefs, ARegUses: RegUses, A: *Paired.I, B: *MBBI))
1306 return nullptr;
1307 }
1308 Where = &CI;
1309 } else {
1310 // Try to sink CI down to Paired.
1311 addDefsUsesToList(MI: *CI.I, RegDefs, RegUses);
1312 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) {
1313 if (!canSwapInstructions(ARegDefs: RegDefs, ARegUses: RegUses, A: *CI.I, B: *MBBI))
1314 return nullptr;
1315 }
1316 Where = &Paired;
1317 }
1318
1319 // Call offsetsCanBeCombined with modify = true so that the offsets are
1320 // correct for the new instruction. This should return true, because
1321 // this function should only be called on CombineInfo objects that
1322 // have already been confirmed to be mergeable.
1323 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) {
1324 if (STM->hasNeedsAligned2addrDS() &&
1325 (CI.I->memoperands_empty() ||
1326 (*CI.I->memoperands_begin())->getAlign().value() < CI.Width * 4))
1327 return nullptr;
1328 offsetsCanBeCombined(CI, STI: *STM, Paired, Modify: true);
1329 }
1330
1331 if (CI.InstClass == DS_WRITE) {
1332 // Both data operands must be AGPR or VGPR, so the data registers needs to
1333 // be constrained to one or the other. We expect to only emit the VGPR form
1334 // here for now.
1335 //
1336 // FIXME: There is currently a hack in getRegClass to report that the write2
1337 // operands are VGPRs. In the future we should have separate agpr
1338 // instruction definitions.
1339 const MachineOperand *Data0 =
1340 TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::data0);
1341 const MachineOperand *Data1 =
1342 TII->getNamedOperand(MI&: *Paired.I, OperandName: AMDGPU::OpName::data0);
1343
1344 const MCInstrDesc &Write2Opc = TII->get(Opcode: getWrite2Opcode(CI));
1345 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Write2Opc.getOpcode(),
1346 Name: AMDGPU::OpName::data0);
1347 int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Write2Opc.getOpcode(),
1348 Name: AMDGPU::OpName::data1);
1349
1350 const TargetRegisterClass *DataRC0 = TII->getRegClass(MCID: Write2Opc, OpNum: Data0Idx);
1351
1352 const TargetRegisterClass *DataRC1 = TII->getRegClass(MCID: Write2Opc, OpNum: Data1Idx);
1353
1354 if (unsigned SubReg = Data0->getSubReg()) {
1355 DataRC0 = TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: Data0->getReg()),
1356 B: DataRC0, Idx: SubReg);
1357 }
1358
1359 if (unsigned SubReg = Data1->getSubReg()) {
1360 DataRC1 = TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: Data1->getReg()),
1361 B: DataRC1, Idx: SubReg);
1362 }
1363
1364 if (!MRI->constrainRegClass(Reg: Data0->getReg(), RC: DataRC0) ||
1365 !MRI->constrainRegClass(Reg: Data1->getReg(), RC: DataRC1))
1366 return nullptr;
1367
1368 // TODO: If one register can be constrained, and not the other, insert a
1369 // copy.
1370 }
1371
1372 return Where;
1373}
1374
1375// Copy the merged load result from DestReg to the original dest regs of CI and
1376// Paired.
1377void SILoadStoreOptimizer::copyToDestRegs(
1378 CombineInfo &CI, CombineInfo &Paired,
1379 MachineBasicBlock::iterator InsertBefore, const DebugLoc &DL,
1380 AMDGPU::OpName OpName, Register DestReg) const {
1381 MachineBasicBlock *MBB = CI.I->getParent();
1382
1383 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1384
1385 // Copy to the old destination registers.
1386 const MCInstrDesc &CopyDesc = TII->get(Opcode: TargetOpcode::COPY);
1387 auto *Dest0 = TII->getNamedOperand(MI&: *CI.I, OperandName: OpName);
1388 auto *Dest1 = TII->getNamedOperand(MI&: *Paired.I, OperandName: OpName);
1389
1390 // The constrained sload instructions in S_LOAD_IMM class will have
1391 // `early-clobber` flag in the dst operand. Remove the flag before using the
1392 // MOs in copies.
1393 Dest0->setIsEarlyClobber(false);
1394 Dest1->setIsEarlyClobber(false);
1395
1396 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: CopyDesc)
1397 .add(MO: *Dest0) // Copy to same destination including flags and sub reg.
1398 .addReg(RegNo: DestReg, Flags: {}, SubReg: SubRegIdx0);
1399 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: CopyDesc)
1400 .add(MO: *Dest1)
1401 .addReg(RegNo: DestReg, Flags: RegState::Kill, SubReg: SubRegIdx1);
1402}
1403
1404// Return a register for the source of the merged store after copying the
1405// original source regs of CI and Paired into it.
1406Register
1407SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
1408 MachineBasicBlock::iterator InsertBefore,
1409 const DebugLoc &DL,
1410 AMDGPU::OpName OpName) const {
1411 MachineBasicBlock *MBB = CI.I->getParent();
1412
1413 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1414
1415 // Copy to the new source register.
1416 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1417 Register SrcReg = MRI->createVirtualRegister(RegClass: SuperRC);
1418
1419 const auto *Src0 = TII->getNamedOperand(MI&: *CI.I, OperandName: OpName);
1420 const auto *Src1 = TII->getNamedOperand(MI&: *Paired.I, OperandName: OpName);
1421
1422 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SrcReg)
1423 .add(MO: *Src0)
1424 .addImm(Val: SubRegIdx0)
1425 .add(MO: *Src1)
1426 .addImm(Val: SubRegIdx1);
1427
1428 return SrcReg;
1429}
1430
1431unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize) const {
1432 if (STM->ldsRequiresM0Init())
1433 return (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64;
1434 return (EltSize == 4) ? AMDGPU::DS_READ2_B32_gfx9 : AMDGPU::DS_READ2_B64_gfx9;
1435}
1436
1437unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize) const {
1438 if (STM->ldsRequiresM0Init())
1439 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64;
1440
1441 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32_gfx9
1442 : AMDGPU::DS_READ2ST64_B64_gfx9;
1443}
1444
1445MachineBasicBlock::iterator
1446SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
1447 MachineBasicBlock::iterator InsertBefore) {
1448 MachineBasicBlock *MBB = CI.I->getParent();
1449
1450 // Be careful, since the addresses could be subregisters themselves in weird
1451 // cases, like vectors of pointers.
1452 const auto *AddrReg = TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::addr);
1453
1454 unsigned NewOffset0 = std::min(a: CI.Offset, b: Paired.Offset);
1455 unsigned NewOffset1 = std::max(a: CI.Offset, b: Paired.Offset);
1456 unsigned Opc =
1457 CI.UseST64 ? read2ST64Opcode(EltSize: CI.EltSize) : read2Opcode(EltSize: CI.EltSize);
1458
1459 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1460 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1461
1462 const MCInstrDesc &Read2Desc = TII->get(Opcode: Opc);
1463
1464 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1465 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1466
1467 DebugLoc DL =
1468 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1469
1470 Register BaseReg = AddrReg->getReg();
1471 unsigned BaseSubReg = AddrReg->getSubReg();
1472 RegState BaseRegFlags = {};
1473 if (CI.BaseOff) {
1474 Register ImmReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1475 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: ImmReg)
1476 .addImm(Val: CI.BaseOff);
1477
1478 BaseReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1479 BaseRegFlags = RegState::Kill;
1480
1481 TII->getAddNoCarry(MBB&: *MBB, I: InsertBefore, DL, DestReg: BaseReg)
1482 .addReg(RegNo: ImmReg)
1483 .addReg(RegNo: AddrReg->getReg(), Flags: {}, SubReg: BaseSubReg)
1484 .addImm(Val: 0); // clamp bit
1485 BaseSubReg = 0;
1486 }
1487
1488 MachineInstrBuilder Read2 =
1489 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: Read2Desc, DestReg)
1490 .addReg(RegNo: BaseReg, Flags: BaseRegFlags, SubReg: BaseSubReg) // addr
1491 .addImm(Val: NewOffset0) // offset0
1492 .addImm(Val: NewOffset1) // offset1
1493 .addImm(Val: 0) // gds
1494 .cloneMergedMemRefs(OtherMIs: {&*CI.I, &*Paired.I});
1495
1496 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdst, DestReg);
1497
1498 CI.I->eraseFromParent();
1499 Paired.I->eraseFromParent();
1500
1501 LLVM_DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n');
1502 return Read2;
1503}
1504
1505unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize) const {
1506 if (STM->ldsRequiresM0Init())
1507 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64;
1508 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32_gfx9
1509 : AMDGPU::DS_WRITE2_B64_gfx9;
1510}
1511
1512unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize) const {
1513 if (STM->ldsRequiresM0Init())
1514 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32
1515 : AMDGPU::DS_WRITE2ST64_B64;
1516
1517 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32_gfx9
1518 : AMDGPU::DS_WRITE2ST64_B64_gfx9;
1519}
1520
1521unsigned SILoadStoreOptimizer::getWrite2Opcode(const CombineInfo &CI) const {
1522 return CI.UseST64 ? write2ST64Opcode(EltSize: CI.EltSize) : write2Opcode(EltSize: CI.EltSize);
1523}
1524
1525MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
1526 CombineInfo &CI, CombineInfo &Paired,
1527 MachineBasicBlock::iterator InsertBefore) {
1528 MachineBasicBlock *MBB = CI.I->getParent();
1529
1530 // Be sure to use .addOperand(), and not .addReg() with these. We want to be
1531 // sure we preserve the subregister index and any register flags set on them.
1532 const MachineOperand *AddrReg =
1533 TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::addr);
1534 const MachineOperand *Data0 =
1535 TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::data0);
1536 const MachineOperand *Data1 =
1537 TII->getNamedOperand(MI&: *Paired.I, OperandName: AMDGPU::OpName::data0);
1538
1539 unsigned NewOffset0 = CI.Offset;
1540 unsigned NewOffset1 = Paired.Offset;
1541 unsigned Opc = getWrite2Opcode(CI);
1542
1543 if (NewOffset0 > NewOffset1) {
1544 // Canonicalize the merged instruction so the smaller offset comes first.
1545 std::swap(a&: NewOffset0, b&: NewOffset1);
1546 std::swap(a&: Data0, b&: Data1);
1547 }
1548
1549 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1550 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1551
1552 const MCInstrDesc &Write2Desc = TII->get(Opcode: Opc);
1553 DebugLoc DL =
1554 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1555
1556 Register BaseReg = AddrReg->getReg();
1557 unsigned BaseSubReg = AddrReg->getSubReg();
1558 RegState BaseRegFlags = {};
1559 if (CI.BaseOff) {
1560 Register ImmReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1561 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: ImmReg)
1562 .addImm(Val: CI.BaseOff);
1563
1564 BaseReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1565 BaseRegFlags = RegState::Kill;
1566
1567 TII->getAddNoCarry(MBB&: *MBB, I: InsertBefore, DL, DestReg: BaseReg)
1568 .addReg(RegNo: ImmReg)
1569 .addReg(RegNo: AddrReg->getReg(), Flags: {}, SubReg: BaseSubReg)
1570 .addImm(Val: 0); // clamp bit
1571 BaseSubReg = 0;
1572 }
1573
1574 MachineInstrBuilder Write2 =
1575 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: Write2Desc)
1576 .addReg(RegNo: BaseReg, Flags: BaseRegFlags, SubReg: BaseSubReg) // addr
1577 .add(MO: *Data0) // data0
1578 .add(MO: *Data1) // data1
1579 .addImm(Val: NewOffset0) // offset0
1580 .addImm(Val: NewOffset1) // offset1
1581 .addImm(Val: 0) // gds
1582 .cloneMergedMemRefs(OtherMIs: {&*CI.I, &*Paired.I});
1583
1584 CI.I->eraseFromParent();
1585 Paired.I->eraseFromParent();
1586
1587 LLVM_DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n');
1588 return Write2;
1589}
1590
1591MachineBasicBlock::iterator
1592SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
1593 MachineBasicBlock::iterator InsertBefore) {
1594 MachineBasicBlock *MBB = CI.I->getParent();
1595 DebugLoc DL =
1596 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1597
1598 const unsigned Opcode = getNewOpcode(CI, Paired);
1599
1600 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1601
1602 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1603 unsigned MergedDMask = CI.DMask | Paired.DMask;
1604 unsigned DMaskIdx =
1605 AMDGPU::getNamedOperandIdx(Opcode: CI.I->getOpcode(), Name: AMDGPU::OpName::dmask);
1606
1607 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1608 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) {
1609 if (I == DMaskIdx)
1610 MIB.addImm(Val: MergedDMask);
1611 else
1612 MIB.add(MO: (*CI.I).getOperand(i: I));
1613 }
1614
1615 // It shouldn't be possible to get this far if the two instructions
1616 // don't have a single memoperand, because MachineInstr::mayAlias()
1617 // will return true if this is the case.
1618 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1619
1620 MachineInstr *New = MIB.addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1621
1622 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata, DestReg);
1623
1624 CI.I->eraseFromParent();
1625 Paired.I->eraseFromParent();
1626 return New;
1627}
1628
1629MachineBasicBlock::iterator SILoadStoreOptimizer::mergeSMemLoadImmPair(
1630 CombineInfo &CI, CombineInfo &Paired,
1631 MachineBasicBlock::iterator InsertBefore) {
1632 MachineBasicBlock *MBB = CI.I->getParent();
1633 DebugLoc DL =
1634 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1635
1636 const unsigned Opcode = getNewOpcode(CI, Paired);
1637
1638 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1639
1640 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1641 unsigned MergedOffset = std::min(a: CI.Offset, b: Paired.Offset);
1642
1643 // It shouldn't be possible to get this far if the two instructions
1644 // don't have a single memoperand, because MachineInstr::mayAlias()
1645 // will return true if this is the case.
1646 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1647
1648 MachineInstrBuilder New =
1649 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg)
1650 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::sbase));
1651 if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM)
1652 New.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset));
1653 New.addImm(Val: MergedOffset);
1654 New.addImm(Val: CI.CPol).addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1655
1656 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::sdst, DestReg);
1657
1658 CI.I->eraseFromParent();
1659 Paired.I->eraseFromParent();
1660 return New;
1661}
1662
1663MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferLoadPair(
1664 CombineInfo &CI, CombineInfo &Paired,
1665 MachineBasicBlock::iterator InsertBefore) {
1666 MachineBasicBlock *MBB = CI.I->getParent();
1667
1668 DebugLoc DL =
1669 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1670
1671 const unsigned Opcode = getNewOpcode(CI, Paired);
1672
1673 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1674
1675 // Copy to the new source register.
1676 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1677 unsigned MergedOffset = std::min(a: CI.Offset, b: Paired.Offset);
1678
1679 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1680
1681 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
1682
1683 if (Regs.VAddr)
1684 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
1685
1686 // It shouldn't be possible to get this far if the two instructions
1687 // don't have a single memoperand, because MachineInstr::mayAlias()
1688 // will return true if this is the case.
1689 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1690
1691 MachineInstr *New =
1692 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
1693 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
1694 .addImm(Val: MergedOffset) // offset
1695 .addImm(Val: CI.CPol) // cpol
1696 .addImm(Val: 0) // swz
1697 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1698
1699 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata, DestReg);
1700
1701 CI.I->eraseFromParent();
1702 Paired.I->eraseFromParent();
1703 return New;
1704}
1705
1706MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair(
1707 CombineInfo &CI, CombineInfo &Paired,
1708 MachineBasicBlock::iterator InsertBefore) {
1709 MachineBasicBlock *MBB = CI.I->getParent();
1710
1711 DebugLoc DL =
1712 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1713
1714 const unsigned Opcode = getNewOpcode(CI, Paired);
1715
1716 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1717
1718 // Copy to the new source register.
1719 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1720 unsigned MergedOffset = std::min(a: CI.Offset, b: Paired.Offset);
1721
1722 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1723
1724 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
1725
1726 if (Regs.VAddr)
1727 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
1728
1729 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1730 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1731 // and use XYZ of XYZW to enable the merge.
1732 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1733 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1734 NumCombinedComponents = 4;
1735 unsigned JoinedFormat =
1736 getBufferFormatWithCompCount(OldFormat: CI.Format, ComponentCount: NumCombinedComponents, STI: *STM);
1737
1738 // It shouldn't be possible to get this far if the two instructions
1739 // don't have a single memoperand, because MachineInstr::mayAlias()
1740 // will return true if this is the case.
1741 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1742
1743 MachineInstr *New =
1744 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
1745 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
1746 .addImm(Val: MergedOffset) // offset
1747 .addImm(Val: JoinedFormat) // format
1748 .addImm(Val: CI.CPol) // cpol
1749 .addImm(Val: 0) // swz
1750 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1751
1752 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata, DestReg);
1753
1754 CI.I->eraseFromParent();
1755 Paired.I->eraseFromParent();
1756 return New;
1757}
1758
1759MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair(
1760 CombineInfo &CI, CombineInfo &Paired,
1761 MachineBasicBlock::iterator InsertBefore) {
1762 MachineBasicBlock *MBB = CI.I->getParent();
1763 DebugLoc DL =
1764 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1765
1766 const unsigned Opcode = getNewOpcode(CI, Paired);
1767
1768 Register SrcReg =
1769 copyFromSrcRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata);
1770
1771 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode))
1772 .addReg(RegNo: SrcReg, Flags: RegState::Kill);
1773
1774 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
1775
1776 if (Regs.VAddr)
1777 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
1778
1779 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1780 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1781 // and use XYZ of XYZW to enable the merge.
1782 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1783 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1784 NumCombinedComponents = 4;
1785 unsigned JoinedFormat =
1786 getBufferFormatWithCompCount(OldFormat: CI.Format, ComponentCount: NumCombinedComponents, STI: *STM);
1787
1788 // It shouldn't be possible to get this far if the two instructions
1789 // don't have a single memoperand, because MachineInstr::mayAlias()
1790 // will return true if this is the case.
1791 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1792
1793 MachineInstr *New =
1794 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
1795 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
1796 .addImm(Val: std::min(a: CI.Offset, b: Paired.Offset)) // offset
1797 .addImm(Val: JoinedFormat) // format
1798 .addImm(Val: CI.CPol) // cpol
1799 .addImm(Val: 0) // swz
1800 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1801
1802 CI.I->eraseFromParent();
1803 Paired.I->eraseFromParent();
1804 return New;
1805}
1806
1807MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatLoadPair(
1808 CombineInfo &CI, CombineInfo &Paired,
1809 MachineBasicBlock::iterator InsertBefore) {
1810 MachineBasicBlock *MBB = CI.I->getParent();
1811
1812 DebugLoc DL =
1813 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1814
1815 const unsigned Opcode = getNewOpcode(CI, Paired);
1816
1817 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1818 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1819
1820 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1821
1822 if (auto *SAddr = TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::saddr))
1823 MIB.add(MO: *SAddr);
1824
1825 MachineInstr *New =
1826 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr))
1827 .addImm(Val: std::min(a: CI.Offset, b: Paired.Offset))
1828 .addImm(Val: CI.CPol)
1829 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1830
1831 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdst, DestReg);
1832
1833 CI.I->eraseFromParent();
1834 Paired.I->eraseFromParent();
1835 return New;
1836}
1837
1838MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatStorePair(
1839 CombineInfo &CI, CombineInfo &Paired,
1840 MachineBasicBlock::iterator InsertBefore) {
1841 MachineBasicBlock *MBB = CI.I->getParent();
1842
1843 DebugLoc DL =
1844 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1845
1846 const unsigned Opcode = getNewOpcode(CI, Paired);
1847
1848 Register SrcReg =
1849 copyFromSrcRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata);
1850
1851 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode))
1852 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr))
1853 .addReg(RegNo: SrcReg, Flags: RegState::Kill);
1854
1855 if (auto *SAddr = TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::saddr))
1856 MIB.add(MO: *SAddr);
1857
1858 MachineInstr *New =
1859 MIB.addImm(Val: std::min(a: CI.Offset, b: Paired.Offset))
1860 .addImm(Val: CI.CPol)
1861 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1862
1863 CI.I->eraseFromParent();
1864 Paired.I->eraseFromParent();
1865 return New;
1866}
1867
1868static bool needsConstrainedOpcode(const GCNSubtarget &STM,
1869 ArrayRef<MachineMemOperand *> MMOs,
1870 unsigned Width) {
1871 // Conservatively returns true if not found the MMO.
1872 return STM.isXNACKEnabled() &&
1873 (MMOs.size() != 1 || MMOs[0]->getAlign().value() < Width * 4);
1874}
1875
1876unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
1877 const CombineInfo &Paired) {
1878 const unsigned Width = CI.Width + Paired.Width;
1879 const CombineInfo &Leading = Paired < CI ? Paired : CI;
1880 // If XNACK is enabled, use the constrained opcodes when the first load is
1881 // under-aligned.
1882 const bool NeedsConstrainedOpc =
1883 needsConstrainedOpcode(STM: *STM, MMOs: Leading.I->memoperands(), Width);
1884
1885 switch (getCommonInstClass(CI, Paired)) {
1886 default:
1887 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE);
1888 // FIXME: Handle d16 correctly
1889 return AMDGPU::getMUBUFOpcode(BaseOpc: AMDGPU::getMUBUFBaseOpcode(Opc: CI.I->getOpcode()),
1890 Elements: Width);
1891 case TBUFFER_LOAD:
1892 case TBUFFER_STORE:
1893 return AMDGPU::getMTBUFOpcode(BaseOpc: AMDGPU::getMTBUFBaseOpcode(Opc: CI.I->getOpcode()),
1894 Elements: Width);
1895
1896 case UNKNOWN:
1897 llvm_unreachable("Unknown instruction class");
1898 case S_BUFFER_LOAD_IMM: {
1899 switch (Width) {
1900 default:
1901 return 0;
1902 case 2:
1903 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec
1904 : AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM;
1905 case 3:
1906 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec
1907 : AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM;
1908 case 4:
1909 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec
1910 : AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM;
1911 case 8:
1912 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec
1913 : AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM;
1914 }
1915 }
1916 case S_BUFFER_LOAD_SGPR_IMM: {
1917 switch (Width) {
1918 default:
1919 return 0;
1920 case 2:
1921 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec
1922 : AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM;
1923 case 3:
1924 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec
1925 : AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM;
1926 case 4:
1927 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec
1928 : AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM;
1929 case 8:
1930 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec
1931 : AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
1932 }
1933 }
1934 case S_LOAD_IMM: {
1935 switch (Width) {
1936 default:
1937 return 0;
1938 case 2:
1939 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX2_IMM_ec
1940 : AMDGPU::S_LOAD_DWORDX2_IMM;
1941 case 3:
1942 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX3_IMM_ec
1943 : AMDGPU::S_LOAD_DWORDX3_IMM;
1944 case 4:
1945 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX4_IMM_ec
1946 : AMDGPU::S_LOAD_DWORDX4_IMM;
1947 case 8:
1948 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX8_IMM_ec
1949 : AMDGPU::S_LOAD_DWORDX8_IMM;
1950 }
1951 }
1952 case GLOBAL_LOAD:
1953 switch (Width) {
1954 default:
1955 return 0;
1956 case 2:
1957 return AMDGPU::GLOBAL_LOAD_DWORDX2;
1958 case 3:
1959 return AMDGPU::GLOBAL_LOAD_DWORDX3;
1960 case 4:
1961 return AMDGPU::GLOBAL_LOAD_DWORDX4;
1962 }
1963 case GLOBAL_LOAD_SADDR:
1964 switch (Width) {
1965 default:
1966 return 0;
1967 case 2:
1968 return AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR;
1969 case 3:
1970 return AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR;
1971 case 4:
1972 return AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR;
1973 }
1974 case GLOBAL_STORE:
1975 switch (Width) {
1976 default:
1977 return 0;
1978 case 2:
1979 return AMDGPU::GLOBAL_STORE_DWORDX2;
1980 case 3:
1981 return AMDGPU::GLOBAL_STORE_DWORDX3;
1982 case 4:
1983 return AMDGPU::GLOBAL_STORE_DWORDX4;
1984 }
1985 case GLOBAL_STORE_SADDR:
1986 switch (Width) {
1987 default:
1988 return 0;
1989 case 2:
1990 return AMDGPU::GLOBAL_STORE_DWORDX2_SADDR;
1991 case 3:
1992 return AMDGPU::GLOBAL_STORE_DWORDX3_SADDR;
1993 case 4:
1994 return AMDGPU::GLOBAL_STORE_DWORDX4_SADDR;
1995 }
1996 case FLAT_LOAD:
1997 switch (Width) {
1998 default:
1999 return 0;
2000 case 2:
2001 return AMDGPU::FLAT_LOAD_DWORDX2;
2002 case 3:
2003 return AMDGPU::FLAT_LOAD_DWORDX3;
2004 case 4:
2005 return AMDGPU::FLAT_LOAD_DWORDX4;
2006 }
2007 case FLAT_STORE:
2008 switch (Width) {
2009 default:
2010 return 0;
2011 case 2:
2012 return AMDGPU::FLAT_STORE_DWORDX2;
2013 case 3:
2014 return AMDGPU::FLAT_STORE_DWORDX3;
2015 case 4:
2016 return AMDGPU::FLAT_STORE_DWORDX4;
2017 }
2018 case FLAT_LOAD_SADDR:
2019 switch (Width) {
2020 default:
2021 return 0;
2022 case 2:
2023 return AMDGPU::FLAT_LOAD_DWORDX2_SADDR;
2024 case 3:
2025 return AMDGPU::FLAT_LOAD_DWORDX3_SADDR;
2026 case 4:
2027 return AMDGPU::FLAT_LOAD_DWORDX4_SADDR;
2028 }
2029 case FLAT_STORE_SADDR:
2030 switch (Width) {
2031 default:
2032 return 0;
2033 case 2:
2034 return AMDGPU::FLAT_STORE_DWORDX2_SADDR;
2035 case 3:
2036 return AMDGPU::FLAT_STORE_DWORDX3_SADDR;
2037 case 4:
2038 return AMDGPU::FLAT_STORE_DWORDX4_SADDR;
2039 }
2040 case MIMG:
2041 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) &&
2042 "No overlaps");
2043 return AMDGPU::getMaskedMIMGOp(Opc: CI.I->getOpcode(), NewChannels: Width);
2044 }
2045}
2046
2047std::pair<unsigned, unsigned>
2048SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI,
2049 const CombineInfo &Paired) {
2050 assert((CI.InstClass != MIMG ||
2051 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) ==
2052 CI.Width + Paired.Width)) &&
2053 "No overlaps");
2054
2055 unsigned Idx0;
2056 unsigned Idx1;
2057
2058 static const unsigned Idxs[5][4] = {
2059 {AMDGPU::sub0, AMDGPU::sub0_sub1, AMDGPU::sub0_sub1_sub2, AMDGPU::sub0_sub1_sub2_sub3},
2060 {AMDGPU::sub1, AMDGPU::sub1_sub2, AMDGPU::sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4},
2061 {AMDGPU::sub2, AMDGPU::sub2_sub3, AMDGPU::sub2_sub3_sub4, AMDGPU::sub2_sub3_sub4_sub5},
2062 {AMDGPU::sub3, AMDGPU::sub3_sub4, AMDGPU::sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6},
2063 {AMDGPU::sub4, AMDGPU::sub4_sub5, AMDGPU::sub4_sub5_sub6, AMDGPU::sub4_sub5_sub6_sub7},
2064 };
2065
2066 assert(CI.Width >= 1 && CI.Width <= 4);
2067 assert(Paired.Width >= 1 && Paired.Width <= 4);
2068
2069 if (Paired < CI) {
2070 Idx1 = Idxs[0][Paired.Width - 1];
2071 Idx0 = Idxs[Paired.Width][CI.Width - 1];
2072 } else {
2073 Idx0 = Idxs[0][CI.Width - 1];
2074 Idx1 = Idxs[CI.Width][Paired.Width - 1];
2075 }
2076
2077 return {Idx0, Idx1};
2078}
2079
2080const TargetRegisterClass *
2081SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI,
2082 const CombineInfo &Paired) const {
2083 if (CI.InstClass == S_BUFFER_LOAD_IMM ||
2084 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM || CI.InstClass == S_LOAD_IMM) {
2085 switch (CI.Width + Paired.Width) {
2086 default:
2087 return nullptr;
2088 case 2:
2089 return &AMDGPU::SReg_64_XEXECRegClass;
2090 case 3:
2091 return &AMDGPU::SGPR_96RegClass;
2092 case 4:
2093 return &AMDGPU::SGPR_128RegClass;
2094 case 8:
2095 return &AMDGPU::SGPR_256RegClass;
2096 case 16:
2097 return &AMDGPU::SGPR_512RegClass;
2098 }
2099 }
2100
2101 // FIXME: This should compute the instruction to use, and then use the result
2102 // of TII->getRegClass.
2103 unsigned BitWidth = 32 * (CI.Width + Paired.Width);
2104 return TRI->isAGPRClass(RC: getDataRegClass(MI: *CI.I))
2105 ? TRI->getAGPRClassForBitWidth(BitWidth)
2106 : TRI->getVGPRClassForBitWidth(BitWidth);
2107}
2108
2109MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferStorePair(
2110 CombineInfo &CI, CombineInfo &Paired,
2111 MachineBasicBlock::iterator InsertBefore) {
2112 MachineBasicBlock *MBB = CI.I->getParent();
2113 DebugLoc DL =
2114 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
2115
2116 const unsigned Opcode = getNewOpcode(CI, Paired);
2117
2118 Register SrcReg =
2119 copyFromSrcRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata);
2120
2121 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode))
2122 .addReg(RegNo: SrcReg, Flags: RegState::Kill);
2123
2124 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
2125
2126 if (Regs.VAddr)
2127 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
2128
2129
2130 // It shouldn't be possible to get this far if the two instructions
2131 // don't have a single memoperand, because MachineInstr::mayAlias()
2132 // will return true if this is the case.
2133 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
2134
2135 MachineInstr *New =
2136 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
2137 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
2138 .addImm(Val: std::min(a: CI.Offset, b: Paired.Offset)) // offset
2139 .addImm(Val: CI.CPol) // cpol
2140 .addImm(Val: 0) // swz
2141 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
2142
2143 CI.I->eraseFromParent();
2144 Paired.I->eraseFromParent();
2145 return New;
2146}
2147
2148MachineOperand
2149SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) const {
2150 APInt V(32, Val, true);
2151 if (TII->isInlineConstant(Imm: V))
2152 return MachineOperand::CreateImm(Val);
2153
2154 Register Reg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2155 MachineInstr *Mov =
2156 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
2157 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Reg)
2158 .addImm(Val);
2159 (void)Mov;
2160 LLVM_DEBUG(dbgs() << " "; Mov->dump());
2161 return MachineOperand::CreateReg(Reg, isDef: false);
2162}
2163
2164// Compute base address using Addr and return the final register.
2165Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
2166 const MemAddress &Addr) const {
2167 MachineBasicBlock *MBB = MI.getParent();
2168 MachineBasicBlock::iterator MBBI = MI.getIterator();
2169 const DebugLoc &DL = MI.getDebugLoc();
2170
2171 LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n");
2172
2173 // Use V_ADD_U64_e64 when the original pattern used it (gfx1250+)
2174 if (Addr.Base.UseV64Pattern) {
2175 Register FullDestReg = MRI->createVirtualRegister(
2176 RegClass: TII->getRegClass(MCID: TII->get(Opcode: AMDGPU::V_ADD_U64_e64), OpNum: 0));
2177
2178 // Load the 64-bit offset into an SGPR pair if needed
2179 Register OffsetReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2180 MachineInstr *MovOffset =
2181 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64_IMM_PSEUDO),
2182 DestReg: OffsetReg)
2183 .addImm(Val: Addr.Offset);
2184 MachineInstr *Add64 =
2185 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U64_e64), DestReg: FullDestReg)
2186 .addReg(RegNo: Addr.Base.LoReg)
2187 .addReg(RegNo: OffsetReg, Flags: RegState::Kill)
2188 .addImm(Val: 0);
2189 (void)MovOffset;
2190 (void)Add64;
2191 LLVM_DEBUG(dbgs() << " " << *MovOffset << "\n";
2192 dbgs() << " " << *Add64 << "\n\n";);
2193
2194 return FullDestReg;
2195 }
2196
2197 // Original carry-chain pattern (V_ADD_CO_U32 + V_ADDC_U32)
2198 assert((TRI->getRegSizeInBits(Addr.Base.LoReg, *MRI) == 32 ||
2199 Addr.Base.LoSubReg) &&
2200 "Expected 32-bit Base-Register-Low!!");
2201
2202 assert((TRI->getRegSizeInBits(Addr.Base.HiReg, *MRI) == 32 ||
2203 Addr.Base.HiSubReg) &&
2204 "Expected 32-bit Base-Register-Hi!!");
2205
2206 MachineOperand OffsetLo = createRegOrImm(Val: static_cast<int32_t>(Addr.Offset), MI);
2207 MachineOperand OffsetHi =
2208 createRegOrImm(Val: static_cast<int32_t>(Addr.Offset >> 32), MI);
2209
2210 const auto *CarryRC = TRI->getWaveMaskRegClass();
2211 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
2212 Register DeadCarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
2213
2214 Register DestSub0 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2215 Register DestSub1 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2216 MachineInstr *LoHalf =
2217 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DestSub0)
2218 .addReg(RegNo: CarryReg, Flags: RegState::Define)
2219 .addReg(RegNo: Addr.Base.LoReg, Flags: {}, SubReg: Addr.Base.LoSubReg)
2220 .add(MO: OffsetLo)
2221 .addImm(Val: 0); // clamp bit
2222
2223 MachineInstr *HiHalf =
2224 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DestSub1)
2225 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
2226 .addReg(RegNo: Addr.Base.HiReg, Flags: {}, SubReg: Addr.Base.HiSubReg)
2227 .add(MO: OffsetHi)
2228 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
2229 .addImm(Val: 0); // clamp bit
2230
2231 Register FullDestReg = MRI->createVirtualRegister(RegClass: TRI->getVGPR64Class());
2232 MachineInstr *FullBase =
2233 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
2234 .addReg(RegNo: DestSub0)
2235 .addImm(Val: AMDGPU::sub0)
2236 .addReg(RegNo: DestSub1)
2237 .addImm(Val: AMDGPU::sub1);
2238
2239 (void)LoHalf;
2240 (void)HiHalf;
2241 (void)FullBase;
2242 LLVM_DEBUG(dbgs() << " " << *LoHalf << "\n";
2243 dbgs() << " " << *HiHalf << "\n";
2244 dbgs() << " " << *FullBase << "\n\n";);
2245
2246 return FullDestReg;
2247}
2248
2249// Update base and offset with the NewBase and NewOffset in MI.
2250void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
2251 Register NewBase,
2252 int32_t NewOffset) const {
2253 auto *Base = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
2254 Base->setReg(NewBase);
2255 Base->setIsKill(false);
2256 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->setImm(NewOffset);
2257}
2258
2259// Helper to extract a 64-bit constant offset from a V_ADD_U64_e64 instruction.
2260// Returns true if successful, populating Addr with base register info and
2261// offset.
2262bool SILoadStoreOptimizer::processBaseWithConstOffset64(
2263 MachineInstr *AddDef, const MachineOperand &Base, MemAddress &Addr) const {
2264 if (!Base.isReg())
2265 return false;
2266
2267 MachineOperand *Src0 = TII->getNamedOperand(MI&: *AddDef, OperandName: AMDGPU::OpName::src0);
2268 MachineOperand *Src1 = TII->getNamedOperand(MI&: *AddDef, OperandName: AMDGPU::OpName::src1);
2269
2270 const MachineOperand *BaseOp = nullptr;
2271
2272 auto Offset = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2273
2274 if (Offset) {
2275 BaseOp = Src0;
2276 Addr.Offset = *Offset;
2277 } else {
2278 // Both or neither are constants - can't handle this pattern
2279 return false;
2280 }
2281
2282 // Now extract the base register (which should be a 64-bit VGPR).
2283 Addr.Base.LoReg = BaseOp->getReg();
2284 Addr.Base.UseV64Pattern = true;
2285 return true;
2286}
2287
2288// Analyze Base and extracts:
2289// - 32bit base registers, subregisters
2290// - 64bit constant offset
2291// Expecting base computation as:
2292// %OFFSET0:sgpr_32 = S_MOV_B32 8000
2293// %LO:vgpr_32, %c:sreg_64_xexec =
2294// V_ADD_CO_U32_e64 %BASE_LO:vgpr_32, %103:sgpr_32,
2295// %HI:vgpr_32, = V_ADDC_U32_e64 %BASE_HI:vgpr_32, 0, killed %c:sreg_64_xexec
2296// %Base:vreg_64 =
2297// REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1
2298//
2299// Also handles V_ADD_U64_e64 pattern (gfx1250+):
2300// %OFFSET:sreg_64 = S_MOV_B64_IMM_PSEUDO 256
2301// %Base:vreg_64 = V_ADD_U64_e64 %BASE:vreg_64, %OFFSET:sreg_64, 0
2302void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base,
2303 MemAddress &Addr) const {
2304 if (!Base.isReg())
2305 return;
2306
2307 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Base.getReg());
2308 if (!Def)
2309 return;
2310
2311 // Try V_ADD_U64_e64 pattern first (simpler, used on gfx1250+)
2312 if (Def->getOpcode() == AMDGPU::V_ADD_U64_e64) {
2313 if (processBaseWithConstOffset64(AddDef: Def, Base, Addr))
2314 return;
2315 }
2316
2317 // Fall through to REG_SEQUENCE + V_ADD_CO_U32 + V_ADDC_U32 pattern
2318 if (Def->getOpcode() != AMDGPU::REG_SEQUENCE || Def->getNumOperands() != 5)
2319 return;
2320
2321 MachineOperand BaseLo = Def->getOperand(i: 1);
2322 MachineOperand BaseHi = Def->getOperand(i: 3);
2323 if (!BaseLo.isReg() || !BaseHi.isReg())
2324 return;
2325
2326 MachineInstr *BaseLoDef = MRI->getUniqueVRegDef(Reg: BaseLo.getReg());
2327 MachineInstr *BaseHiDef = MRI->getUniqueVRegDef(Reg: BaseHi.getReg());
2328
2329 if (!BaseLoDef || BaseLoDef->getOpcode() != AMDGPU::V_ADD_CO_U32_e64 ||
2330 !BaseHiDef || BaseHiDef->getOpcode() != AMDGPU::V_ADDC_U32_e64)
2331 return;
2332
2333 MachineOperand *Src0 = TII->getNamedOperand(MI&: *BaseLoDef, OperandName: AMDGPU::OpName::src0);
2334 MachineOperand *Src1 = TII->getNamedOperand(MI&: *BaseLoDef, OperandName: AMDGPU::OpName::src1);
2335
2336 auto Offset0P = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
2337 if (Offset0P)
2338 BaseLo = *Src1;
2339 else {
2340 if (!(Offset0P = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1)))
2341 return;
2342 BaseLo = *Src0;
2343 }
2344
2345 if (!BaseLo.isReg())
2346 return;
2347
2348 Src0 = TII->getNamedOperand(MI&: *BaseHiDef, OperandName: AMDGPU::OpName::src0);
2349 Src1 = TII->getNamedOperand(MI&: *BaseHiDef, OperandName: AMDGPU::OpName::src1);
2350
2351 if (Src0->isImm())
2352 std::swap(a&: Src0, b&: Src1);
2353
2354 if (!Src1->isImm() || Src0->isImm())
2355 return;
2356
2357 uint64_t Offset1 = Src1->getImm();
2358 BaseHi = *Src0;
2359
2360 if (!BaseHi.isReg())
2361 return;
2362
2363 Addr.Base.LoReg = BaseLo.getReg();
2364 Addr.Base.HiReg = BaseHi.getReg();
2365 Addr.Base.LoSubReg = BaseLo.getSubReg();
2366 Addr.Base.HiSubReg = BaseHi.getSubReg();
2367 Addr.Offset = (*Offset0P & 0x00000000ffffffff) | (Offset1 << 32);
2368}
2369
2370// Maintain the correct LDS address for async loads and stores.
2371// It becomes incorrect when promoteConstantOffsetToImm adds an offset only
2372// meant for the global address operand. For async loads the LDS address is in
2373// vdst. For async stores, the LDS address is in vdata.
2374void SILoadStoreOptimizer::updateAsyncLDSAddress(MachineInstr &MI,
2375 int32_t OffsetDiff) const {
2376 if (!TII->usesASYNC_CNT(MI) || OffsetDiff == 0)
2377 return;
2378
2379 MachineOperand *LDSAddr = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
2380 if (!LDSAddr)
2381 LDSAddr = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
2382 assert(LDSAddr);
2383
2384 Register OldReg = LDSAddr->getReg();
2385 Register NewReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: OldReg));
2386 MachineBasicBlock &MBB = *MI.getParent();
2387 const DebugLoc &DL = MI.getDebugLoc();
2388 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg: NewReg)
2389 .addReg(RegNo: OldReg)
2390 .addImm(Val: -OffsetDiff)
2391 .addImm(Val: 0);
2392
2393 LDSAddr->setReg(NewReg);
2394}
2395
2396bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
2397 MachineInstr &MI,
2398 MemInfoMap &Visited,
2399 SmallPtrSet<MachineInstr *, 4> &AnchorList) const {
2400
2401 if (!STM->hasFlatInstOffsets() || !SIInstrInfo::isFLAT(MI))
2402 return false;
2403
2404 // TODO: Support FLAT_SCRATCH. Currently code expects 64-bit pointers.
2405 if (SIInstrInfo::isFLATScratch(MI))
2406 return false;
2407
2408 unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS
2409 : AMDGPUAS::FLAT_ADDRESS;
2410
2411 AMDGPU::FlatAddrSpace FlatVariant = AS == AMDGPUAS::GLOBAL_ADDRESS
2412 ? AMDGPU::FlatAddrSpace::FlatGlobal
2413 : AMDGPU::FlatAddrSpace::FLAT;
2414 bool AllowNegativeOffset =
2415 TII->allowNegativeFlatOffset(FlatVariant) && !TII->usesASYNC_CNT(MI);
2416 // The async global instructions use i24 offset for global address but u16
2417 // offset for LDS address. In this case, we just only promote when the offset
2418 // is u16.
2419 bool IsOffsetU16 = TII->usesASYNC_CNT(MI);
2420
2421 if (AnchorList.count(Ptr: &MI))
2422 return false;
2423
2424 LLVM_DEBUG(dbgs() << "\nTryToPromoteConstantOffsetToImmFor "; MI.dump());
2425
2426 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm()) {
2427 LLVM_DEBUG(dbgs() << " Const-offset is already promoted.\n";);
2428 return false;
2429 }
2430
2431 // Step1: Find the base-registers and a 64bit constant offset.
2432 MachineOperand &Base = *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
2433 auto [It, Inserted] = Visited.try_emplace(Key: &MI);
2434 MemAddress MAddr;
2435 if (Inserted) {
2436 processBaseWithConstOffset(Base, Addr&: MAddr);
2437 It->second = MAddr;
2438 } else
2439 MAddr = It->second;
2440
2441 if (MAddr.Offset == 0) {
2442 LLVM_DEBUG(dbgs() << " Failed to extract constant-offset or there are no"
2443 " constant offsets that can be promoted.\n";);
2444 return false;
2445 }
2446
2447 LLVM_DEBUG(dbgs() << " BASE: {" << printReg(MAddr.Base.HiReg, TRI) << ", "
2448 << printReg(MAddr.Base.LoReg, TRI)
2449 << "} Offset: " << MAddr.Offset << "\n\n";);
2450
2451 // Step2: Traverse through MI's basic block and find an anchor(that has the
2452 // same base-registers) with the highest 13bit distance from MI's offset.
2453 // E.g. (64bit loads)
2454 // bb:
2455 // addr1 = &a + 4096; load1 = load(addr1, 0)
2456 // addr2 = &a + 6144; load2 = load(addr2, 0)
2457 // addr3 = &a + 8192; load3 = load(addr3, 0)
2458 // addr4 = &a + 10240; load4 = load(addr4, 0)
2459 // addr5 = &a + 12288; load5 = load(addr5, 0)
2460 //
2461 // Starting from the first load, the optimization will try to find a new base
2462 // from which (&a + 4096) has 13 bit distance. Both &a + 6144 and &a + 8192
2463 // has 13bit distance from &a + 4096. The heuristic considers &a + 8192
2464 // as the new-base(anchor) because of the maximum distance which can
2465 // accommodate more intermediate bases presumably.
2466 //
2467 // Step3: move (&a + 8192) above load1. Compute and promote offsets from
2468 // (&a + 8192) for load1, load2, load4.
2469 // addr = &a + 8192
2470 // load1 = load(addr, -4096)
2471 // load2 = load(addr, -2048)
2472 // load3 = load(addr, 0)
2473 // load4 = load(addr, 2048)
2474 // addr5 = &a + 12288; load5 = load(addr5, 0)
2475 //
2476 MachineInstr *AnchorInst = nullptr;
2477 MemAddress AnchorAddr;
2478 uint32_t MaxDist = std::numeric_limits<uint32_t>::min();
2479 SmallVector<std::pair<MachineInstr *, int64_t>, 4> InstsWCommonBase;
2480 bool MIIsAnchor = false;
2481
2482 MachineBasicBlock *MBB = MI.getParent();
2483 MachineBasicBlock::iterator E = MBB->end();
2484 MachineBasicBlock::iterator MBBI = MI.getIterator();
2485 ++MBBI;
2486 const SITargetLowering *TLI = STM->getTargetLowering();
2487
2488 for ( ; MBBI != E; ++MBBI) {
2489 MachineInstr &MINext = *MBBI;
2490 // TODO: Support finding an anchor(with same base) from store addresses or
2491 // any other load addresses where the opcodes are different.
2492 if (MINext.getOpcode() != MI.getOpcode() ||
2493 TII->getNamedOperand(MI&: MINext, OperandName: AMDGPU::OpName::offset)->getImm())
2494 continue;
2495
2496 const MachineOperand &BaseNext =
2497 *TII->getNamedOperand(MI&: MINext, OperandName: AMDGPU::OpName::vaddr);
2498 MemAddress MAddrNext;
2499 auto [It, Inserted] = Visited.try_emplace(Key: &MINext);
2500 if (Inserted) {
2501 processBaseWithConstOffset(Base: BaseNext, Addr&: MAddrNext);
2502 It->second = MAddrNext;
2503 } else
2504 MAddrNext = It->second;
2505
2506 if (MAddrNext.Base.LoReg != MAddr.Base.LoReg ||
2507 MAddrNext.Base.HiReg != MAddr.Base.HiReg ||
2508 MAddrNext.Base.LoSubReg != MAddr.Base.LoSubReg ||
2509 MAddrNext.Base.HiSubReg != MAddr.Base.HiSubReg)
2510 continue;
2511
2512 InstsWCommonBase.emplace_back(Args: &MINext, Args&: MAddrNext.Offset);
2513
2514 if (AllowNegativeOffset) {
2515 int64_t Dist = MAddr.Offset - MAddrNext.Offset;
2516 TargetLoweringBase::AddrMode AM;
2517 AM.HasBaseReg = true;
2518 AM.BaseOffs = Dist;
2519 if (TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2520 (uint32_t)std::abs(i: Dist) > MaxDist) {
2521 MaxDist = std::abs(i: Dist);
2522
2523 AnchorAddr = MAddrNext;
2524 AnchorInst = &MINext;
2525 }
2526 }
2527 }
2528
2529 // When negative offsets are not allowed, pick the candidate with the smallest
2530 // offset as anchor so all promoted offsets are non-negative. If MI itself has
2531 // the smallest offset, MI becomes the reference point (MIIsAnchor).
2532 if (!AllowNegativeOffset && !InstsWCommonBase.empty()) {
2533 for (auto &[Inst, Offset] : InstsWCommonBase) {
2534 int64_t Dist = MAddr.Offset - Offset;
2535 TargetLoweringBase::AddrMode AM;
2536 AM.HasBaseReg = true;
2537 AM.BaseOffs = Dist;
2538 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2539 (!IsOffsetU16 || isUInt<16>(x: Dist)) &&
2540 (!AnchorInst || Offset < AnchorAddr.Offset)) {
2541 AnchorAddr = Visited[Inst];
2542 AnchorInst = Inst;
2543 }
2544 }
2545 if (!AnchorInst)
2546 MIIsAnchor = true;
2547 }
2548
2549 if (AnchorInst) {
2550 LLVM_DEBUG(dbgs() << " Anchor-Inst(with max-distance from Offset): ";
2551 AnchorInst->dump());
2552 LLVM_DEBUG(dbgs() << " Anchor-Offset from BASE: "
2553 << AnchorAddr.Offset << "\n\n");
2554
2555 // Instead of moving up, just re-compute anchor-instruction's base address.
2556 Register Base = computeBase(MI, Addr: AnchorAddr);
2557
2558 int32_t OffsetDiff = MAddr.Offset - AnchorAddr.Offset;
2559 updateBaseAndOffset(MI, NewBase: Base, NewOffset: OffsetDiff);
2560 updateAsyncLDSAddress(MI, OffsetDiff);
2561 LLVM_DEBUG(dbgs() << " After promotion: "; MI.dump(););
2562
2563 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2564 TargetLoweringBase::AddrMode AM;
2565 AM.HasBaseReg = true;
2566 AM.BaseOffs = OtherOffset - AnchorAddr.Offset;
2567
2568 if (TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2569 (AllowNegativeOffset || AM.BaseOffs >= 0) &&
2570 (!IsOffsetU16 || isUInt<16>(x: AM.BaseOffs))) {
2571 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
2572 OtherMI->dump());
2573 int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
2574 updateBaseAndOffset(MI&: *OtherMI, NewBase: Base, NewOffset: OtherOffsetDiff);
2575 updateAsyncLDSAddress(MI&: *OtherMI, OffsetDiff: OtherOffsetDiff);
2576 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2577 }
2578 }
2579 AnchorList.insert(Ptr: AnchorInst);
2580 return true;
2581 }
2582
2583 if (MIIsAnchor) {
2584 LLVM_DEBUG(dbgs() << " MI is anchor (smallest offset); promoting "
2585 "candidates relative to MI's base.\n");
2586
2587 Register Base = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)->getReg();
2588 bool AnyPromoted = false;
2589
2590 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2591 int64_t Dist = OtherOffset - MAddr.Offset;
2592 TargetLoweringBase::AddrMode AM;
2593 AM.HasBaseReg = true;
2594 AM.BaseOffs = Dist;
2595 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2596 (!IsOffsetU16 || isUInt<16>(x: Dist))) {
2597 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset << ")";
2598 OtherMI->dump());
2599 updateBaseAndOffset(MI&: *OtherMI, NewBase: Base, NewOffset: Dist);
2600 updateAsyncLDSAddress(MI&: *OtherMI, OffsetDiff: Dist);
2601 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2602 AnyPromoted = true;
2603 }
2604 }
2605
2606 if (AnyPromoted) {
2607 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)->setIsKill(false);
2608 AnchorList.insert(Ptr: &MI);
2609 return true;
2610 }
2611 }
2612
2613 return false;
2614}
2615
2616void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI,
2617 std::list<std::list<CombineInfo> > &MergeableInsts) const {
2618 for (std::list<CombineInfo> &AddrList : MergeableInsts) {
2619 if (AddrList.front().InstClass == CI.InstClass &&
2620 AddrList.front().hasSameBaseAddress(CI)) {
2621 AddrList.emplace_back(args: CI);
2622 return;
2623 }
2624 }
2625
2626 // Base address not found, so add a new list.
2627 MergeableInsts.emplace_back(args: 1, args: CI);
2628}
2629
2630std::pair<MachineBasicBlock::iterator, bool>
2631SILoadStoreOptimizer::collectMergeableInsts(
2632 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
2633 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
2634 std::list<std::list<CombineInfo>> &MergeableInsts) const {
2635 bool Modified = false;
2636
2637 // Sort potential mergeable instructions into lists. One list per base address.
2638 unsigned Order = 0;
2639 MachineBasicBlock::iterator BlockI = Begin;
2640 for (; BlockI != End; ++BlockI) {
2641 MachineInstr &MI = *BlockI;
2642
2643 // We run this before checking if an address is mergeable, because it can produce
2644 // better code even if the instructions aren't mergeable.
2645 if (promoteConstantOffsetToImm(MI, Visited, AnchorList))
2646 Modified = true;
2647
2648 // Treat volatile accesses, ordered accesses and unmodeled side effects as
2649 // barriers. We can look after this barrier for separate merges.
2650 if (MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects()) {
2651 LLVM_DEBUG(dbgs() << "Breaking search on barrier: " << MI);
2652
2653 // Search will resume after this instruction in a separate merge list.
2654 ++BlockI;
2655 break;
2656 }
2657
2658 const InstClassEnum InstClass = getInstClass(Opc: MI.getOpcode(), TII: *TII);
2659 if (InstClass == UNKNOWN)
2660 continue;
2661
2662 // Do not merge VMEM buffer instructions with "swizzled" bit set.
2663 int Swizzled =
2664 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::swz);
2665 if (Swizzled != -1 && MI.getOperand(i: Swizzled).getImm())
2666 continue;
2667
2668 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
2669 if (!STM->hasRelaxedTBufferOOBMode()) {
2670 LLVM_DEBUG(
2671 dbgs() << "Skip tbuffer combine: relaxed OOB mode not enabled\n");
2672 continue;
2673 }
2674
2675 const MachineOperand *Fmt =
2676 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::format);
2677 if (!AMDGPU::getGcnBufferFormatInfo(Format: Fmt->getImm(), STI: *STM)) {
2678 LLVM_DEBUG(dbgs() << "Skip tbuffer with unknown format: " << MI);
2679 continue;
2680 }
2681 } else if (InstClass == MIMG) {
2682 // Do not merge MIMG instructions with tfe or lwe enabled.
2683 // TFE/LWE add a status result that the image merge path does not model.
2684 const auto *TFEOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
2685 if (TFEOp && TFEOp->getImm())
2686 continue;
2687
2688 const auto *LWEOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
2689 if (LWEOp && LWEOp->getImm())
2690 continue;
2691 }
2692
2693 CombineInfo CI;
2694 CI.setMI(MI, LSO: *this);
2695 CI.Order = Order++;
2696
2697 if (!CI.hasMergeableAddress(MRI: *MRI))
2698 continue;
2699
2700 LLVM_DEBUG(dbgs() << "Mergeable: " << MI);
2701
2702 addInstToMergeableList(CI, MergeableInsts);
2703 }
2704
2705 // At this point we have lists of Mergeable instructions.
2706 //
2707 // Part 2: Sort lists by offset and then for each CombineInfo object in the
2708 // list try to find an instruction that can be merged with I. If an instruction
2709 // is found, it is stored in the Paired field. If no instructions are found, then
2710 // the CombineInfo object is deleted from the list.
2711
2712 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2713 E = MergeableInsts.end(); I != E;) {
2714
2715 std::list<CombineInfo> &MergeList = *I;
2716 if (MergeList.size() <= 1) {
2717 // This means we have found only one instruction with a given address
2718 // that can be merged, and we need at least 2 instructions to do a merge,
2719 // so this list can be discarded.
2720 I = MergeableInsts.erase(position: I);
2721 continue;
2722 }
2723
2724 // Sort the lists by offsets, this way mergeable instructions will be
2725 // adjacent to each other in the list, which will make it easier to find
2726 // matches.
2727 MergeList.sort(
2728 comp: [] (const CombineInfo &A, const CombineInfo &B) {
2729 return A.Offset < B.Offset;
2730 });
2731 ++I;
2732 }
2733
2734 return {BlockI, Modified};
2735}
2736
2737// Scan through looking for adjacent LDS operations with constant offsets from
2738// the same base register. We rely on the scheduler to do the hard work of
2739// clustering nearby loads, and assume these are all adjacent.
2740bool SILoadStoreOptimizer::optimizeBlock(
2741 std::list<std::list<CombineInfo> > &MergeableInsts) {
2742 bool Modified = false;
2743
2744 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2745 E = MergeableInsts.end(); I != E;) {
2746 std::list<CombineInfo> &MergeList = *I;
2747
2748 bool OptimizeListAgain = false;
2749 if (!optimizeInstsWithSameBaseAddr(MergeList, OptimizeListAgain)) {
2750 // We weren't able to make any changes, so delete the list so we don't
2751 // process the same instructions the next time we try to optimize this
2752 // block.
2753 I = MergeableInsts.erase(position: I);
2754 continue;
2755 }
2756
2757 Modified = true;
2758
2759 // We made changes, but also determined that there were no more optimization
2760 // opportunities, so we don't need to reprocess the list
2761 if (!OptimizeListAgain) {
2762 I = MergeableInsts.erase(position: I);
2763 continue;
2764 }
2765 OptimizeAgain = true;
2766 }
2767 return Modified;
2768}
2769
2770bool
2771SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr(
2772 std::list<CombineInfo> &MergeList,
2773 bool &OptimizeListAgain) {
2774 if (MergeList.empty())
2775 return false;
2776
2777 bool Modified = false;
2778
2779 for (auto I = MergeList.begin(), Next = std::next(x: I); Next != MergeList.end();
2780 Next = std::next(x: I)) {
2781
2782 auto First = I;
2783 auto Second = Next;
2784
2785 if ((*First).Order > (*Second).Order)
2786 std::swap(a&: First, b&: Second);
2787 CombineInfo &CI = *First;
2788 CombineInfo &Paired = *Second;
2789
2790 CombineInfo *Where = checkAndPrepareMerge(CI, Paired);
2791 if (!Where) {
2792 ++I;
2793 continue;
2794 }
2795
2796 Modified = true;
2797
2798 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I);
2799
2800 MachineBasicBlock::iterator NewMI;
2801 switch (CI.InstClass) {
2802 default:
2803 llvm_unreachable("unknown InstClass");
2804 break;
2805 case DS_READ:
2806 NewMI = mergeRead2Pair(CI, Paired, InsertBefore: Where->I);
2807 break;
2808 case DS_WRITE:
2809 NewMI = mergeWrite2Pair(CI, Paired, InsertBefore: Where->I);
2810 break;
2811 case S_BUFFER_LOAD_IMM:
2812 case S_BUFFER_LOAD_SGPR_IMM:
2813 case S_LOAD_IMM:
2814 NewMI = mergeSMemLoadImmPair(CI, Paired, InsertBefore: Where->I);
2815 OptimizeListAgain |= CI.Width + Paired.Width < 8;
2816 break;
2817 case BUFFER_LOAD:
2818 NewMI = mergeBufferLoadPair(CI, Paired, InsertBefore: Where->I);
2819 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2820 break;
2821 case BUFFER_STORE:
2822 NewMI = mergeBufferStorePair(CI, Paired, InsertBefore: Where->I);
2823 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2824 break;
2825 case MIMG:
2826 NewMI = mergeImagePair(CI, Paired, InsertBefore: Where->I);
2827 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2828 break;
2829 case TBUFFER_LOAD:
2830 NewMI = mergeTBufferLoadPair(CI, Paired, InsertBefore: Where->I);
2831 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2832 break;
2833 case TBUFFER_STORE:
2834 NewMI = mergeTBufferStorePair(CI, Paired, InsertBefore: Where->I);
2835 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2836 break;
2837 case FLAT_LOAD:
2838 case FLAT_LOAD_SADDR:
2839 case GLOBAL_LOAD:
2840 case GLOBAL_LOAD_SADDR:
2841 NewMI = mergeFlatLoadPair(CI, Paired, InsertBefore: Where->I);
2842 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2843 break;
2844 case FLAT_STORE:
2845 case FLAT_STORE_SADDR:
2846 case GLOBAL_STORE:
2847 case GLOBAL_STORE_SADDR:
2848 NewMI = mergeFlatStorePair(CI, Paired, InsertBefore: Where->I);
2849 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2850 break;
2851 }
2852 CI.setMI(MI: NewMI, LSO: *this);
2853 CI.Order = Where->Order;
2854 if (I == Second)
2855 I = Next;
2856
2857 MergeList.erase(position: Second);
2858 }
2859
2860 return Modified;
2861}
2862
2863bool SILoadStoreOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
2864 if (skipFunction(F: MF.getFunction()))
2865 return false;
2866 return SILoadStoreOptimizer(
2867 &getAnalysis<AAResultsWrapperPass>().getAAResults())
2868 .run(MF);
2869}
2870
2871bool SILoadStoreOptimizer::run(MachineFunction &MF) {
2872 this->MF = &MF;
2873 STM = &MF.getSubtarget<GCNSubtarget>();
2874 if (!STM->loadStoreOptEnabled())
2875 return false;
2876
2877 TII = STM->getInstrInfo();
2878 TRI = &TII->getRegisterInfo();
2879
2880 MRI = &MF.getRegInfo();
2881
2882 LLVM_DEBUG(dbgs() << "Running SILoadStoreOptimizer\n");
2883
2884 bool Modified = false;
2885
2886 // Contains the list of instructions for which constant offsets are being
2887 // promoted to the IMM. This is tracked for an entire block at time.
2888 SmallPtrSet<MachineInstr *, 4> AnchorList;
2889 MemInfoMap Visited;
2890
2891 for (MachineBasicBlock &MBB : MF) {
2892 MachineBasicBlock::iterator SectionEnd;
2893 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
2894 I = SectionEnd) {
2895 bool CollectModified;
2896 std::list<std::list<CombineInfo>> MergeableInsts;
2897
2898 // First pass: Collect list of all instructions we know how to merge in a
2899 // subset of the block.
2900 std::tie(args&: SectionEnd, args&: CollectModified) =
2901 collectMergeableInsts(Begin: I, End: E, Visited, AnchorList, MergeableInsts);
2902
2903 Modified |= CollectModified;
2904
2905 do {
2906 OptimizeAgain = false;
2907 Modified |= optimizeBlock(MergeableInsts);
2908 } while (OptimizeAgain);
2909 }
2910
2911 Visited.clear();
2912 AnchorList.clear();
2913 }
2914
2915 return Modified;
2916}
2917
2918PreservedAnalyses
2919SILoadStoreOptimizerPass::run(MachineFunction &MF,
2920 MachineFunctionAnalysisManager &MFAM) {
2921 MFPropsModifier _(*this, MF);
2922
2923 if (MF.getFunction().hasOptNone())
2924 return PreservedAnalyses::all();
2925
2926 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
2927 .getManager();
2928 AAResults &AA = FAM.getResult<AAManager>(IR&: MF.getFunction());
2929
2930 bool Changed = SILoadStoreOptimizer(&AA).run(MF);
2931 if (!Changed)
2932 return PreservedAnalyses::all();
2933
2934 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
2935 PA.preserveSet<CFGAnalyses>();
2936 return PA;
2937}
2938