1//===- SILoadStoreOptimizer.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass tries to fuse DS instructions with close by immediate offsets.
10// This will fuse operations such as
11// ds_read_b32 v0, v2 offset:16
12// ds_read_b32 v1, v2 offset:32
13// ==>
14// ds_read2_b32 v[0:1], v2, offset0:4 offset1:8
15//
16// The same is done for certain SMEM and VMEM opcodes, e.g.:
17// s_buffer_load_dword s4, s[0:3], 4
18// s_buffer_load_dword s5, s[0:3], 8
19// ==>
20// s_buffer_load_dwordx2 s[4:5], s[0:3], 4
21//
22// This pass also tries to promote constant offset to the immediate by
23// adjusting the base. It tries to use a base from the nearby instructions that
24// allows it to have a 13bit constant offset and then promotes the 13bit offset
25// to the immediate.
26// E.g.
27// s_movk_i32 s0, 0x1800
28// v_add_co_u32_e32 v0, vcc, s0, v2
29// v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
30//
31// s_movk_i32 s0, 0x1000
32// v_add_co_u32_e32 v5, vcc, s0, v2
33// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
34// global_load_dwordx2 v[5:6], v[5:6], off
35// global_load_dwordx2 v[0:1], v[0:1], off
36// =>
37// s_movk_i32 s0, 0x1000
38// v_add_co_u32_e32 v5, vcc, s0, v2
39// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
40// global_load_dwordx2 v[5:6], v[5:6], off
41// global_load_dwordx2 v[0:1], v[5:6], off offset:2048
42//
43// Future improvements:
44//
45// - This is currently missing stores of constants because loading
46// the constant into the data register is placed between the stores, although
47// this is arguably a scheduling problem.
48//
49// - Live interval recomputing seems inefficient. This currently only matches
50// one pair, and recomputes live intervals and moves on to the next pair. It
51// would be better to compute a list of all merges that need to occur.
52//
53// - With a list of instructions to process, we can also merge more. If a
54// cluster of loads have offsets that are too large to fit in the 8-bit
55// offsets, but are close enough to fit in the 8 bits, we can add to the base
56// pointer and use the new reduced offsets.
57//
58//===----------------------------------------------------------------------===//
59
60#include "SILoadStoreOptimizer.h"
61#include "AMDGPU.h"
62#include "GCNSubtarget.h"
63#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
64#include "SIDefines.h"
65#include "llvm/Analysis/AliasAnalysis.h"
66#include "llvm/CodeGen/MachineFunctionPass.h"
67#include "llvm/CodeGen/RegisterClassInfo.h"
68#include "llvm/InitializePasses.h"
69
70using namespace llvm;
71
72#define DEBUG_TYPE "si-load-store-opt"
73
74namespace {
75enum InstClassEnum {
76 UNKNOWN,
77 DS_READ,
78 DS_WRITE,
79 S_BUFFER_LOAD_IMM,
80 S_BUFFER_LOAD_SGPR_IMM,
81 S_LOAD_IMM,
82 BUFFER_LOAD,
83 BUFFER_STORE,
84 MIMG,
85 TBUFFER_LOAD,
86 TBUFFER_STORE,
87 GLOBAL_LOAD_SADDR,
88 GLOBAL_STORE_SADDR,
89 FLAT_LOAD,
90 FLAT_STORE,
91 FLAT_LOAD_SADDR,
92 FLAT_STORE_SADDR,
93 GLOBAL_LOAD, // GLOBAL_LOAD/GLOBAL_STORE are never used as the InstClass of
94 GLOBAL_STORE // any CombineInfo, they are only ever returned by
95 // getCommonInstClass.
96};
97
98struct AddressRegs {
99 unsigned char NumVAddrs = 0;
100 bool SBase = false;
101 bool SRsrc = false;
102 bool SOffset = false;
103 bool SAddr = false;
104 bool VAddr = false;
105 bool Addr = false;
106 bool SSamp = false;
107};
108
109// GFX10 image_sample instructions can have 12 vaddrs + srsrc + ssamp.
110const unsigned MaxAddressRegs = 12 + 1 + 1;
111
112class SILoadStoreOptimizer {
113 struct CombineInfo {
114 MachineBasicBlock::iterator I;
115 unsigned EltSize;
116 unsigned Offset;
117 unsigned Width;
118 unsigned Format;
119 unsigned BaseOff;
120 unsigned DMask;
121 InstClassEnum InstClass;
122 unsigned CPol = 0;
123 const TargetRegisterClass *DataRC;
124 bool UseST64;
125 int AddrIdx[MaxAddressRegs];
126 const MachineOperand *AddrReg[MaxAddressRegs];
127 unsigned NumAddresses;
128 unsigned Order;
129
130 bool hasSameBaseAddress(const CombineInfo &CI) {
131 if (NumAddresses != CI.NumAddresses)
132 return false;
133
134 const MachineInstr &MI = *CI.I;
135 for (unsigned i = 0; i < NumAddresses; i++) {
136 const MachineOperand &AddrRegNext = MI.getOperand(i: AddrIdx[i]);
137
138 if (AddrReg[i]->isImm() || AddrRegNext.isImm()) {
139 if (AddrReg[i]->isImm() != AddrRegNext.isImm() ||
140 AddrReg[i]->getImm() != AddrRegNext.getImm()) {
141 return false;
142 }
143 continue;
144 }
145
146 // Check same base pointer. Be careful of subregisters, which can occur
147 // with vectors of pointers.
148 if (AddrReg[i]->getReg() != AddrRegNext.getReg() ||
149 AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) {
150 return false;
151 }
152 }
153 return true;
154 }
155
156 bool hasMergeableAddress(const MachineRegisterInfo &MRI) {
157 for (unsigned i = 0; i < NumAddresses; ++i) {
158 const MachineOperand *AddrOp = AddrReg[i];
159 // Immediates are always OK.
160 if (AddrOp->isImm())
161 continue;
162
163 // Don't try to merge addresses that aren't either immediates or registers.
164 // TODO: Should be possible to merge FrameIndexes and maybe some other
165 // non-register
166 if (!AddrOp->isReg())
167 return false;
168
169 // TODO: We should be able to merge instructions with other physical reg
170 // addresses too.
171 if (AddrOp->getReg().isPhysical() &&
172 AddrOp->getReg() != AMDGPU::SGPR_NULL)
173 return false;
174
175 // If an address has only one use then there will be no other
176 // instructions with the same address, so we can't merge this one.
177 if (MRI.hasOneNonDBGUse(RegNo: AddrOp->getReg()))
178 return false;
179 }
180 return true;
181 }
182
183 void setMI(MachineBasicBlock::iterator MI, const SILoadStoreOptimizer &LSO);
184
185 // Compare by pointer order.
186 bool operator<(const CombineInfo& Other) const {
187 return (InstClass == MIMG) ? DMask < Other.DMask : Offset < Other.Offset;
188 }
189 };
190
191 struct BaseRegisters {
192 Register LoReg;
193 Register HiReg;
194
195 unsigned LoSubReg = 0;
196 unsigned HiSubReg = 0;
197 // True when using V_ADD_U64_e64 pattern
198 bool UseV64Pattern = false;
199 };
200
201 struct MemAddress {
202 BaseRegisters Base;
203 int64_t Offset = 0;
204 };
205
206 using MemInfoMap = DenseMap<MachineInstr *, MemAddress>;
207
208private:
209 MachineFunction *MF = nullptr;
210 const GCNSubtarget *STM = nullptr;
211 const SIInstrInfo *TII = nullptr;
212 const SIRegisterInfo *TRI = nullptr;
213 MachineRegisterInfo *MRI = nullptr;
214 AliasAnalysis *AA = nullptr;
215 bool OptimizeAgain;
216
217 bool canSwapInstructions(const DenseSet<Register> &ARegDefs,
218 const DenseSet<Register> &ARegUses,
219 const MachineInstr &A, const MachineInstr &B) const;
220 static bool dmasksCanBeCombined(const CombineInfo &CI,
221 const SIInstrInfo &TII,
222 const CombineInfo &Paired);
223 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
224 CombineInfo &Paired, bool Modify = false);
225 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
226 const CombineInfo &Paired);
227 unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
228 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
229 const CombineInfo &Paired);
230 const TargetRegisterClass *
231 getTargetRegisterClass(const CombineInfo &CI,
232 const CombineInfo &Paired) const;
233 const TargetRegisterClass *getDataRegClass(const MachineInstr &MI) const;
234
235 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
236
237 void copyToDestRegs(CombineInfo &CI, CombineInfo &Paired,
238 MachineBasicBlock::iterator InsertBefore,
239 const DebugLoc &DL, AMDGPU::OpName OpName,
240 Register DestReg) const;
241 Register copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
242 MachineBasicBlock::iterator InsertBefore,
243 const DebugLoc &DL, AMDGPU::OpName OpName) const;
244
245 unsigned read2Opcode(unsigned EltSize) const;
246 unsigned read2ST64Opcode(unsigned EltSize) const;
247 MachineBasicBlock::iterator
248 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
249 MachineBasicBlock::iterator InsertBefore);
250
251 unsigned write2Opcode(unsigned EltSize) const;
252 unsigned write2ST64Opcode(unsigned EltSize) const;
253 unsigned getWrite2Opcode(const CombineInfo &CI) const;
254
255 MachineBasicBlock::iterator
256 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
257 MachineBasicBlock::iterator InsertBefore);
258 MachineBasicBlock::iterator
259 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
260 MachineBasicBlock::iterator InsertBefore);
261 MachineBasicBlock::iterator
262 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
263 MachineBasicBlock::iterator InsertBefore);
264 MachineBasicBlock::iterator
265 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
266 MachineBasicBlock::iterator InsertBefore);
267 MachineBasicBlock::iterator
268 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
269 MachineBasicBlock::iterator InsertBefore);
270 MachineBasicBlock::iterator
271 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
272 MachineBasicBlock::iterator InsertBefore);
273 MachineBasicBlock::iterator
274 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
275 MachineBasicBlock::iterator InsertBefore);
276 MachineBasicBlock::iterator
277 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
278 MachineBasicBlock::iterator InsertBefore);
279 MachineBasicBlock::iterator
280 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
281 MachineBasicBlock::iterator InsertBefore);
282
283 void updateBaseAndOffset(MachineInstr &I, Register NewBase,
284 int32_t NewOffset) const;
285 void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
286 Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
287 MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
288 bool processBaseWithConstOffset64(MachineInstr *AddDef,
289 const MachineOperand &Base,
290 MemAddress &Addr) const;
291 void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
292 /// Promotes constant offset to the immediate by adjusting the base. It
293 /// tries to use a base from the nearby instructions that allows it to have
294 /// a 13bit constant offset which gets promoted to the immediate.
295 bool promoteConstantOffsetToImm(MachineInstr &CI,
296 MemInfoMap &Visited,
297 SmallPtrSet<MachineInstr *, 4> &Promoted) const;
298 void addInstToMergeableList(const CombineInfo &CI,
299 std::list<std::list<CombineInfo> > &MergeableInsts) const;
300
301 std::pair<MachineBasicBlock::iterator, bool> collectMergeableInsts(
302 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
303 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
304 std::list<std::list<CombineInfo>> &MergeableInsts) const;
305
306 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
307 const CombineInfo &Paired);
308
309 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
310 const CombineInfo &Paired);
311
312 bool optimizeInstsWithSameBaseAddr(std::list<CombineInfo> &MergeList,
313 bool &OptimizeListAgain);
314 bool optimizeBlock(std::list<std::list<CombineInfo> > &MergeableInsts);
315
316public:
317 SILoadStoreOptimizer(AliasAnalysis *AA) : AA(AA) {}
318 bool run(MachineFunction &MF);
319};
320
321class SILoadStoreOptimizerLegacy : public MachineFunctionPass {
322public:
323 static char ID;
324
325 SILoadStoreOptimizerLegacy() : MachineFunctionPass(ID) {}
326
327 bool runOnMachineFunction(MachineFunction &MF) override;
328
329 StringRef getPassName() const override { return "SI Load Store Optimizer"; }
330
331 void getAnalysisUsage(AnalysisUsage &AU) const override {
332 AU.setPreservesCFG();
333 AU.addRequired<AAResultsWrapperPass>();
334
335 MachineFunctionPass::getAnalysisUsage(AU);
336 }
337
338 MachineFunctionProperties getRequiredProperties() const override {
339 return MachineFunctionProperties().setIsSSA();
340 }
341};
342
343static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
344 const unsigned Opc = MI.getOpcode();
345
346 if (TII.isMUBUF(Opcode: Opc)) {
347 // FIXME: Handle d16 correctly
348 return AMDGPU::getMUBUFElements(Opc);
349 }
350 if (TII.isImage(MI)) {
351 uint64_t DMaskImm =
352 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask)->getImm();
353 return llvm::popcount(Value: DMaskImm);
354 }
355 if (TII.isMTBUF(Opcode: Opc)) {
356 return AMDGPU::getMTBUFElements(Opc);
357 }
358
359 switch (Opc) {
360 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
361 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
362 case AMDGPU::S_LOAD_DWORD_IMM:
363 case AMDGPU::GLOBAL_LOAD_DWORD:
364 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
365 case AMDGPU::GLOBAL_STORE_DWORD:
366 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
367 case AMDGPU::FLAT_LOAD_DWORD:
368 case AMDGPU::FLAT_STORE_DWORD:
369 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
370 case AMDGPU::FLAT_STORE_DWORD_SADDR:
371 return 1;
372 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
373 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
374 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
375 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
376 case AMDGPU::S_LOAD_DWORDX2_IMM:
377 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
378 case AMDGPU::GLOBAL_LOAD_DWORDX2:
379 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
380 case AMDGPU::GLOBAL_STORE_DWORDX2:
381 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
382 case AMDGPU::FLAT_LOAD_DWORDX2:
383 case AMDGPU::FLAT_STORE_DWORDX2:
384 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
385 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
386 return 2;
387 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
388 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
389 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
390 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
391 case AMDGPU::S_LOAD_DWORDX3_IMM:
392 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
393 case AMDGPU::GLOBAL_LOAD_DWORDX3:
394 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
395 case AMDGPU::GLOBAL_STORE_DWORDX3:
396 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
397 case AMDGPU::FLAT_LOAD_DWORDX3:
398 case AMDGPU::FLAT_STORE_DWORDX3:
399 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
400 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
401 return 3;
402 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
403 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
404 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
405 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
406 case AMDGPU::S_LOAD_DWORDX4_IMM:
407 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
408 case AMDGPU::GLOBAL_LOAD_DWORDX4:
409 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
410 case AMDGPU::GLOBAL_STORE_DWORDX4:
411 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
412 case AMDGPU::FLAT_LOAD_DWORDX4:
413 case AMDGPU::FLAT_STORE_DWORDX4:
414 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
415 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
416 return 4;
417 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
418 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
419 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
420 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
421 case AMDGPU::S_LOAD_DWORDX8_IMM:
422 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
423 return 8;
424 case AMDGPU::DS_READ_B32:
425 case AMDGPU::DS_READ_B32_gfx9:
426 case AMDGPU::DS_WRITE_B32:
427 case AMDGPU::DS_WRITE_B32_gfx9:
428 return 1;
429 case AMDGPU::DS_READ_B64:
430 case AMDGPU::DS_READ_B64_gfx9:
431 case AMDGPU::DS_WRITE_B64:
432 case AMDGPU::DS_WRITE_B64_gfx9:
433 return 2;
434 default:
435 return 0;
436 }
437}
438
439/// Maps instruction opcode to enum InstClassEnum.
440static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
441 switch (Opc) {
442 default:
443 if (TII.isMUBUF(Opcode: Opc)) {
444 switch (AMDGPU::getMUBUFBaseOpcode(Opc)) {
445 default:
446 return UNKNOWN;
447 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN:
448 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN_exact:
449 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN:
450 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN_exact:
451 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
452 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
453 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
454 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
455 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:
456 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact:
457 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN:
458 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact:
459 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN:
460 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN_exact:
461 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET:
462 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET_exact:
463 return BUFFER_LOAD;
464 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN:
465 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN_exact:
466 case AMDGPU::BUFFER_STORE_DWORD_IDXEN:
467 case AMDGPU::BUFFER_STORE_DWORD_IDXEN_exact:
468 case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
469 case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
470 case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
471 case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
472 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN:
473 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact:
474 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN:
475 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN_exact:
476 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN:
477 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact:
478 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET:
479 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact:
480 return BUFFER_STORE;
481 }
482 }
483 if (TII.isImage(Opcode: Opc)) {
484 // Ignore instructions encoded without vaddr.
485 if (!AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
486 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr0))
487 return UNKNOWN;
488 // Ignore BVH instructions
489 if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH)
490 return UNKNOWN;
491 // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD.
492 if (TII.get(Opcode: Opc).mayStore() || !TII.get(Opcode: Opc).mayLoad() ||
493 TII.isGather4(Opcode: Opc))
494 return UNKNOWN;
495 return MIMG;
496 }
497 if (TII.isMTBUF(Opcode: Opc)) {
498 switch (AMDGPU::getMTBUFBaseOpcode(Opc)) {
499 default:
500 return UNKNOWN;
501 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN:
502 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN_exact:
503 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN:
504 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN_exact:
505 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN:
506 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN_exact:
507 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET:
508 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET_exact:
509 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN:
510 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN_exact:
511 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN:
512 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN_exact:
513 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN:
514 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN_exact:
515 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET:
516 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET_exact:
517 return TBUFFER_LOAD;
518 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN:
519 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN_exact:
520 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET:
521 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET_exact:
522 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN:
523 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact:
524 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET:
525 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET_exact:
526 return TBUFFER_STORE;
527 }
528 }
529 return UNKNOWN;
530 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
531 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
532 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
533 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
534 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
535 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
536 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
537 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
538 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
539 return S_BUFFER_LOAD_IMM;
540 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
541 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
542 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
543 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
544 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
545 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
546 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
547 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
548 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
549 return S_BUFFER_LOAD_SGPR_IMM;
550 case AMDGPU::S_LOAD_DWORD_IMM:
551 case AMDGPU::S_LOAD_DWORDX2_IMM:
552 case AMDGPU::S_LOAD_DWORDX3_IMM:
553 case AMDGPU::S_LOAD_DWORDX4_IMM:
554 case AMDGPU::S_LOAD_DWORDX8_IMM:
555 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
556 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
557 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
558 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
559 return S_LOAD_IMM;
560 case AMDGPU::DS_READ_B32:
561 case AMDGPU::DS_READ_B32_gfx9:
562 case AMDGPU::DS_READ_B64:
563 case AMDGPU::DS_READ_B64_gfx9:
564 return DS_READ;
565 case AMDGPU::DS_WRITE_B32:
566 case AMDGPU::DS_WRITE_B32_gfx9:
567 case AMDGPU::DS_WRITE_B64:
568 case AMDGPU::DS_WRITE_B64_gfx9:
569 return DS_WRITE;
570 case AMDGPU::GLOBAL_LOAD_DWORD:
571 case AMDGPU::GLOBAL_LOAD_DWORDX2:
572 case AMDGPU::GLOBAL_LOAD_DWORDX3:
573 case AMDGPU::GLOBAL_LOAD_DWORDX4:
574 case AMDGPU::FLAT_LOAD_DWORD:
575 case AMDGPU::FLAT_LOAD_DWORDX2:
576 case AMDGPU::FLAT_LOAD_DWORDX3:
577 case AMDGPU::FLAT_LOAD_DWORDX4:
578 return FLAT_LOAD;
579 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
580 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
581 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
582 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
583 return GLOBAL_LOAD_SADDR;
584 case AMDGPU::GLOBAL_STORE_DWORD:
585 case AMDGPU::GLOBAL_STORE_DWORDX2:
586 case AMDGPU::GLOBAL_STORE_DWORDX3:
587 case AMDGPU::GLOBAL_STORE_DWORDX4:
588 case AMDGPU::FLAT_STORE_DWORD:
589 case AMDGPU::FLAT_STORE_DWORDX2:
590 case AMDGPU::FLAT_STORE_DWORDX3:
591 case AMDGPU::FLAT_STORE_DWORDX4:
592 return FLAT_STORE;
593 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
594 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
595 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
596 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
597 return GLOBAL_STORE_SADDR;
598 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
599 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
600 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
601 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
602 return FLAT_LOAD_SADDR;
603 case AMDGPU::FLAT_STORE_DWORD_SADDR:
604 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
605 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
606 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
607 return FLAT_STORE_SADDR;
608 }
609}
610
611/// Determines instruction subclass from opcode. Only instructions
612/// of the same subclass can be merged together. The merged instruction may have
613/// a different subclass but must have the same class.
614static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
615 switch (Opc) {
616 default:
617 if (TII.isMUBUF(Opcode: Opc))
618 return AMDGPU::getMUBUFBaseOpcode(Opc);
619 if (TII.isImage(Opcode: Opc)) {
620 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
621 assert(Info);
622 return Info->BaseOpcode;
623 }
624 if (TII.isMTBUF(Opcode: Opc))
625 return AMDGPU::getMTBUFBaseOpcode(Opc);
626 return -1;
627 case AMDGPU::DS_READ_B32:
628 case AMDGPU::DS_READ_B32_gfx9:
629 case AMDGPU::DS_READ_B64:
630 case AMDGPU::DS_READ_B64_gfx9:
631 case AMDGPU::DS_WRITE_B32:
632 case AMDGPU::DS_WRITE_B32_gfx9:
633 case AMDGPU::DS_WRITE_B64:
634 case AMDGPU::DS_WRITE_B64_gfx9:
635 return Opc;
636 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
637 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
638 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
639 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
640 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
641 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
642 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
643 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
644 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
645 return AMDGPU::S_BUFFER_LOAD_DWORD_IMM;
646 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
647 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
648 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
649 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
650 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
651 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
652 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
653 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
654 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
655 return AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM;
656 case AMDGPU::S_LOAD_DWORD_IMM:
657 case AMDGPU::S_LOAD_DWORDX2_IMM:
658 case AMDGPU::S_LOAD_DWORDX3_IMM:
659 case AMDGPU::S_LOAD_DWORDX4_IMM:
660 case AMDGPU::S_LOAD_DWORDX8_IMM:
661 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
662 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
663 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
664 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
665 return AMDGPU::S_LOAD_DWORD_IMM;
666 case AMDGPU::GLOBAL_LOAD_DWORD:
667 case AMDGPU::GLOBAL_LOAD_DWORDX2:
668 case AMDGPU::GLOBAL_LOAD_DWORDX3:
669 case AMDGPU::GLOBAL_LOAD_DWORDX4:
670 case AMDGPU::FLAT_LOAD_DWORD:
671 case AMDGPU::FLAT_LOAD_DWORDX2:
672 case AMDGPU::FLAT_LOAD_DWORDX3:
673 case AMDGPU::FLAT_LOAD_DWORDX4:
674 return AMDGPU::FLAT_LOAD_DWORD;
675 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
676 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
677 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
678 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
679 return AMDGPU::GLOBAL_LOAD_DWORD_SADDR;
680 case AMDGPU::GLOBAL_STORE_DWORD:
681 case AMDGPU::GLOBAL_STORE_DWORDX2:
682 case AMDGPU::GLOBAL_STORE_DWORDX3:
683 case AMDGPU::GLOBAL_STORE_DWORDX4:
684 case AMDGPU::FLAT_STORE_DWORD:
685 case AMDGPU::FLAT_STORE_DWORDX2:
686 case AMDGPU::FLAT_STORE_DWORDX3:
687 case AMDGPU::FLAT_STORE_DWORDX4:
688 return AMDGPU::FLAT_STORE_DWORD;
689 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
690 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
691 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
692 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
693 return AMDGPU::GLOBAL_STORE_DWORD_SADDR;
694 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
695 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
696 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
697 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
698 return AMDGPU::FLAT_LOAD_DWORD_SADDR;
699 case AMDGPU::FLAT_STORE_DWORD_SADDR:
700 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
701 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
702 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
703 return AMDGPU::FLAT_STORE_DWORD_SADDR;
704 }
705}
706
707// GLOBAL loads and stores are classified as FLAT initially. If both combined
708// instructions are FLAT GLOBAL adjust the class to GLOBAL_LOAD or GLOBAL_STORE.
709// If either or both instructions are non segment specific FLAT the resulting
710// combined operation will be FLAT, potentially promoting one of the GLOBAL
711// operations to FLAT.
712// For other instructions return the original unmodified class.
713InstClassEnum
714SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI,
715 const CombineInfo &Paired) {
716 assert(CI.InstClass == Paired.InstClass);
717
718 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) &&
719 SIInstrInfo::isFLATGlobal(MI: *CI.I) && SIInstrInfo::isFLATGlobal(MI: *Paired.I))
720 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD;
721
722 return CI.InstClass;
723}
724
725static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
726 AddressRegs Result;
727
728 if (TII.isMUBUF(Opcode: Opc)) {
729 if (AMDGPU::getMUBUFHasVAddr(Opc))
730 Result.VAddr = true;
731 if (AMDGPU::getMUBUFHasSrsrc(Opc))
732 Result.SRsrc = true;
733 if (AMDGPU::getMUBUFHasSoffset(Opc))
734 Result.SOffset = true;
735
736 return Result;
737 }
738
739 if (TII.isImage(Opcode: Opc)) {
740 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
741 if (VAddr0Idx >= 0) {
742 AMDGPU::OpName RsrcName =
743 TII.isMIMG(Opcode: Opc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
744 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcName);
745 Result.NumVAddrs = RsrcIdx - VAddr0Idx;
746 } else {
747 Result.VAddr = true;
748 }
749 Result.SRsrc = true;
750 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
751 if (Info && AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode)->Sampler)
752 Result.SSamp = true;
753
754 return Result;
755 }
756 if (TII.isMTBUF(Opcode: Opc)) {
757 if (AMDGPU::getMTBUFHasVAddr(Opc))
758 Result.VAddr = true;
759 if (AMDGPU::getMTBUFHasSrsrc(Opc))
760 Result.SRsrc = true;
761 if (AMDGPU::getMTBUFHasSoffset(Opc))
762 Result.SOffset = true;
763
764 return Result;
765 }
766
767 switch (Opc) {
768 default:
769 return Result;
770 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
771 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
772 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
773 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
774 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
775 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
776 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
777 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
778 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
779 Result.SOffset = true;
780 [[fallthrough]];
781 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
782 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
783 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
784 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
785 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
786 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
787 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
788 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
789 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
790 case AMDGPU::S_LOAD_DWORD_IMM:
791 case AMDGPU::S_LOAD_DWORDX2_IMM:
792 case AMDGPU::S_LOAD_DWORDX3_IMM:
793 case AMDGPU::S_LOAD_DWORDX4_IMM:
794 case AMDGPU::S_LOAD_DWORDX8_IMM:
795 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
796 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
797 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
798 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
799 Result.SBase = true;
800 return Result;
801 case AMDGPU::DS_READ_B32:
802 case AMDGPU::DS_READ_B64:
803 case AMDGPU::DS_READ_B32_gfx9:
804 case AMDGPU::DS_READ_B64_gfx9:
805 case AMDGPU::DS_WRITE_B32:
806 case AMDGPU::DS_WRITE_B64:
807 case AMDGPU::DS_WRITE_B32_gfx9:
808 case AMDGPU::DS_WRITE_B64_gfx9:
809 Result.Addr = true;
810 return Result;
811 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
812 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
813 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
814 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
815 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
816 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
817 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
818 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
819 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
820 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
821 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
822 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
823 case AMDGPU::FLAT_STORE_DWORD_SADDR:
824 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
825 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
826 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
827 Result.SAddr = true;
828 [[fallthrough]];
829 case AMDGPU::GLOBAL_LOAD_DWORD:
830 case AMDGPU::GLOBAL_LOAD_DWORDX2:
831 case AMDGPU::GLOBAL_LOAD_DWORDX3:
832 case AMDGPU::GLOBAL_LOAD_DWORDX4:
833 case AMDGPU::GLOBAL_STORE_DWORD:
834 case AMDGPU::GLOBAL_STORE_DWORDX2:
835 case AMDGPU::GLOBAL_STORE_DWORDX3:
836 case AMDGPU::GLOBAL_STORE_DWORDX4:
837 case AMDGPU::FLAT_LOAD_DWORD:
838 case AMDGPU::FLAT_LOAD_DWORDX2:
839 case AMDGPU::FLAT_LOAD_DWORDX3:
840 case AMDGPU::FLAT_LOAD_DWORDX4:
841 case AMDGPU::FLAT_STORE_DWORD:
842 case AMDGPU::FLAT_STORE_DWORDX2:
843 case AMDGPU::FLAT_STORE_DWORDX3:
844 case AMDGPU::FLAT_STORE_DWORDX4:
845 Result.VAddr = true;
846 return Result;
847 }
848}
849
850void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
851 const SILoadStoreOptimizer &LSO) {
852 I = MI;
853 unsigned Opc = MI->getOpcode();
854 InstClass = getInstClass(Opc, TII: *LSO.TII);
855
856 if (InstClass == UNKNOWN)
857 return;
858
859 DataRC = LSO.getDataRegClass(MI: *MI);
860
861 switch (InstClass) {
862 case DS_READ:
863 EltSize =
864 (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8
865 : 4;
866 break;
867 case DS_WRITE:
868 EltSize =
869 (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8
870 : 4;
871 break;
872 case S_BUFFER_LOAD_IMM:
873 case S_BUFFER_LOAD_SGPR_IMM:
874 case S_LOAD_IMM:
875 EltSize = AMDGPU::convertSMRDOffsetUnits(ST: *LSO.STM, ByteOffset: 4);
876 break;
877 default:
878 EltSize = 4;
879 break;
880 }
881
882 if (InstClass == MIMG) {
883 DMask = LSO.TII->getNamedOperand(MI&: *I, OperandName: AMDGPU::OpName::dmask)->getImm();
884 // Offset is not considered for MIMG instructions.
885 Offset = 0;
886 } else {
887 int OffsetIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::offset);
888 Offset = I->getOperand(i: OffsetIdx).getImm();
889 }
890
891 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
892 Format = LSO.TII->getNamedOperand(MI&: *I, OperandName: AMDGPU::OpName::format)->getImm();
893 const AMDGPU::GcnBufferFormatInfo *Info =
894 AMDGPU::getGcnBufferFormatInfo(Format, STI: *LSO.STM);
895 EltSize = Info->BitsPerComp / 8;
896 }
897
898 Width = getOpcodeWidth(MI: *I, TII: *LSO.TII);
899
900 if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
901 Offset &= 0xffff;
902 } else if (InstClass != MIMG) {
903 CPol = LSO.TII->getNamedOperand(MI&: *I, OperandName: AMDGPU::OpName::cpol)->getImm();
904 }
905
906 AddressRegs Regs = getRegs(Opc, TII: *LSO.TII);
907 bool isVIMAGEorVSAMPLE = LSO.TII->isVIMAGE(MI: *I) || LSO.TII->isVSAMPLE(MI: *I);
908
909 NumAddresses = 0;
910 for (unsigned J = 0; J < Regs.NumVAddrs; J++)
911 AddrIdx[NumAddresses++] =
912 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0) + J;
913 if (Regs.Addr)
914 AddrIdx[NumAddresses++] =
915 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::addr);
916 if (Regs.SBase)
917 AddrIdx[NumAddresses++] =
918 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sbase);
919 if (Regs.SRsrc)
920 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
921 Opcode: Opc, Name: isVIMAGEorVSAMPLE ? AMDGPU::OpName::rsrc : AMDGPU::OpName::srsrc);
922 if (Regs.SOffset)
923 AddrIdx[NumAddresses++] =
924 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::soffset);
925 if (Regs.SAddr)
926 AddrIdx[NumAddresses++] =
927 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
928 if (Regs.VAddr)
929 AddrIdx[NumAddresses++] =
930 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
931 if (Regs.SSamp)
932 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
933 Opcode: Opc, Name: isVIMAGEorVSAMPLE ? AMDGPU::OpName::samp : AMDGPU::OpName::ssamp);
934 assert(NumAddresses <= MaxAddressRegs);
935
936 for (unsigned J = 0; J < NumAddresses; J++)
937 AddrReg[J] = &I->getOperand(i: AddrIdx[J]);
938}
939
940} // end anonymous namespace.
941
942INITIALIZE_PASS_BEGIN(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
943 "SI Load Store Optimizer", false, false)
944INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
945INITIALIZE_PASS_END(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
946 "SI Load Store Optimizer", false, false)
947
948char SILoadStoreOptimizerLegacy::ID = 0;
949
950char &llvm::SILoadStoreOptimizerLegacyID = SILoadStoreOptimizerLegacy::ID;
951
952FunctionPass *llvm::createSILoadStoreOptimizerLegacyPass() {
953 return new SILoadStoreOptimizerLegacy();
954}
955
956static void addDefsUsesToList(const MachineInstr &MI,
957 DenseSet<Register> &RegDefs,
958 DenseSet<Register> &RegUses) {
959 for (const auto &Op : MI.operands()) {
960 if (!Op.isReg())
961 continue;
962 if (Op.isDef())
963 RegDefs.insert(V: Op.getReg());
964 if (Op.readsReg())
965 RegUses.insert(V: Op.getReg());
966 }
967}
968
969bool SILoadStoreOptimizer::canSwapInstructions(
970 const DenseSet<Register> &ARegDefs, const DenseSet<Register> &ARegUses,
971 const MachineInstr &A, const MachineInstr &B) const {
972 if (A.mayLoadOrStore() && B.mayLoadOrStore() &&
973 (A.mayStore() || B.mayStore()) && A.mayAlias(AA, Other: B, UseTBAA: true))
974 return false;
975 for (const auto &BOp : B.operands()) {
976 if (!BOp.isReg())
977 continue;
978 if ((BOp.isDef() || BOp.readsReg()) && ARegDefs.contains(V: BOp.getReg()))
979 return false;
980 if (BOp.isDef() && ARegUses.contains(V: BOp.getReg()))
981 return false;
982 }
983 return true;
984}
985
986// Given that \p CI and \p Paired are adjacent memory operations produce a new
987// MMO for the combined operation with a new access size.
988MachineMemOperand *
989SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI,
990 const CombineInfo &Paired) {
991 const MachineMemOperand *MMOa = *CI.I->memoperands_begin();
992 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin();
993
994 unsigned Size = MMOa->getSize().getValue() + MMOb->getSize().getValue();
995
996 // A base pointer for the combined operation is the same as the leading
997 // operation's pointer.
998 if (Paired < CI)
999 std::swap(a&: MMOa, b&: MMOb);
1000
1001 MachinePointerInfo PtrInfo(MMOa->getPointerInfo());
1002 // If merging FLAT and GLOBAL set address space to FLAT.
1003 if (MMOb->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
1004 PtrInfo.AddrSpace = AMDGPUAS::FLAT_ADDRESS;
1005
1006 MachineFunction *MF = CI.I->getMF();
1007 return MF->getMachineMemOperand(MMO: MMOa, PtrInfo, Size);
1008}
1009
1010bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI,
1011 const SIInstrInfo &TII,
1012 const CombineInfo &Paired) {
1013 assert(CI.InstClass == MIMG);
1014
1015 // Ignore instructions with tfe/lwe set.
1016 const auto *TFEOp = TII.getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::tfe);
1017 const auto *LWEOp = TII.getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::lwe);
1018
1019 if ((TFEOp && TFEOp->getImm()) || (LWEOp && LWEOp->getImm()))
1020 return false;
1021
1022 // Check other optional immediate operands for equality.
1023 AMDGPU::OpName OperandsToMatch[] = {
1024 AMDGPU::OpName::cpol, AMDGPU::OpName::d16, AMDGPU::OpName::unorm,
1025 AMDGPU::OpName::da, AMDGPU::OpName::r128, AMDGPU::OpName::a16};
1026
1027 for (AMDGPU::OpName op : OperandsToMatch) {
1028 int Idx = AMDGPU::getNamedOperandIdx(Opcode: CI.I->getOpcode(), Name: op);
1029 if (AMDGPU::getNamedOperandIdx(Opcode: Paired.I->getOpcode(), Name: op) != Idx)
1030 return false;
1031 if (Idx != -1 &&
1032 CI.I->getOperand(i: Idx).getImm() != Paired.I->getOperand(i: Idx).getImm())
1033 return false;
1034 }
1035
1036 // Check DMask for overlaps.
1037 unsigned MaxMask = std::max(a: CI.DMask, b: Paired.DMask);
1038 unsigned MinMask = std::min(a: CI.DMask, b: Paired.DMask);
1039
1040 if (!MaxMask)
1041 return false;
1042
1043 unsigned AllowedBitsForMin = llvm::countr_zero(Val: MaxMask);
1044 if ((1u << AllowedBitsForMin) <= MinMask)
1045 return false;
1046
1047 return true;
1048}
1049
1050static unsigned getBufferFormatWithCompCount(unsigned OldFormat,
1051 unsigned ComponentCount,
1052 const GCNSubtarget &STI) {
1053 if (ComponentCount > 4)
1054 return 0;
1055
1056 const llvm::AMDGPU::GcnBufferFormatInfo *OldFormatInfo =
1057 llvm::AMDGPU::getGcnBufferFormatInfo(Format: OldFormat, STI);
1058 if (!OldFormatInfo)
1059 return 0;
1060
1061 const llvm::AMDGPU::GcnBufferFormatInfo *NewFormatInfo =
1062 llvm::AMDGPU::getGcnBufferFormatInfo(BitsPerComp: OldFormatInfo->BitsPerComp,
1063 NumComponents: ComponentCount,
1064 NumFormat: OldFormatInfo->NumFormat, STI);
1065
1066 if (!NewFormatInfo)
1067 return 0;
1068
1069 assert(NewFormatInfo->NumFormat == OldFormatInfo->NumFormat &&
1070 NewFormatInfo->BitsPerComp == OldFormatInfo->BitsPerComp);
1071
1072 return NewFormatInfo->Format;
1073}
1074
1075// Return the value in the inclusive range [Lo,Hi] that is aligned to the
1076// highest power of two. Note that the result is well defined for all inputs
1077// including corner cases like:
1078// - if Lo == Hi, return that value
1079// - if Lo == 0, return 0 (even though the "- 1" below underflows
1080// - if Lo > Hi, return 0 (as if the range wrapped around)
1081static uint32_t mostAlignedValueInRange(uint32_t Lo, uint32_t Hi) {
1082 return Hi & maskLeadingOnes<uint32_t>(N: llvm::countl_zero(Val: (Lo - 1) ^ Hi) + 1);
1083}
1084
1085bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI,
1086 const GCNSubtarget &STI,
1087 CombineInfo &Paired,
1088 bool Modify) {
1089 assert(CI.InstClass != MIMG);
1090
1091 // XXX - Would the same offset be OK? Is there any reason this would happen or
1092 // be useful?
1093 if (CI.Offset == Paired.Offset)
1094 return false;
1095
1096 // This won't be valid if the offset isn't aligned.
1097 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0))
1098 return false;
1099
1100 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) {
1101
1102 const llvm::AMDGPU::GcnBufferFormatInfo *Info0 =
1103 llvm::AMDGPU::getGcnBufferFormatInfo(Format: CI.Format, STI);
1104 const llvm::AMDGPU::GcnBufferFormatInfo *Info1 =
1105 llvm::AMDGPU::getGcnBufferFormatInfo(Format: Paired.Format, STI);
1106
1107 if (Info0->BitsPerComp != Info1->BitsPerComp ||
1108 Info0->NumFormat != Info1->NumFormat)
1109 return false;
1110
1111 // For 8-bit or 16-bit formats there is no 3-component variant.
1112 // If NumCombinedComponents is 3, try the 4-component format and use XYZ.
1113 // Example:
1114 // tbuffer_load_format_x + tbuffer_load_format_x + tbuffer_load_format_x
1115 // ==> tbuffer_load_format_xyz with format:[BUF_FMT_16_16_16_16_SNORM]
1116 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1117 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1118 NumCombinedComponents = 4;
1119
1120 if (getBufferFormatWithCompCount(OldFormat: CI.Format, ComponentCount: NumCombinedComponents, STI) ==
1121 0)
1122 return false;
1123
1124 // Merge only when the two access ranges are strictly back-to-back,
1125 // any gap or overlap can over-write data or leave holes.
1126 unsigned ElemIndex0 = CI.Offset / CI.EltSize;
1127 unsigned ElemIndex1 = Paired.Offset / Paired.EltSize;
1128 if (ElemIndex0 + CI.Width != ElemIndex1 &&
1129 ElemIndex1 + Paired.Width != ElemIndex0)
1130 return false;
1131
1132 // 1-byte formats require 1-byte alignment.
1133 // 2-byte formats require 2-byte alignment.
1134 // 4-byte and larger formats require 4-byte alignment.
1135 unsigned MergedBytes = CI.EltSize * NumCombinedComponents;
1136 unsigned RequiredAlign = std::min(a: MergedBytes, b: 4u);
1137 unsigned MinOff = std::min(a: CI.Offset, b: Paired.Offset);
1138 if (MinOff % RequiredAlign != 0)
1139 return false;
1140
1141 return true;
1142 }
1143
1144 uint32_t EltOffset0 = CI.Offset / CI.EltSize;
1145 uint32_t EltOffset1 = Paired.Offset / CI.EltSize;
1146 CI.UseST64 = false;
1147 CI.BaseOff = 0;
1148
1149 // Handle all non-DS instructions.
1150 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) {
1151 if (EltOffset0 + CI.Width != EltOffset1 &&
1152 EltOffset1 + Paired.Width != EltOffset0)
1153 return false;
1154 // Instructions with scale_offset modifier cannot be combined unless we
1155 // also generate a code to scale the offset and reset that bit.
1156 if (CI.CPol != Paired.CPol || (CI.CPol & AMDGPU::CPol::SCAL))
1157 return false;
1158 if (CI.InstClass == S_LOAD_IMM || CI.InstClass == S_BUFFER_LOAD_IMM ||
1159 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) {
1160 // Reject cases like:
1161 // dword + dwordx2 -> dwordx3
1162 // dword + dwordx3 -> dwordx4
1163 // If we tried to combine these cases, we would fail to extract a subreg
1164 // for the result of the second load due to SGPR alignment requirements.
1165 if (CI.Width != Paired.Width &&
1166 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset))
1167 return false;
1168 }
1169 return true;
1170 }
1171
1172 // If the offset in elements doesn't fit in 8-bits, we might be able to use
1173 // the stride 64 versions.
1174 if ((EltOffset0 % 64 == 0) && (EltOffset1 % 64) == 0 &&
1175 isUInt<8>(x: EltOffset0 / 64) && isUInt<8>(x: EltOffset1 / 64)) {
1176 if (Modify) {
1177 CI.Offset = EltOffset0 / 64;
1178 Paired.Offset = EltOffset1 / 64;
1179 CI.UseST64 = true;
1180 }
1181 return true;
1182 }
1183
1184 // Check if the new offsets fit in the reduced 8-bit range.
1185 if (isUInt<8>(x: EltOffset0) && isUInt<8>(x: EltOffset1)) {
1186 if (Modify) {
1187 CI.Offset = EltOffset0;
1188 Paired.Offset = EltOffset1;
1189 }
1190 return true;
1191 }
1192
1193 // Try to shift base address to decrease offsets.
1194 uint32_t Min = std::min(a: EltOffset0, b: EltOffset1);
1195 uint32_t Max = std::max(a: EltOffset0, b: EltOffset1);
1196
1197 const uint32_t Mask = maskTrailingOnes<uint32_t>(N: 8) * 64;
1198 if (((Max - Min) & ~Mask) == 0) {
1199 if (Modify) {
1200 // From the range of values we could use for BaseOff, choose the one that
1201 // is aligned to the highest power of two, to maximise the chance that
1202 // the same offset can be reused for other load/store pairs.
1203 uint32_t BaseOff = mostAlignedValueInRange(Lo: Max - 0xff * 64, Hi: Min);
1204 // Copy the low bits of the offsets, so that when we adjust them by
1205 // subtracting BaseOff they will be multiples of 64.
1206 BaseOff |= Min & maskTrailingOnes<uint32_t>(N: 6);
1207 CI.BaseOff = BaseOff * CI.EltSize;
1208 CI.Offset = (EltOffset0 - BaseOff) / 64;
1209 Paired.Offset = (EltOffset1 - BaseOff) / 64;
1210 CI.UseST64 = true;
1211 }
1212 return true;
1213 }
1214
1215 if (isUInt<8>(x: Max - Min)) {
1216 if (Modify) {
1217 // From the range of values we could use for BaseOff, choose the one that
1218 // is aligned to the highest power of two, to maximise the chance that
1219 // the same offset can be reused for other load/store pairs.
1220 uint32_t BaseOff = mostAlignedValueInRange(Lo: Max - 0xff, Hi: Min);
1221 CI.BaseOff = BaseOff * CI.EltSize;
1222 CI.Offset = EltOffset0 - BaseOff;
1223 Paired.Offset = EltOffset1 - BaseOff;
1224 }
1225 return true;
1226 }
1227
1228 return false;
1229}
1230
1231bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM,
1232 const CombineInfo &CI,
1233 const CombineInfo &Paired) {
1234 const unsigned Width = (CI.Width + Paired.Width);
1235 switch (CI.InstClass) {
1236 default:
1237 return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3));
1238 case S_BUFFER_LOAD_IMM:
1239 case S_BUFFER_LOAD_SGPR_IMM:
1240 case S_LOAD_IMM:
1241 switch (Width) {
1242 default:
1243 return false;
1244 case 2:
1245 case 4:
1246 case 8:
1247 return true;
1248 case 3:
1249 return STM.hasScalarDwordx3Loads();
1250 }
1251 }
1252}
1253
1254const TargetRegisterClass *
1255SILoadStoreOptimizer::getDataRegClass(const MachineInstr &MI) const {
1256 if (const auto *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)) {
1257 return TRI->getRegClassForReg(MRI: *MRI, Reg: Dst->getReg());
1258 }
1259 if (const auto *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata)) {
1260 return TRI->getRegClassForReg(MRI: *MRI, Reg: Src->getReg());
1261 }
1262 if (const auto *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0)) {
1263 return TRI->getRegClassForReg(MRI: *MRI, Reg: Src->getReg());
1264 }
1265 if (const auto *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
1266 return TRI->getRegClassForReg(MRI: *MRI, Reg: Dst->getReg());
1267 }
1268 if (const auto *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdata)) {
1269 return TRI->getRegClassForReg(MRI: *MRI, Reg: Src->getReg());
1270 }
1271 return nullptr;
1272}
1273
1274/// This function assumes that CI comes before Paired in a basic block. Return
1275/// an insertion point for the merged instruction or nullptr on failure.
1276SILoadStoreOptimizer::CombineInfo *
1277SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI,
1278 CombineInfo &Paired) {
1279 // If another instruction has already been merged into CI, it may now be a
1280 // type that we can't do any further merging into.
1281 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN)
1282 return nullptr;
1283 assert(CI.InstClass == Paired.InstClass);
1284
1285 if (getInstSubclass(Opc: CI.I->getOpcode(), TII: *TII) !=
1286 getInstSubclass(Opc: Paired.I->getOpcode(), TII: *TII))
1287 return nullptr;
1288
1289 // Check both offsets (or masks for MIMG) can be combined and fit in the
1290 // reduced range.
1291 if (CI.InstClass == MIMG) {
1292 if (!dmasksCanBeCombined(CI, TII: *TII, Paired))
1293 return nullptr;
1294 } else {
1295 if (!widthsFit(STM: *STM, CI, Paired) || !offsetsCanBeCombined(CI, STI: *STM, Paired))
1296 return nullptr;
1297 }
1298
1299 DenseSet<Register> RegDefs;
1300 DenseSet<Register> RegUses;
1301 CombineInfo *Where;
1302 if (CI.I->mayLoad()) {
1303 // Try to hoist Paired up to CI.
1304 addDefsUsesToList(MI: *Paired.I, RegDefs, RegUses);
1305 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) {
1306 if (!canSwapInstructions(ARegDefs: RegDefs, ARegUses: RegUses, A: *Paired.I, B: *MBBI))
1307 return nullptr;
1308 }
1309 Where = &CI;
1310 } else {
1311 // Try to sink CI down to Paired.
1312 addDefsUsesToList(MI: *CI.I, RegDefs, RegUses);
1313 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) {
1314 if (!canSwapInstructions(ARegDefs: RegDefs, ARegUses: RegUses, A: *CI.I, B: *MBBI))
1315 return nullptr;
1316 }
1317 Where = &Paired;
1318 }
1319
1320 // Call offsetsCanBeCombined with modify = true so that the offsets are
1321 // correct for the new instruction. This should return true, because
1322 // this function should only be called on CombineInfo objects that
1323 // have already been confirmed to be mergeable.
1324 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) {
1325 if (STM->hasNeedsAligned2addrDS() &&
1326 (CI.I->memoperands_empty() ||
1327 (*CI.I->memoperands_begin())->getAlign().value() < CI.Width * 4))
1328 return nullptr;
1329 offsetsCanBeCombined(CI, STI: *STM, Paired, Modify: true);
1330 }
1331
1332 if (CI.InstClass == DS_WRITE) {
1333 // Both data operands must be AGPR or VGPR, so the data registers needs to
1334 // be constrained to one or the other. We expect to only emit the VGPR form
1335 // here for now.
1336 //
1337 // FIXME: There is currently a hack in getRegClass to report that the write2
1338 // operands are VGPRs. In the future we should have separate agpr
1339 // instruction definitions.
1340 const MachineOperand *Data0 =
1341 TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::data0);
1342 const MachineOperand *Data1 =
1343 TII->getNamedOperand(MI&: *Paired.I, OperandName: AMDGPU::OpName::data0);
1344
1345 const MCInstrDesc &Write2Opc = TII->get(Opcode: getWrite2Opcode(CI));
1346 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Write2Opc.getOpcode(),
1347 Name: AMDGPU::OpName::data0);
1348 int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Write2Opc.getOpcode(),
1349 Name: AMDGPU::OpName::data1);
1350
1351 const TargetRegisterClass *DataRC0 = TII->getRegClass(MCID: Write2Opc, OpNum: Data0Idx);
1352
1353 const TargetRegisterClass *DataRC1 = TII->getRegClass(MCID: Write2Opc, OpNum: Data1Idx);
1354
1355 if (unsigned SubReg = Data0->getSubReg()) {
1356 DataRC0 = TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: Data0->getReg()),
1357 B: DataRC0, Idx: SubReg);
1358 }
1359
1360 if (unsigned SubReg = Data1->getSubReg()) {
1361 DataRC1 = TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: Data1->getReg()),
1362 B: DataRC1, Idx: SubReg);
1363 }
1364
1365 if (!MRI->constrainRegClass(Reg: Data0->getReg(), RC: DataRC0) ||
1366 !MRI->constrainRegClass(Reg: Data1->getReg(), RC: DataRC1))
1367 return nullptr;
1368
1369 // TODO: If one register can be constrained, and not the other, insert a
1370 // copy.
1371 }
1372
1373 return Where;
1374}
1375
1376// Copy the merged load result from DestReg to the original dest regs of CI and
1377// Paired.
1378void SILoadStoreOptimizer::copyToDestRegs(
1379 CombineInfo &CI, CombineInfo &Paired,
1380 MachineBasicBlock::iterator InsertBefore, const DebugLoc &DL,
1381 AMDGPU::OpName OpName, Register DestReg) const {
1382 MachineBasicBlock *MBB = CI.I->getParent();
1383
1384 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1385
1386 // Copy to the old destination registers.
1387 const MCInstrDesc &CopyDesc = TII->get(Opcode: TargetOpcode::COPY);
1388 auto *Dest0 = TII->getNamedOperand(MI&: *CI.I, OperandName: OpName);
1389 auto *Dest1 = TII->getNamedOperand(MI&: *Paired.I, OperandName: OpName);
1390
1391 // The constrained sload instructions in S_LOAD_IMM class will have
1392 // `early-clobber` flag in the dst operand. Remove the flag before using the
1393 // MOs in copies.
1394 Dest0->setIsEarlyClobber(false);
1395 Dest1->setIsEarlyClobber(false);
1396
1397 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: CopyDesc)
1398 .add(MO: *Dest0) // Copy to same destination including flags and sub reg.
1399 .addReg(RegNo: DestReg, Flags: {}, SubReg: SubRegIdx0);
1400 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: CopyDesc)
1401 .add(MO: *Dest1)
1402 .addReg(RegNo: DestReg, Flags: RegState::Kill, SubReg: SubRegIdx1);
1403}
1404
1405// Return a register for the source of the merged store after copying the
1406// original source regs of CI and Paired into it.
1407Register
1408SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
1409 MachineBasicBlock::iterator InsertBefore,
1410 const DebugLoc &DL,
1411 AMDGPU::OpName OpName) const {
1412 MachineBasicBlock *MBB = CI.I->getParent();
1413
1414 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1415
1416 // Copy to the new source register.
1417 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1418 Register SrcReg = MRI->createVirtualRegister(RegClass: SuperRC);
1419
1420 const auto *Src0 = TII->getNamedOperand(MI&: *CI.I, OperandName: OpName);
1421 const auto *Src1 = TII->getNamedOperand(MI&: *Paired.I, OperandName: OpName);
1422
1423 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SrcReg)
1424 .add(MO: *Src0)
1425 .addImm(Val: SubRegIdx0)
1426 .add(MO: *Src1)
1427 .addImm(Val: SubRegIdx1);
1428
1429 return SrcReg;
1430}
1431
1432unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize) const {
1433 if (STM->ldsRequiresM0Init())
1434 return (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64;
1435 return (EltSize == 4) ? AMDGPU::DS_READ2_B32_gfx9 : AMDGPU::DS_READ2_B64_gfx9;
1436}
1437
1438unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize) const {
1439 if (STM->ldsRequiresM0Init())
1440 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64;
1441
1442 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32_gfx9
1443 : AMDGPU::DS_READ2ST64_B64_gfx9;
1444}
1445
1446MachineBasicBlock::iterator
1447SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
1448 MachineBasicBlock::iterator InsertBefore) {
1449 MachineBasicBlock *MBB = CI.I->getParent();
1450
1451 // Be careful, since the addresses could be subregisters themselves in weird
1452 // cases, like vectors of pointers.
1453 const auto *AddrReg = TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::addr);
1454
1455 unsigned NewOffset0 = std::min(a: CI.Offset, b: Paired.Offset);
1456 unsigned NewOffset1 = std::max(a: CI.Offset, b: Paired.Offset);
1457 unsigned Opc =
1458 CI.UseST64 ? read2ST64Opcode(EltSize: CI.EltSize) : read2Opcode(EltSize: CI.EltSize);
1459
1460 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1461 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1462
1463 const MCInstrDesc &Read2Desc = TII->get(Opcode: Opc);
1464
1465 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1466 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1467
1468 DebugLoc DL =
1469 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1470
1471 Register BaseReg = AddrReg->getReg();
1472 unsigned BaseSubReg = AddrReg->getSubReg();
1473 RegState BaseRegFlags = {};
1474 if (CI.BaseOff) {
1475 Register ImmReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1476 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: ImmReg)
1477 .addImm(Val: CI.BaseOff);
1478
1479 BaseReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1480 BaseRegFlags = RegState::Kill;
1481
1482 TII->getAddNoCarry(MBB&: *MBB, I: InsertBefore, DL, DestReg: BaseReg)
1483 .addReg(RegNo: ImmReg)
1484 .addReg(RegNo: AddrReg->getReg(), Flags: {}, SubReg: BaseSubReg)
1485 .addImm(Val: 0); // clamp bit
1486 BaseSubReg = 0;
1487 }
1488
1489 MachineInstrBuilder Read2 =
1490 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: Read2Desc, DestReg)
1491 .addReg(RegNo: BaseReg, Flags: BaseRegFlags, SubReg: BaseSubReg) // addr
1492 .addImm(Val: NewOffset0) // offset0
1493 .addImm(Val: NewOffset1) // offset1
1494 .addImm(Val: 0) // gds
1495 .cloneMergedMemRefs(OtherMIs: {&*CI.I, &*Paired.I});
1496
1497 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdst, DestReg);
1498
1499 CI.I->eraseFromParent();
1500 Paired.I->eraseFromParent();
1501
1502 LLVM_DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n');
1503 return Read2;
1504}
1505
1506unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize) const {
1507 if (STM->ldsRequiresM0Init())
1508 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64;
1509 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32_gfx9
1510 : AMDGPU::DS_WRITE2_B64_gfx9;
1511}
1512
1513unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize) const {
1514 if (STM->ldsRequiresM0Init())
1515 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32
1516 : AMDGPU::DS_WRITE2ST64_B64;
1517
1518 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32_gfx9
1519 : AMDGPU::DS_WRITE2ST64_B64_gfx9;
1520}
1521
1522unsigned SILoadStoreOptimizer::getWrite2Opcode(const CombineInfo &CI) const {
1523 return CI.UseST64 ? write2ST64Opcode(EltSize: CI.EltSize) : write2Opcode(EltSize: CI.EltSize);
1524}
1525
1526MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
1527 CombineInfo &CI, CombineInfo &Paired,
1528 MachineBasicBlock::iterator InsertBefore) {
1529 MachineBasicBlock *MBB = CI.I->getParent();
1530
1531 // Be sure to use .addOperand(), and not .addReg() with these. We want to be
1532 // sure we preserve the subregister index and any register flags set on them.
1533 const MachineOperand *AddrReg =
1534 TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::addr);
1535 const MachineOperand *Data0 =
1536 TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::data0);
1537 const MachineOperand *Data1 =
1538 TII->getNamedOperand(MI&: *Paired.I, OperandName: AMDGPU::OpName::data0);
1539
1540 unsigned NewOffset0 = CI.Offset;
1541 unsigned NewOffset1 = Paired.Offset;
1542 unsigned Opc = getWrite2Opcode(CI);
1543
1544 if (NewOffset0 > NewOffset1) {
1545 // Canonicalize the merged instruction so the smaller offset comes first.
1546 std::swap(a&: NewOffset0, b&: NewOffset1);
1547 std::swap(a&: Data0, b&: Data1);
1548 }
1549
1550 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1551 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1552
1553 const MCInstrDesc &Write2Desc = TII->get(Opcode: Opc);
1554 DebugLoc DL =
1555 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1556
1557 Register BaseReg = AddrReg->getReg();
1558 unsigned BaseSubReg = AddrReg->getSubReg();
1559 RegState BaseRegFlags = {};
1560 if (CI.BaseOff) {
1561 Register ImmReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1562 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: ImmReg)
1563 .addImm(Val: CI.BaseOff);
1564
1565 BaseReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1566 BaseRegFlags = RegState::Kill;
1567
1568 TII->getAddNoCarry(MBB&: *MBB, I: InsertBefore, DL, DestReg: BaseReg)
1569 .addReg(RegNo: ImmReg)
1570 .addReg(RegNo: AddrReg->getReg(), Flags: {}, SubReg: BaseSubReg)
1571 .addImm(Val: 0); // clamp bit
1572 BaseSubReg = 0;
1573 }
1574
1575 MachineInstrBuilder Write2 =
1576 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: Write2Desc)
1577 .addReg(RegNo: BaseReg, Flags: BaseRegFlags, SubReg: BaseSubReg) // addr
1578 .add(MO: *Data0) // data0
1579 .add(MO: *Data1) // data1
1580 .addImm(Val: NewOffset0) // offset0
1581 .addImm(Val: NewOffset1) // offset1
1582 .addImm(Val: 0) // gds
1583 .cloneMergedMemRefs(OtherMIs: {&*CI.I, &*Paired.I});
1584
1585 CI.I->eraseFromParent();
1586 Paired.I->eraseFromParent();
1587
1588 LLVM_DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n');
1589 return Write2;
1590}
1591
1592MachineBasicBlock::iterator
1593SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
1594 MachineBasicBlock::iterator InsertBefore) {
1595 MachineBasicBlock *MBB = CI.I->getParent();
1596 DebugLoc DL =
1597 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1598
1599 const unsigned Opcode = getNewOpcode(CI, Paired);
1600
1601 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1602
1603 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1604 unsigned MergedDMask = CI.DMask | Paired.DMask;
1605 unsigned DMaskIdx =
1606 AMDGPU::getNamedOperandIdx(Opcode: CI.I->getOpcode(), Name: AMDGPU::OpName::dmask);
1607
1608 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1609 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) {
1610 if (I == DMaskIdx)
1611 MIB.addImm(Val: MergedDMask);
1612 else
1613 MIB.add(MO: (*CI.I).getOperand(i: I));
1614 }
1615
1616 // It shouldn't be possible to get this far if the two instructions
1617 // don't have a single memoperand, because MachineInstr::mayAlias()
1618 // will return true if this is the case.
1619 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1620
1621 MachineInstr *New = MIB.addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1622
1623 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata, DestReg);
1624
1625 CI.I->eraseFromParent();
1626 Paired.I->eraseFromParent();
1627 return New;
1628}
1629
1630MachineBasicBlock::iterator SILoadStoreOptimizer::mergeSMemLoadImmPair(
1631 CombineInfo &CI, CombineInfo &Paired,
1632 MachineBasicBlock::iterator InsertBefore) {
1633 MachineBasicBlock *MBB = CI.I->getParent();
1634 DebugLoc DL =
1635 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1636
1637 const unsigned Opcode = getNewOpcode(CI, Paired);
1638
1639 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1640
1641 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1642 unsigned MergedOffset = std::min(a: CI.Offset, b: Paired.Offset);
1643
1644 // It shouldn't be possible to get this far if the two instructions
1645 // don't have a single memoperand, because MachineInstr::mayAlias()
1646 // will return true if this is the case.
1647 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1648
1649 MachineInstrBuilder New =
1650 BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg)
1651 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::sbase));
1652 if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM)
1653 New.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset));
1654 New.addImm(Val: MergedOffset);
1655 New.addImm(Val: CI.CPol).addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1656
1657 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::sdst, DestReg);
1658
1659 CI.I->eraseFromParent();
1660 Paired.I->eraseFromParent();
1661 return New;
1662}
1663
1664MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferLoadPair(
1665 CombineInfo &CI, CombineInfo &Paired,
1666 MachineBasicBlock::iterator InsertBefore) {
1667 MachineBasicBlock *MBB = CI.I->getParent();
1668
1669 DebugLoc DL =
1670 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1671
1672 const unsigned Opcode = getNewOpcode(CI, Paired);
1673
1674 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1675
1676 // Copy to the new source register.
1677 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1678 unsigned MergedOffset = std::min(a: CI.Offset, b: Paired.Offset);
1679
1680 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1681
1682 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
1683
1684 if (Regs.VAddr)
1685 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
1686
1687 // It shouldn't be possible to get this far if the two instructions
1688 // don't have a single memoperand, because MachineInstr::mayAlias()
1689 // will return true if this is the case.
1690 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1691
1692 MachineInstr *New =
1693 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
1694 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
1695 .addImm(Val: MergedOffset) // offset
1696 .addImm(Val: CI.CPol) // cpol
1697 .addImm(Val: 0) // swz
1698 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1699
1700 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata, DestReg);
1701
1702 CI.I->eraseFromParent();
1703 Paired.I->eraseFromParent();
1704 return New;
1705}
1706
1707MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair(
1708 CombineInfo &CI, CombineInfo &Paired,
1709 MachineBasicBlock::iterator InsertBefore) {
1710 MachineBasicBlock *MBB = CI.I->getParent();
1711
1712 DebugLoc DL =
1713 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1714
1715 const unsigned Opcode = getNewOpcode(CI, Paired);
1716
1717 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1718
1719 // Copy to the new source register.
1720 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1721 unsigned MergedOffset = std::min(a: CI.Offset, b: Paired.Offset);
1722
1723 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1724
1725 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
1726
1727 if (Regs.VAddr)
1728 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
1729
1730 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1731 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1732 // and use XYZ of XYZW to enable the merge.
1733 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1734 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1735 NumCombinedComponents = 4;
1736 unsigned JoinedFormat =
1737 getBufferFormatWithCompCount(OldFormat: CI.Format, ComponentCount: NumCombinedComponents, STI: *STM);
1738
1739 // It shouldn't be possible to get this far if the two instructions
1740 // don't have a single memoperand, because MachineInstr::mayAlias()
1741 // will return true if this is the case.
1742 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1743
1744 MachineInstr *New =
1745 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
1746 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
1747 .addImm(Val: MergedOffset) // offset
1748 .addImm(Val: JoinedFormat) // format
1749 .addImm(Val: CI.CPol) // cpol
1750 .addImm(Val: 0) // swz
1751 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1752
1753 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata, DestReg);
1754
1755 CI.I->eraseFromParent();
1756 Paired.I->eraseFromParent();
1757 return New;
1758}
1759
1760MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair(
1761 CombineInfo &CI, CombineInfo &Paired,
1762 MachineBasicBlock::iterator InsertBefore) {
1763 MachineBasicBlock *MBB = CI.I->getParent();
1764 DebugLoc DL =
1765 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1766
1767 const unsigned Opcode = getNewOpcode(CI, Paired);
1768
1769 Register SrcReg =
1770 copyFromSrcRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata);
1771
1772 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode))
1773 .addReg(RegNo: SrcReg, Flags: RegState::Kill);
1774
1775 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
1776
1777 if (Regs.VAddr)
1778 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
1779
1780 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1781 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1782 // and use XYZ of XYZW to enable the merge.
1783 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1784 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1785 NumCombinedComponents = 4;
1786 unsigned JoinedFormat =
1787 getBufferFormatWithCompCount(OldFormat: CI.Format, ComponentCount: NumCombinedComponents, STI: *STM);
1788
1789 // It shouldn't be possible to get this far if the two instructions
1790 // don't have a single memoperand, because MachineInstr::mayAlias()
1791 // will return true if this is the case.
1792 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1793
1794 MachineInstr *New =
1795 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
1796 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
1797 .addImm(Val: std::min(a: CI.Offset, b: Paired.Offset)) // offset
1798 .addImm(Val: JoinedFormat) // format
1799 .addImm(Val: CI.CPol) // cpol
1800 .addImm(Val: 0) // swz
1801 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1802
1803 CI.I->eraseFromParent();
1804 Paired.I->eraseFromParent();
1805 return New;
1806}
1807
1808MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatLoadPair(
1809 CombineInfo &CI, CombineInfo &Paired,
1810 MachineBasicBlock::iterator InsertBefore) {
1811 MachineBasicBlock *MBB = CI.I->getParent();
1812
1813 DebugLoc DL =
1814 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1815
1816 const unsigned Opcode = getNewOpcode(CI, Paired);
1817
1818 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1819 Register DestReg = MRI->createVirtualRegister(RegClass: SuperRC);
1820
1821 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode), DestReg);
1822
1823 if (auto *SAddr = TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::saddr))
1824 MIB.add(MO: *SAddr);
1825
1826 MachineInstr *New =
1827 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr))
1828 .addImm(Val: std::min(a: CI.Offset, b: Paired.Offset))
1829 .addImm(Val: CI.CPol)
1830 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1831
1832 copyToDestRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdst, DestReg);
1833
1834 CI.I->eraseFromParent();
1835 Paired.I->eraseFromParent();
1836 return New;
1837}
1838
1839MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatStorePair(
1840 CombineInfo &CI, CombineInfo &Paired,
1841 MachineBasicBlock::iterator InsertBefore) {
1842 MachineBasicBlock *MBB = CI.I->getParent();
1843
1844 DebugLoc DL =
1845 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
1846
1847 const unsigned Opcode = getNewOpcode(CI, Paired);
1848
1849 Register SrcReg =
1850 copyFromSrcRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata);
1851
1852 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode))
1853 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr))
1854 .addReg(RegNo: SrcReg, Flags: RegState::Kill);
1855
1856 if (auto *SAddr = TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::saddr))
1857 MIB.add(MO: *SAddr);
1858
1859 MachineInstr *New =
1860 MIB.addImm(Val: std::min(a: CI.Offset, b: Paired.Offset))
1861 .addImm(Val: CI.CPol)
1862 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
1863
1864 CI.I->eraseFromParent();
1865 Paired.I->eraseFromParent();
1866 return New;
1867}
1868
1869static bool needsConstrainedOpcode(const GCNSubtarget &STM,
1870 ArrayRef<MachineMemOperand *> MMOs,
1871 unsigned Width) {
1872 // Conservatively returns true if not found the MMO.
1873 return STM.isXNACKEnabled() &&
1874 (MMOs.size() != 1 || MMOs[0]->getAlign().value() < Width * 4);
1875}
1876
1877unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
1878 const CombineInfo &Paired) {
1879 const unsigned Width = CI.Width + Paired.Width;
1880
1881 switch (getCommonInstClass(CI, Paired)) {
1882 default:
1883 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE);
1884 // FIXME: Handle d16 correctly
1885 return AMDGPU::getMUBUFOpcode(BaseOpc: AMDGPU::getMUBUFBaseOpcode(Opc: CI.I->getOpcode()),
1886 Elements: Width);
1887 case TBUFFER_LOAD:
1888 case TBUFFER_STORE:
1889 return AMDGPU::getMTBUFOpcode(BaseOpc: AMDGPU::getMTBUFBaseOpcode(Opc: CI.I->getOpcode()),
1890 Elements: Width);
1891
1892 case UNKNOWN:
1893 llvm_unreachable("Unknown instruction class");
1894 case S_BUFFER_LOAD_IMM: {
1895 // If XNACK is enabled, use the constrained opcodes when the first load is
1896 // under-aligned.
1897 bool NeedsConstrainedOpc =
1898 needsConstrainedOpcode(STM: *STM, MMOs: CI.I->memoperands(), Width);
1899 switch (Width) {
1900 default:
1901 return 0;
1902 case 2:
1903 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec
1904 : AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM;
1905 case 3:
1906 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec
1907 : AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM;
1908 case 4:
1909 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec
1910 : AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM;
1911 case 8:
1912 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec
1913 : AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM;
1914 }
1915 }
1916 case S_BUFFER_LOAD_SGPR_IMM: {
1917 // If XNACK is enabled, use the constrained opcodes when the first load is
1918 // under-aligned.
1919 bool NeedsConstrainedOpc =
1920 needsConstrainedOpcode(STM: *STM, MMOs: CI.I->memoperands(), Width);
1921 switch (Width) {
1922 default:
1923 return 0;
1924 case 2:
1925 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec
1926 : AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM;
1927 case 3:
1928 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec
1929 : AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM;
1930 case 4:
1931 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec
1932 : AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM;
1933 case 8:
1934 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec
1935 : AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
1936 }
1937 }
1938 case S_LOAD_IMM: {
1939 // If XNACK is enabled, use the constrained opcodes when the first load is
1940 // under-aligned.
1941 bool NeedsConstrainedOpc =
1942 needsConstrainedOpcode(STM: *STM, MMOs: CI.I->memoperands(), Width);
1943 switch (Width) {
1944 default:
1945 return 0;
1946 case 2:
1947 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX2_IMM_ec
1948 : AMDGPU::S_LOAD_DWORDX2_IMM;
1949 case 3:
1950 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX3_IMM_ec
1951 : AMDGPU::S_LOAD_DWORDX3_IMM;
1952 case 4:
1953 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX4_IMM_ec
1954 : AMDGPU::S_LOAD_DWORDX4_IMM;
1955 case 8:
1956 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX8_IMM_ec
1957 : AMDGPU::S_LOAD_DWORDX8_IMM;
1958 }
1959 }
1960 case GLOBAL_LOAD:
1961 switch (Width) {
1962 default:
1963 return 0;
1964 case 2:
1965 return AMDGPU::GLOBAL_LOAD_DWORDX2;
1966 case 3:
1967 return AMDGPU::GLOBAL_LOAD_DWORDX3;
1968 case 4:
1969 return AMDGPU::GLOBAL_LOAD_DWORDX4;
1970 }
1971 case GLOBAL_LOAD_SADDR:
1972 switch (Width) {
1973 default:
1974 return 0;
1975 case 2:
1976 return AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR;
1977 case 3:
1978 return AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR;
1979 case 4:
1980 return AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR;
1981 }
1982 case GLOBAL_STORE:
1983 switch (Width) {
1984 default:
1985 return 0;
1986 case 2:
1987 return AMDGPU::GLOBAL_STORE_DWORDX2;
1988 case 3:
1989 return AMDGPU::GLOBAL_STORE_DWORDX3;
1990 case 4:
1991 return AMDGPU::GLOBAL_STORE_DWORDX4;
1992 }
1993 case GLOBAL_STORE_SADDR:
1994 switch (Width) {
1995 default:
1996 return 0;
1997 case 2:
1998 return AMDGPU::GLOBAL_STORE_DWORDX2_SADDR;
1999 case 3:
2000 return AMDGPU::GLOBAL_STORE_DWORDX3_SADDR;
2001 case 4:
2002 return AMDGPU::GLOBAL_STORE_DWORDX4_SADDR;
2003 }
2004 case FLAT_LOAD:
2005 switch (Width) {
2006 default:
2007 return 0;
2008 case 2:
2009 return AMDGPU::FLAT_LOAD_DWORDX2;
2010 case 3:
2011 return AMDGPU::FLAT_LOAD_DWORDX3;
2012 case 4:
2013 return AMDGPU::FLAT_LOAD_DWORDX4;
2014 }
2015 case FLAT_STORE:
2016 switch (Width) {
2017 default:
2018 return 0;
2019 case 2:
2020 return AMDGPU::FLAT_STORE_DWORDX2;
2021 case 3:
2022 return AMDGPU::FLAT_STORE_DWORDX3;
2023 case 4:
2024 return AMDGPU::FLAT_STORE_DWORDX4;
2025 }
2026 case FLAT_LOAD_SADDR:
2027 switch (Width) {
2028 default:
2029 return 0;
2030 case 2:
2031 return AMDGPU::FLAT_LOAD_DWORDX2_SADDR;
2032 case 3:
2033 return AMDGPU::FLAT_LOAD_DWORDX3_SADDR;
2034 case 4:
2035 return AMDGPU::FLAT_LOAD_DWORDX4_SADDR;
2036 }
2037 case FLAT_STORE_SADDR:
2038 switch (Width) {
2039 default:
2040 return 0;
2041 case 2:
2042 return AMDGPU::FLAT_STORE_DWORDX2_SADDR;
2043 case 3:
2044 return AMDGPU::FLAT_STORE_DWORDX3_SADDR;
2045 case 4:
2046 return AMDGPU::FLAT_STORE_DWORDX4_SADDR;
2047 }
2048 case MIMG:
2049 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) &&
2050 "No overlaps");
2051 return AMDGPU::getMaskedMIMGOp(Opc: CI.I->getOpcode(), NewChannels: Width);
2052 }
2053}
2054
2055std::pair<unsigned, unsigned>
2056SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI,
2057 const CombineInfo &Paired) {
2058 assert((CI.InstClass != MIMG ||
2059 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) ==
2060 CI.Width + Paired.Width)) &&
2061 "No overlaps");
2062
2063 unsigned Idx0;
2064 unsigned Idx1;
2065
2066 static const unsigned Idxs[5][4] = {
2067 {AMDGPU::sub0, AMDGPU::sub0_sub1, AMDGPU::sub0_sub1_sub2, AMDGPU::sub0_sub1_sub2_sub3},
2068 {AMDGPU::sub1, AMDGPU::sub1_sub2, AMDGPU::sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4},
2069 {AMDGPU::sub2, AMDGPU::sub2_sub3, AMDGPU::sub2_sub3_sub4, AMDGPU::sub2_sub3_sub4_sub5},
2070 {AMDGPU::sub3, AMDGPU::sub3_sub4, AMDGPU::sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6},
2071 {AMDGPU::sub4, AMDGPU::sub4_sub5, AMDGPU::sub4_sub5_sub6, AMDGPU::sub4_sub5_sub6_sub7},
2072 };
2073
2074 assert(CI.Width >= 1 && CI.Width <= 4);
2075 assert(Paired.Width >= 1 && Paired.Width <= 4);
2076
2077 if (Paired < CI) {
2078 Idx1 = Idxs[0][Paired.Width - 1];
2079 Idx0 = Idxs[Paired.Width][CI.Width - 1];
2080 } else {
2081 Idx0 = Idxs[0][CI.Width - 1];
2082 Idx1 = Idxs[CI.Width][Paired.Width - 1];
2083 }
2084
2085 return {Idx0, Idx1};
2086}
2087
2088const TargetRegisterClass *
2089SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI,
2090 const CombineInfo &Paired) const {
2091 if (CI.InstClass == S_BUFFER_LOAD_IMM ||
2092 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM || CI.InstClass == S_LOAD_IMM) {
2093 switch (CI.Width + Paired.Width) {
2094 default:
2095 return nullptr;
2096 case 2:
2097 return &AMDGPU::SReg_64_XEXECRegClass;
2098 case 3:
2099 return &AMDGPU::SGPR_96RegClass;
2100 case 4:
2101 return &AMDGPU::SGPR_128RegClass;
2102 case 8:
2103 return &AMDGPU::SGPR_256RegClass;
2104 case 16:
2105 return &AMDGPU::SGPR_512RegClass;
2106 }
2107 }
2108
2109 // FIXME: This should compute the instruction to use, and then use the result
2110 // of TII->getRegClass.
2111 unsigned BitWidth = 32 * (CI.Width + Paired.Width);
2112 return TRI->isAGPRClass(RC: getDataRegClass(MI: *CI.I))
2113 ? TRI->getAGPRClassForBitWidth(BitWidth)
2114 : TRI->getVGPRClassForBitWidth(BitWidth);
2115}
2116
2117MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferStorePair(
2118 CombineInfo &CI, CombineInfo &Paired,
2119 MachineBasicBlock::iterator InsertBefore) {
2120 MachineBasicBlock *MBB = CI.I->getParent();
2121 DebugLoc DL =
2122 DebugLoc::getMergedLocation(LocA: CI.I->getDebugLoc(), LocB: Paired.I->getDebugLoc());
2123
2124 const unsigned Opcode = getNewOpcode(CI, Paired);
2125
2126 Register SrcReg =
2127 copyFromSrcRegs(CI, Paired, InsertBefore, DL, OpName: AMDGPU::OpName::vdata);
2128
2129 auto MIB = BuildMI(BB&: *MBB, I: InsertBefore, MIMD: DL, MCID: TII->get(Opcode))
2130 .addReg(RegNo: SrcReg, Flags: RegState::Kill);
2131
2132 AddressRegs Regs = getRegs(Opc: Opcode, TII: *TII);
2133
2134 if (Regs.VAddr)
2135 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::vaddr));
2136
2137
2138 // It shouldn't be possible to get this far if the two instructions
2139 // don't have a single memoperand, because MachineInstr::mayAlias()
2140 // will return true if this is the case.
2141 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
2142
2143 MachineInstr *New =
2144 MIB.add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::srsrc))
2145 .add(MO: *TII->getNamedOperand(MI&: *CI.I, OperandName: AMDGPU::OpName::soffset))
2146 .addImm(Val: std::min(a: CI.Offset, b: Paired.Offset)) // offset
2147 .addImm(Val: CI.CPol) // cpol
2148 .addImm(Val: 0) // swz
2149 .addMemOperand(MMO: combineKnownAdjacentMMOs(CI, Paired));
2150
2151 CI.I->eraseFromParent();
2152 Paired.I->eraseFromParent();
2153 return New;
2154}
2155
2156MachineOperand
2157SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) const {
2158 APInt V(32, Val, true);
2159 if (TII->isInlineConstant(Imm: V))
2160 return MachineOperand::CreateImm(Val);
2161
2162 Register Reg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2163 MachineInstr *Mov =
2164 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
2165 MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Reg)
2166 .addImm(Val);
2167 (void)Mov;
2168 LLVM_DEBUG(dbgs() << " "; Mov->dump());
2169 return MachineOperand::CreateReg(Reg, isDef: false);
2170}
2171
2172// Compute base address using Addr and return the final register.
2173Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
2174 const MemAddress &Addr) const {
2175 MachineBasicBlock *MBB = MI.getParent();
2176 MachineBasicBlock::iterator MBBI = MI.getIterator();
2177 const DebugLoc &DL = MI.getDebugLoc();
2178
2179 LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n");
2180
2181 // Use V_ADD_U64_e64 when the original pattern used it (gfx1250+)
2182 if (Addr.Base.UseV64Pattern) {
2183 Register FullDestReg = MRI->createVirtualRegister(
2184 RegClass: TII->getRegClass(MCID: TII->get(Opcode: AMDGPU::V_ADD_U64_e64), OpNum: 0));
2185
2186 // Load the 64-bit offset into an SGPR pair if needed
2187 Register OffsetReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2188 MachineInstr *MovOffset =
2189 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64_IMM_PSEUDO),
2190 DestReg: OffsetReg)
2191 .addImm(Val: Addr.Offset);
2192 MachineInstr *Add64 =
2193 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U64_e64), DestReg: FullDestReg)
2194 .addReg(RegNo: Addr.Base.LoReg)
2195 .addReg(RegNo: OffsetReg, Flags: RegState::Kill)
2196 .addImm(Val: 0);
2197 (void)MovOffset;
2198 (void)Add64;
2199 LLVM_DEBUG(dbgs() << " " << *MovOffset << "\n";
2200 dbgs() << " " << *Add64 << "\n\n";);
2201
2202 return FullDestReg;
2203 }
2204
2205 // Original carry-chain pattern (V_ADD_CO_U32 + V_ADDC_U32)
2206 assert((TRI->getRegSizeInBits(Addr.Base.LoReg, *MRI) == 32 ||
2207 Addr.Base.LoSubReg) &&
2208 "Expected 32-bit Base-Register-Low!!");
2209
2210 assert((TRI->getRegSizeInBits(Addr.Base.HiReg, *MRI) == 32 ||
2211 Addr.Base.HiSubReg) &&
2212 "Expected 32-bit Base-Register-Hi!!");
2213
2214 MachineOperand OffsetLo = createRegOrImm(Val: static_cast<int32_t>(Addr.Offset), MI);
2215 MachineOperand OffsetHi =
2216 createRegOrImm(Val: static_cast<int32_t>(Addr.Offset >> 32), MI);
2217
2218 const auto *CarryRC = TRI->getWaveMaskRegClass();
2219 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
2220 Register DeadCarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
2221
2222 Register DestSub0 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2223 Register DestSub1 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2224 MachineInstr *LoHalf =
2225 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DestSub0)
2226 .addReg(RegNo: CarryReg, Flags: RegState::Define)
2227 .addReg(RegNo: Addr.Base.LoReg, Flags: {}, SubReg: Addr.Base.LoSubReg)
2228 .add(MO: OffsetLo)
2229 .addImm(Val: 0); // clamp bit
2230
2231 MachineInstr *HiHalf =
2232 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DestSub1)
2233 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
2234 .addReg(RegNo: Addr.Base.HiReg, Flags: {}, SubReg: Addr.Base.HiSubReg)
2235 .add(MO: OffsetHi)
2236 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
2237 .addImm(Val: 0); // clamp bit
2238
2239 Register FullDestReg = MRI->createVirtualRegister(RegClass: TRI->getVGPR64Class());
2240 MachineInstr *FullBase =
2241 BuildMI(BB&: *MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
2242 .addReg(RegNo: DestSub0)
2243 .addImm(Val: AMDGPU::sub0)
2244 .addReg(RegNo: DestSub1)
2245 .addImm(Val: AMDGPU::sub1);
2246
2247 (void)LoHalf;
2248 (void)HiHalf;
2249 (void)FullBase;
2250 LLVM_DEBUG(dbgs() << " " << *LoHalf << "\n";
2251 dbgs() << " " << *HiHalf << "\n";
2252 dbgs() << " " << *FullBase << "\n\n";);
2253
2254 return FullDestReg;
2255}
2256
2257// Update base and offset with the NewBase and NewOffset in MI.
2258void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
2259 Register NewBase,
2260 int32_t NewOffset) const {
2261 auto *Base = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
2262 Base->setReg(NewBase);
2263 Base->setIsKill(false);
2264 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->setImm(NewOffset);
2265}
2266
2267// Helper to extract a 64-bit constant offset from a V_ADD_U64_e64 instruction.
2268// Returns true if successful, populating Addr with base register info and
2269// offset.
2270bool SILoadStoreOptimizer::processBaseWithConstOffset64(
2271 MachineInstr *AddDef, const MachineOperand &Base, MemAddress &Addr) const {
2272 if (!Base.isReg())
2273 return false;
2274
2275 MachineOperand *Src0 = TII->getNamedOperand(MI&: *AddDef, OperandName: AMDGPU::OpName::src0);
2276 MachineOperand *Src1 = TII->getNamedOperand(MI&: *AddDef, OperandName: AMDGPU::OpName::src1);
2277
2278 const MachineOperand *BaseOp = nullptr;
2279
2280 auto Offset = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2281
2282 if (Offset) {
2283 BaseOp = Src0;
2284 Addr.Offset = *Offset;
2285 } else {
2286 // Both or neither are constants - can't handle this pattern
2287 return false;
2288 }
2289
2290 // Now extract the base register (which should be a 64-bit VGPR).
2291 Addr.Base.LoReg = BaseOp->getReg();
2292 Addr.Base.UseV64Pattern = true;
2293 return true;
2294}
2295
2296// Analyze Base and extracts:
2297// - 32bit base registers, subregisters
2298// - 64bit constant offset
2299// Expecting base computation as:
2300// %OFFSET0:sgpr_32 = S_MOV_B32 8000
2301// %LO:vgpr_32, %c:sreg_64_xexec =
2302// V_ADD_CO_U32_e64 %BASE_LO:vgpr_32, %103:sgpr_32,
2303// %HI:vgpr_32, = V_ADDC_U32_e64 %BASE_HI:vgpr_32, 0, killed %c:sreg_64_xexec
2304// %Base:vreg_64 =
2305// REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1
2306//
2307// Also handles V_ADD_U64_e64 pattern (gfx1250+):
2308// %OFFSET:sreg_64 = S_MOV_B64_IMM_PSEUDO 256
2309// %Base:vreg_64 = V_ADD_U64_e64 %BASE:vreg_64, %OFFSET:sreg_64, 0
2310void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base,
2311 MemAddress &Addr) const {
2312 if (!Base.isReg())
2313 return;
2314
2315 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Base.getReg());
2316 if (!Def)
2317 return;
2318
2319 // Try V_ADD_U64_e64 pattern first (simpler, used on gfx1250+)
2320 if (Def->getOpcode() == AMDGPU::V_ADD_U64_e64) {
2321 if (processBaseWithConstOffset64(AddDef: Def, Base, Addr))
2322 return;
2323 }
2324
2325 // Fall through to REG_SEQUENCE + V_ADD_CO_U32 + V_ADDC_U32 pattern
2326 if (Def->getOpcode() != AMDGPU::REG_SEQUENCE || Def->getNumOperands() != 5)
2327 return;
2328
2329 MachineOperand BaseLo = Def->getOperand(i: 1);
2330 MachineOperand BaseHi = Def->getOperand(i: 3);
2331 if (!BaseLo.isReg() || !BaseHi.isReg())
2332 return;
2333
2334 MachineInstr *BaseLoDef = MRI->getUniqueVRegDef(Reg: BaseLo.getReg());
2335 MachineInstr *BaseHiDef = MRI->getUniqueVRegDef(Reg: BaseHi.getReg());
2336
2337 if (!BaseLoDef || BaseLoDef->getOpcode() != AMDGPU::V_ADD_CO_U32_e64 ||
2338 !BaseHiDef || BaseHiDef->getOpcode() != AMDGPU::V_ADDC_U32_e64)
2339 return;
2340
2341 MachineOperand *Src0 = TII->getNamedOperand(MI&: *BaseLoDef, OperandName: AMDGPU::OpName::src0);
2342 MachineOperand *Src1 = TII->getNamedOperand(MI&: *BaseLoDef, OperandName: AMDGPU::OpName::src1);
2343
2344 auto Offset0P = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
2345 if (Offset0P)
2346 BaseLo = *Src1;
2347 else {
2348 if (!(Offset0P = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1)))
2349 return;
2350 BaseLo = *Src0;
2351 }
2352
2353 if (!BaseLo.isReg())
2354 return;
2355
2356 Src0 = TII->getNamedOperand(MI&: *BaseHiDef, OperandName: AMDGPU::OpName::src0);
2357 Src1 = TII->getNamedOperand(MI&: *BaseHiDef, OperandName: AMDGPU::OpName::src1);
2358
2359 if (Src0->isImm())
2360 std::swap(a&: Src0, b&: Src1);
2361
2362 if (!Src1->isImm() || Src0->isImm())
2363 return;
2364
2365 uint64_t Offset1 = Src1->getImm();
2366 BaseHi = *Src0;
2367
2368 if (!BaseHi.isReg())
2369 return;
2370
2371 Addr.Base.LoReg = BaseLo.getReg();
2372 Addr.Base.HiReg = BaseHi.getReg();
2373 Addr.Base.LoSubReg = BaseLo.getSubReg();
2374 Addr.Base.HiSubReg = BaseHi.getSubReg();
2375 Addr.Offset = (*Offset0P & 0x00000000ffffffff) | (Offset1 << 32);
2376}
2377
2378// Maintain the correct LDS address for async loads and stores.
2379// It becomes incorrect when promoteConstantOffsetToImm adds an offset only
2380// meant for the global address operand. For async loads the LDS address is in
2381// vdst. For async stores, the LDS address is in vdata.
2382void SILoadStoreOptimizer::updateAsyncLDSAddress(MachineInstr &MI,
2383 int32_t OffsetDiff) const {
2384 if (!TII->usesASYNC_CNT(MI) || OffsetDiff == 0)
2385 return;
2386
2387 MachineOperand *LDSAddr = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
2388 if (!LDSAddr)
2389 LDSAddr = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
2390 assert(LDSAddr);
2391
2392 Register OldReg = LDSAddr->getReg();
2393 Register NewReg = MRI->createVirtualRegister(RegClass: MRI->getRegClass(Reg: OldReg));
2394 MachineBasicBlock &MBB = *MI.getParent();
2395 const DebugLoc &DL = MI.getDebugLoc();
2396 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg: NewReg)
2397 .addReg(RegNo: OldReg)
2398 .addImm(Val: -OffsetDiff)
2399 .addImm(Val: 0);
2400
2401 LDSAddr->setReg(NewReg);
2402}
2403
2404bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
2405 MachineInstr &MI,
2406 MemInfoMap &Visited,
2407 SmallPtrSet<MachineInstr *, 4> &AnchorList) const {
2408
2409 if (!STM->hasFlatInstOffsets() || !SIInstrInfo::isFLAT(MI))
2410 return false;
2411
2412 // TODO: Support FLAT_SCRATCH. Currently code expects 64-bit pointers.
2413 if (SIInstrInfo::isFLATScratch(MI))
2414 return false;
2415
2416 unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS
2417 : AMDGPUAS::FLAT_ADDRESS;
2418
2419 AMDGPU::FlatAddrSpace FlatVariant = AS == AMDGPUAS::GLOBAL_ADDRESS
2420 ? AMDGPU::FlatAddrSpace::FlatGlobal
2421 : AMDGPU::FlatAddrSpace::FLAT;
2422 bool AllowNegativeOffset =
2423 TII->allowNegativeFlatOffset(FlatVariant) && !TII->usesASYNC_CNT(MI);
2424 // The async global instructions use i24 offset for global address but u16
2425 // offset for LDS address. In this case, we just only promote when the offset
2426 // is u16.
2427 bool IsOffsetU16 = TII->usesASYNC_CNT(MI);
2428
2429 if (AnchorList.count(Ptr: &MI))
2430 return false;
2431
2432 LLVM_DEBUG(dbgs() << "\nTryToPromoteConstantOffsetToImmFor "; MI.dump());
2433
2434 if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm()) {
2435 LLVM_DEBUG(dbgs() << " Const-offset is already promoted.\n";);
2436 return false;
2437 }
2438
2439 // Step1: Find the base-registers and a 64bit constant offset.
2440 MachineOperand &Base = *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
2441 auto [It, Inserted] = Visited.try_emplace(Key: &MI);
2442 MemAddress MAddr;
2443 if (Inserted) {
2444 processBaseWithConstOffset(Base, Addr&: MAddr);
2445 It->second = MAddr;
2446 } else
2447 MAddr = It->second;
2448
2449 if (MAddr.Offset == 0) {
2450 LLVM_DEBUG(dbgs() << " Failed to extract constant-offset or there are no"
2451 " constant offsets that can be promoted.\n";);
2452 return false;
2453 }
2454
2455 LLVM_DEBUG(dbgs() << " BASE: {" << printReg(MAddr.Base.HiReg, TRI) << ", "
2456 << printReg(MAddr.Base.LoReg, TRI)
2457 << "} Offset: " << MAddr.Offset << "\n\n";);
2458
2459 // Step2: Traverse through MI's basic block and find an anchor(that has the
2460 // same base-registers) with the highest 13bit distance from MI's offset.
2461 // E.g. (64bit loads)
2462 // bb:
2463 // addr1 = &a + 4096; load1 = load(addr1, 0)
2464 // addr2 = &a + 6144; load2 = load(addr2, 0)
2465 // addr3 = &a + 8192; load3 = load(addr3, 0)
2466 // addr4 = &a + 10240; load4 = load(addr4, 0)
2467 // addr5 = &a + 12288; load5 = load(addr5, 0)
2468 //
2469 // Starting from the first load, the optimization will try to find a new base
2470 // from which (&a + 4096) has 13 bit distance. Both &a + 6144 and &a + 8192
2471 // has 13bit distance from &a + 4096. The heuristic considers &a + 8192
2472 // as the new-base(anchor) because of the maximum distance which can
2473 // accommodate more intermediate bases presumably.
2474 //
2475 // Step3: move (&a + 8192) above load1. Compute and promote offsets from
2476 // (&a + 8192) for load1, load2, load4.
2477 // addr = &a + 8192
2478 // load1 = load(addr, -4096)
2479 // load2 = load(addr, -2048)
2480 // load3 = load(addr, 0)
2481 // load4 = load(addr, 2048)
2482 // addr5 = &a + 12288; load5 = load(addr5, 0)
2483 //
2484 MachineInstr *AnchorInst = nullptr;
2485 MemAddress AnchorAddr;
2486 uint32_t MaxDist = std::numeric_limits<uint32_t>::min();
2487 SmallVector<std::pair<MachineInstr *, int64_t>, 4> InstsWCommonBase;
2488 bool MIIsAnchor = false;
2489
2490 MachineBasicBlock *MBB = MI.getParent();
2491 MachineBasicBlock::iterator E = MBB->end();
2492 MachineBasicBlock::iterator MBBI = MI.getIterator();
2493 ++MBBI;
2494 const SITargetLowering *TLI = STM->getTargetLowering();
2495
2496 for ( ; MBBI != E; ++MBBI) {
2497 MachineInstr &MINext = *MBBI;
2498 // TODO: Support finding an anchor(with same base) from store addresses or
2499 // any other load addresses where the opcodes are different.
2500 if (MINext.getOpcode() != MI.getOpcode() ||
2501 TII->getNamedOperand(MI&: MINext, OperandName: AMDGPU::OpName::offset)->getImm())
2502 continue;
2503
2504 const MachineOperand &BaseNext =
2505 *TII->getNamedOperand(MI&: MINext, OperandName: AMDGPU::OpName::vaddr);
2506 MemAddress MAddrNext;
2507 auto [It, Inserted] = Visited.try_emplace(Key: &MINext);
2508 if (Inserted) {
2509 processBaseWithConstOffset(Base: BaseNext, Addr&: MAddrNext);
2510 It->second = MAddrNext;
2511 } else
2512 MAddrNext = It->second;
2513
2514 if (MAddrNext.Base.LoReg != MAddr.Base.LoReg ||
2515 MAddrNext.Base.HiReg != MAddr.Base.HiReg ||
2516 MAddrNext.Base.LoSubReg != MAddr.Base.LoSubReg ||
2517 MAddrNext.Base.HiSubReg != MAddr.Base.HiSubReg)
2518 continue;
2519
2520 InstsWCommonBase.emplace_back(Args: &MINext, Args&: MAddrNext.Offset);
2521
2522 if (AllowNegativeOffset) {
2523 int64_t Dist = MAddr.Offset - MAddrNext.Offset;
2524 TargetLoweringBase::AddrMode AM;
2525 AM.HasBaseReg = true;
2526 AM.BaseOffs = Dist;
2527 if (TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2528 (uint32_t)std::abs(i: Dist) > MaxDist) {
2529 MaxDist = std::abs(i: Dist);
2530
2531 AnchorAddr = MAddrNext;
2532 AnchorInst = &MINext;
2533 }
2534 }
2535 }
2536
2537 // When negative offsets are not allowed, pick the candidate with the smallest
2538 // offset as anchor so all promoted offsets are non-negative. If MI itself has
2539 // the smallest offset, MI becomes the reference point (MIIsAnchor).
2540 if (!AllowNegativeOffset && !InstsWCommonBase.empty()) {
2541 for (auto &[Inst, Offset] : InstsWCommonBase) {
2542 int64_t Dist = MAddr.Offset - Offset;
2543 TargetLoweringBase::AddrMode AM;
2544 AM.HasBaseReg = true;
2545 AM.BaseOffs = Dist;
2546 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2547 (!IsOffsetU16 || isUInt<16>(x: Dist)) &&
2548 (!AnchorInst || Offset < AnchorAddr.Offset)) {
2549 AnchorAddr = Visited[Inst];
2550 AnchorInst = Inst;
2551 }
2552 }
2553 if (!AnchorInst)
2554 MIIsAnchor = true;
2555 }
2556
2557 if (AnchorInst) {
2558 LLVM_DEBUG(dbgs() << " Anchor-Inst(with max-distance from Offset): ";
2559 AnchorInst->dump());
2560 LLVM_DEBUG(dbgs() << " Anchor-Offset from BASE: "
2561 << AnchorAddr.Offset << "\n\n");
2562
2563 // Instead of moving up, just re-compute anchor-instruction's base address.
2564 Register Base = computeBase(MI, Addr: AnchorAddr);
2565
2566 int32_t OffsetDiff = MAddr.Offset - AnchorAddr.Offset;
2567 updateBaseAndOffset(MI, NewBase: Base, NewOffset: OffsetDiff);
2568 updateAsyncLDSAddress(MI, OffsetDiff);
2569 LLVM_DEBUG(dbgs() << " After promotion: "; MI.dump(););
2570
2571 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2572 TargetLoweringBase::AddrMode AM;
2573 AM.HasBaseReg = true;
2574 AM.BaseOffs = OtherOffset - AnchorAddr.Offset;
2575
2576 if (TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2577 (AllowNegativeOffset || AM.BaseOffs >= 0) &&
2578 (!IsOffsetU16 || isUInt<16>(x: AM.BaseOffs))) {
2579 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
2580 OtherMI->dump());
2581 int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
2582 updateBaseAndOffset(MI&: *OtherMI, NewBase: Base, NewOffset: OtherOffsetDiff);
2583 updateAsyncLDSAddress(MI&: *OtherMI, OffsetDiff: OtherOffsetDiff);
2584 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2585 }
2586 }
2587 AnchorList.insert(Ptr: AnchorInst);
2588 return true;
2589 }
2590
2591 if (MIIsAnchor) {
2592 LLVM_DEBUG(dbgs() << " MI is anchor (smallest offset); promoting "
2593 "candidates relative to MI's base.\n");
2594
2595 Register Base = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)->getReg();
2596 bool AnyPromoted = false;
2597
2598 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2599 int64_t Dist = OtherOffset - MAddr.Offset;
2600 TargetLoweringBase::AddrMode AM;
2601 AM.HasBaseReg = true;
2602 AM.BaseOffs = Dist;
2603 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AddrSpace: AS) &&
2604 (!IsOffsetU16 || isUInt<16>(x: Dist))) {
2605 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset << ")";
2606 OtherMI->dump());
2607 updateBaseAndOffset(MI&: *OtherMI, NewBase: Base, NewOffset: Dist);
2608 updateAsyncLDSAddress(MI&: *OtherMI, OffsetDiff: Dist);
2609 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2610 AnyPromoted = true;
2611 }
2612 }
2613
2614 if (AnyPromoted) {
2615 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)->setIsKill(false);
2616 AnchorList.insert(Ptr: &MI);
2617 return true;
2618 }
2619 }
2620
2621 return false;
2622}
2623
2624void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI,
2625 std::list<std::list<CombineInfo> > &MergeableInsts) const {
2626 for (std::list<CombineInfo> &AddrList : MergeableInsts) {
2627 if (AddrList.front().InstClass == CI.InstClass &&
2628 AddrList.front().hasSameBaseAddress(CI)) {
2629 AddrList.emplace_back(args: CI);
2630 return;
2631 }
2632 }
2633
2634 // Base address not found, so add a new list.
2635 MergeableInsts.emplace_back(args: 1, args: CI);
2636}
2637
2638std::pair<MachineBasicBlock::iterator, bool>
2639SILoadStoreOptimizer::collectMergeableInsts(
2640 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
2641 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
2642 std::list<std::list<CombineInfo>> &MergeableInsts) const {
2643 bool Modified = false;
2644
2645 // Sort potential mergeable instructions into lists. One list per base address.
2646 unsigned Order = 0;
2647 MachineBasicBlock::iterator BlockI = Begin;
2648 for (; BlockI != End; ++BlockI) {
2649 MachineInstr &MI = *BlockI;
2650
2651 // We run this before checking if an address is mergeable, because it can produce
2652 // better code even if the instructions aren't mergeable.
2653 if (promoteConstantOffsetToImm(MI, Visited, AnchorList))
2654 Modified = true;
2655
2656 // Treat volatile accesses, ordered accesses and unmodeled side effects as
2657 // barriers. We can look after this barrier for separate merges.
2658 if (MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects()) {
2659 LLVM_DEBUG(dbgs() << "Breaking search on barrier: " << MI);
2660
2661 // Search will resume after this instruction in a separate merge list.
2662 ++BlockI;
2663 break;
2664 }
2665
2666 const InstClassEnum InstClass = getInstClass(Opc: MI.getOpcode(), TII: *TII);
2667 if (InstClass == UNKNOWN)
2668 continue;
2669
2670 // Do not merge VMEM buffer instructions with "swizzled" bit set.
2671 int Swizzled =
2672 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::swz);
2673 if (Swizzled != -1 && MI.getOperand(i: Swizzled).getImm())
2674 continue;
2675
2676 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
2677 if (!STM->hasRelaxedTBufferOOBMode()) {
2678 LLVM_DEBUG(
2679 dbgs() << "Skip tbuffer combine: relaxed OOB mode not enabled\n");
2680 continue;
2681 }
2682
2683 const MachineOperand *Fmt =
2684 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::format);
2685 if (!AMDGPU::getGcnBufferFormatInfo(Format: Fmt->getImm(), STI: *STM)) {
2686 LLVM_DEBUG(dbgs() << "Skip tbuffer with unknown format: " << MI);
2687 continue;
2688 }
2689 } else if (InstClass == MIMG) {
2690 // Do not merge MIMG instructions with tfe or lwe enabled.
2691 // TFE/LWE add a status result that the image merge path does not model.
2692 const auto *TFEOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
2693 if (TFEOp && TFEOp->getImm())
2694 continue;
2695
2696 const auto *LWEOp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
2697 if (LWEOp && LWEOp->getImm())
2698 continue;
2699 }
2700
2701 CombineInfo CI;
2702 CI.setMI(MI, LSO: *this);
2703 CI.Order = Order++;
2704
2705 if (!CI.hasMergeableAddress(MRI: *MRI))
2706 continue;
2707
2708 LLVM_DEBUG(dbgs() << "Mergeable: " << MI);
2709
2710 addInstToMergeableList(CI, MergeableInsts);
2711 }
2712
2713 // At this point we have lists of Mergeable instructions.
2714 //
2715 // Part 2: Sort lists by offset and then for each CombineInfo object in the
2716 // list try to find an instruction that can be merged with I. If an instruction
2717 // is found, it is stored in the Paired field. If no instructions are found, then
2718 // the CombineInfo object is deleted from the list.
2719
2720 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2721 E = MergeableInsts.end(); I != E;) {
2722
2723 std::list<CombineInfo> &MergeList = *I;
2724 if (MergeList.size() <= 1) {
2725 // This means we have found only one instruction with a given address
2726 // that can be merged, and we need at least 2 instructions to do a merge,
2727 // so this list can be discarded.
2728 I = MergeableInsts.erase(position: I);
2729 continue;
2730 }
2731
2732 // Sort the lists by offsets, this way mergeable instructions will be
2733 // adjacent to each other in the list, which will make it easier to find
2734 // matches.
2735 MergeList.sort(
2736 comp: [] (const CombineInfo &A, const CombineInfo &B) {
2737 return A.Offset < B.Offset;
2738 });
2739 ++I;
2740 }
2741
2742 return {BlockI, Modified};
2743}
2744
2745// Scan through looking for adjacent LDS operations with constant offsets from
2746// the same base register. We rely on the scheduler to do the hard work of
2747// clustering nearby loads, and assume these are all adjacent.
2748bool SILoadStoreOptimizer::optimizeBlock(
2749 std::list<std::list<CombineInfo> > &MergeableInsts) {
2750 bool Modified = false;
2751
2752 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2753 E = MergeableInsts.end(); I != E;) {
2754 std::list<CombineInfo> &MergeList = *I;
2755
2756 bool OptimizeListAgain = false;
2757 if (!optimizeInstsWithSameBaseAddr(MergeList, OptimizeListAgain)) {
2758 // We weren't able to make any changes, so delete the list so we don't
2759 // process the same instructions the next time we try to optimize this
2760 // block.
2761 I = MergeableInsts.erase(position: I);
2762 continue;
2763 }
2764
2765 Modified = true;
2766
2767 // We made changes, but also determined that there were no more optimization
2768 // opportunities, so we don't need to reprocess the list
2769 if (!OptimizeListAgain) {
2770 I = MergeableInsts.erase(position: I);
2771 continue;
2772 }
2773 OptimizeAgain = true;
2774 }
2775 return Modified;
2776}
2777
2778bool
2779SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr(
2780 std::list<CombineInfo> &MergeList,
2781 bool &OptimizeListAgain) {
2782 if (MergeList.empty())
2783 return false;
2784
2785 bool Modified = false;
2786
2787 for (auto I = MergeList.begin(), Next = std::next(x: I); Next != MergeList.end();
2788 Next = std::next(x: I)) {
2789
2790 auto First = I;
2791 auto Second = Next;
2792
2793 if ((*First).Order > (*Second).Order)
2794 std::swap(a&: First, b&: Second);
2795 CombineInfo &CI = *First;
2796 CombineInfo &Paired = *Second;
2797
2798 CombineInfo *Where = checkAndPrepareMerge(CI, Paired);
2799 if (!Where) {
2800 ++I;
2801 continue;
2802 }
2803
2804 Modified = true;
2805
2806 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I);
2807
2808 MachineBasicBlock::iterator NewMI;
2809 switch (CI.InstClass) {
2810 default:
2811 llvm_unreachable("unknown InstClass");
2812 break;
2813 case DS_READ:
2814 NewMI = mergeRead2Pair(CI, Paired, InsertBefore: Where->I);
2815 break;
2816 case DS_WRITE:
2817 NewMI = mergeWrite2Pair(CI, Paired, InsertBefore: Where->I);
2818 break;
2819 case S_BUFFER_LOAD_IMM:
2820 case S_BUFFER_LOAD_SGPR_IMM:
2821 case S_LOAD_IMM:
2822 NewMI = mergeSMemLoadImmPair(CI, Paired, InsertBefore: Where->I);
2823 OptimizeListAgain |= CI.Width + Paired.Width < 8;
2824 break;
2825 case BUFFER_LOAD:
2826 NewMI = mergeBufferLoadPair(CI, Paired, InsertBefore: Where->I);
2827 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2828 break;
2829 case BUFFER_STORE:
2830 NewMI = mergeBufferStorePair(CI, Paired, InsertBefore: Where->I);
2831 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2832 break;
2833 case MIMG:
2834 NewMI = mergeImagePair(CI, Paired, InsertBefore: Where->I);
2835 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2836 break;
2837 case TBUFFER_LOAD:
2838 NewMI = mergeTBufferLoadPair(CI, Paired, InsertBefore: Where->I);
2839 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2840 break;
2841 case TBUFFER_STORE:
2842 NewMI = mergeTBufferStorePair(CI, Paired, InsertBefore: Where->I);
2843 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2844 break;
2845 case FLAT_LOAD:
2846 case FLAT_LOAD_SADDR:
2847 case GLOBAL_LOAD:
2848 case GLOBAL_LOAD_SADDR:
2849 NewMI = mergeFlatLoadPair(CI, Paired, InsertBefore: Where->I);
2850 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2851 break;
2852 case FLAT_STORE:
2853 case FLAT_STORE_SADDR:
2854 case GLOBAL_STORE:
2855 case GLOBAL_STORE_SADDR:
2856 NewMI = mergeFlatStorePair(CI, Paired, InsertBefore: Where->I);
2857 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2858 break;
2859 }
2860 CI.setMI(MI: NewMI, LSO: *this);
2861 CI.Order = Where->Order;
2862 if (I == Second)
2863 I = Next;
2864
2865 MergeList.erase(position: Second);
2866 }
2867
2868 return Modified;
2869}
2870
2871bool SILoadStoreOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
2872 if (skipFunction(F: MF.getFunction()))
2873 return false;
2874 return SILoadStoreOptimizer(
2875 &getAnalysis<AAResultsWrapperPass>().getAAResults())
2876 .run(MF);
2877}
2878
2879bool SILoadStoreOptimizer::run(MachineFunction &MF) {
2880 this->MF = &MF;
2881 STM = &MF.getSubtarget<GCNSubtarget>();
2882 if (!STM->loadStoreOptEnabled())
2883 return false;
2884
2885 TII = STM->getInstrInfo();
2886 TRI = &TII->getRegisterInfo();
2887
2888 MRI = &MF.getRegInfo();
2889
2890 LLVM_DEBUG(dbgs() << "Running SILoadStoreOptimizer\n");
2891
2892 bool Modified = false;
2893
2894 // Contains the list of instructions for which constant offsets are being
2895 // promoted to the IMM. This is tracked for an entire block at time.
2896 SmallPtrSet<MachineInstr *, 4> AnchorList;
2897 MemInfoMap Visited;
2898
2899 for (MachineBasicBlock &MBB : MF) {
2900 MachineBasicBlock::iterator SectionEnd;
2901 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
2902 I = SectionEnd) {
2903 bool CollectModified;
2904 std::list<std::list<CombineInfo>> MergeableInsts;
2905
2906 // First pass: Collect list of all instructions we know how to merge in a
2907 // subset of the block.
2908 std::tie(args&: SectionEnd, args&: CollectModified) =
2909 collectMergeableInsts(Begin: I, End: E, Visited, AnchorList, MergeableInsts);
2910
2911 Modified |= CollectModified;
2912
2913 do {
2914 OptimizeAgain = false;
2915 Modified |= optimizeBlock(MergeableInsts);
2916 } while (OptimizeAgain);
2917 }
2918
2919 Visited.clear();
2920 AnchorList.clear();
2921 }
2922
2923 return Modified;
2924}
2925
2926PreservedAnalyses
2927SILoadStoreOptimizerPass::run(MachineFunction &MF,
2928 MachineFunctionAnalysisManager &MFAM) {
2929 MFPropsModifier _(*this, MF);
2930
2931 if (MF.getFunction().hasOptNone())
2932 return PreservedAnalyses::all();
2933
2934 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
2935 .getManager();
2936 AAResults &AA = FAM.getResult<AAManager>(IR&: MF.getFunction());
2937
2938 bool Changed = SILoadStoreOptimizer(&AA).run(MF);
2939 if (!Changed)
2940 return PreservedAnalyses::all();
2941
2942 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
2943 PA.preserveSet<CFGAnalyses>();
2944 return PA;
2945}
2946