1//===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AMDGPU.
11///
12/// \par
13///
14/// AMDGPU has unique register bank constraints that require special high level
15/// strategies to deal with. There are two main true physical register banks
16/// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a
17/// sort of pseudo-register bank needed to represent SGPRs used in a vector
18/// boolean context. There is also the AGPR bank, which is a special purpose
19/// physical register bank present on some subtargets.
20///
21/// Copying from VGPR to SGPR is generally illegal, unless the value is known to
22/// be uniform. It is generally not valid to legalize operands by inserting
23/// copies as on other targets. Operations which require uniform, SGPR operands
24/// generally require scalarization by repeatedly executing the instruction,
25/// activating each set of lanes using a unique set of input values. This is
26/// referred to as a waterfall loop.
27///
28/// \par Booleans
29///
30/// Booleans (s1 values) requires special consideration. A vector compare result
31/// is naturally a bitmask with one bit per lane, in a 32 or 64-bit
32/// register. These are represented with the VCC bank. During selection, we need
33/// to be able to unambiguously go back from a register class to a register
34/// bank. To distinguish whether an SGPR should use the SGPR or VCC register
35/// bank, we need to know the use context type. An SGPR s1 value always means a
36/// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets
37/// SCC, which is a 1-bit unaddressable register. This will need to be copied to
38/// a 32-bit virtual register. Taken together, this means we need to adjust the
39/// type of boolean operations to be regbank legal. All SALU booleans need to be
40/// widened to 32-bits, and all VALU booleans need to be s1 values.
41///
42/// A noteworthy exception to the s1-means-vcc rule is for legalization artifact
43/// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc
44/// bank. A non-boolean source (such as a truncate from a 1-bit load from
45/// memory) will require a copy to the VCC bank which will require clearing the
46/// high bits and inserting a compare.
47///
48/// \par Constant bus restriction
49///
50/// VALU instructions have a limitation known as the constant bus
51/// restriction. Most VALU instructions can use SGPR operands, but may read at
52/// most 1 SGPR or constant literal value (this to 2 in gfx10 for most
53/// instructions). This is one unique SGPR, so the same SGPR may be used for
54/// multiple operands. From a register bank perspective, any combination of
55/// operands should be legal as an SGPR, but this is contextually dependent on
56/// the SGPR operands all being the same register. There is therefore optimal to
57/// choose the SGPR with the most uses to minimize the number of copies.
58///
59/// We avoid trying to solve this problem in RegBankSelect. Any VALU G_*
60/// operation should have its source operands all mapped to VGPRs (except for
61/// VCC), inserting copies from any SGPR operands. This the most trivial legal
62/// mapping. Anything beyond the simplest 1:1 instruction selection would be too
63/// complicated to solve here. Every optimization pattern or instruction
64/// selected to multiple outputs would have to enforce this rule, and there
65/// would be additional complexity in tracking this rule for every G_*
66/// operation. By forcing all inputs to VGPRs, it also simplifies the task of
67/// picking the optimal operand combination from a post-isel optimization pass.
68///
69//===----------------------------------------------------------------------===//
70
71#include "AMDGPURegisterBankInfo.h"
72
73#include "AMDGPU.h"
74#include "AMDGPUGlobalISelUtils.h"
75#include "AMDGPUInstrInfo.h"
76#include "AMDGPULaneMaskUtils.h"
77#include "GCNSubtarget.h"
78#include "SIMachineFunctionInfo.h"
79#include "SIRegisterInfo.h"
80#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
81#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
82#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
83#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
84#include "llvm/CodeGen/RegisterBank.h"
85#include "llvm/IR/IntrinsicsAMDGPU.h"
86
87#define GET_TARGET_REGBANK_IMPL
88#include "AMDGPUGenRegisterBank.inc"
89
90// This file will be TableGen'ed at some point.
91#include "AMDGPUGenRegisterBankInfo.def"
92
93using namespace llvm;
94using namespace MIPatternMatch;
95
96namespace {
97
98// Observer to apply a register bank to new registers created by LegalizerHelper.
99class ApplyRegBankMapping final : public GISelChangeObserver {
100private:
101 MachineIRBuilder &B;
102 const AMDGPURegisterBankInfo &RBI;
103 MachineRegisterInfo &MRI;
104 const RegisterBank *NewBank;
105 SmallVector<MachineInstr *, 4> NewInsts;
106
107public:
108 ApplyRegBankMapping(MachineIRBuilder &B, const AMDGPURegisterBankInfo &RBI_,
109 MachineRegisterInfo &MRI_, const RegisterBank *RB)
110 : B(B), RBI(RBI_), MRI(MRI_), NewBank(RB) {
111 assert(!B.isObservingChanges());
112 B.setChangeObserver(*this);
113 }
114
115 ~ApplyRegBankMapping() override {
116 for (MachineInstr *MI : NewInsts)
117 applyBank(MI&: *MI);
118
119 B.stopObservingChanges();
120 }
121
122 /// Set any registers that don't have a set register class or bank to SALU.
123 void applyBank(MachineInstr &MI) {
124 const unsigned Opc = MI.getOpcode();
125 if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT ||
126 Opc == AMDGPU::G_SEXT) {
127 // LegalizerHelper wants to use the basic legalization artifacts when
128 // widening etc. We don't handle selection with vcc in artifact sources,
129 // so we need to use a select instead to handle these properly.
130 Register DstReg = MI.getOperand(i: 0).getReg();
131 Register SrcReg = MI.getOperand(i: 1).getReg();
132 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI, TRI: *RBI.TRI);
133 if (SrcBank == &AMDGPU::VCCRegBank) {
134 const LLT S32 = LLT::scalar(SizeInBits: 32);
135 assert(MRI.getType(SrcReg) == LLT::scalar(1));
136 assert(MRI.getType(DstReg) == S32);
137 assert(NewBank == &AMDGPU::VGPRRegBank);
138
139 // Replace the extension with a select, which really uses the boolean
140 // source.
141 B.setInsertPt(MBB&: *MI.getParent(), II: MI);
142
143 auto True = B.buildConstant(Res: S32, Val: Opc == AMDGPU::G_SEXT ? -1 : 1);
144 auto False = B.buildConstant(Res: S32, Val: 0);
145 B.buildSelect(Res: DstReg, Tst: SrcReg, Op0: True, Op1: False);
146 MRI.setRegBank(Reg: True.getReg(Idx: 0), RegBank: *NewBank);
147 MRI.setRegBank(Reg: False.getReg(Idx: 0), RegBank: *NewBank);
148 MI.eraseFromParent();
149 }
150
151 assert(!MRI.getRegClassOrRegBank(DstReg));
152 MRI.setRegBank(Reg: DstReg, RegBank: *NewBank);
153 return;
154 }
155
156#ifndef NDEBUG
157 if (Opc == AMDGPU::G_TRUNC) {
158 Register DstReg = MI.getOperand(0).getReg();
159 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI);
160 assert(DstBank != &AMDGPU::VCCRegBank);
161 }
162#endif
163
164 for (MachineOperand &Op : MI.operands()) {
165 if (!Op.isReg())
166 continue;
167
168 // We may see physical registers if building a real MI
169 Register Reg = Op.getReg();
170 if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg))
171 continue;
172
173 const RegisterBank *RB = NewBank;
174 if (MRI.getType(Reg) == LLT::scalar(SizeInBits: 1)) {
175 assert(NewBank == &AMDGPU::VGPRRegBank &&
176 "s1 operands should only be used for vector bools");
177 assert((MI.getOpcode() != AMDGPU::G_TRUNC &&
178 MI.getOpcode() != AMDGPU::G_ANYEXT) &&
179 "not expecting legalization artifacts here");
180 RB = &AMDGPU::VCCRegBank;
181 }
182
183 MRI.setRegBank(Reg, RegBank: *RB);
184 }
185 }
186
187 void erasingInstr(MachineInstr &MI) override {}
188
189 void createdInstr(MachineInstr &MI) override {
190 // At this point, the instruction was just inserted and has no operands.
191 NewInsts.push_back(Elt: &MI);
192 }
193
194 void changingInstr(MachineInstr &MI) override {}
195 void changedInstr(MachineInstr &MI) override {
196 // FIXME: In principle we should probably add the instruction to NewInsts,
197 // but the way the LegalizerHelper uses the observer, we will always see the
198 // registers we need to set the regbank on also referenced in a new
199 // instruction.
200 }
201};
202
203} // anonymous namespace
204
205AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
206 : Subtarget(ST), TRI(Subtarget.getRegisterInfo()),
207 TII(Subtarget.getInstrInfo()) {
208
209 // HACK: Until this is fully tablegen'd.
210 static llvm::once_flag InitializeRegisterBankFlag;
211
212 static auto InitializeRegisterBankOnce = [this]() {
213 assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank &&
214 &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank &&
215 &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank);
216 (void)this;
217 };
218
219 llvm::call_once(flag&: InitializeRegisterBankFlag, F&: InitializeRegisterBankOnce);
220}
221
222static bool isVectorRegisterBank(const RegisterBank &Bank) {
223 unsigned BankID = Bank.getID();
224 return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID;
225}
226
227bool AMDGPURegisterBankInfo::isDivergentRegBank(const RegisterBank *RB) const {
228 return RB != &AMDGPU::SGPRRegBank;
229}
230
231unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
232 const RegisterBank &Src,
233 TypeSize Size) const {
234 // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane?
235 if (Dst.getID() == AMDGPU::SGPRRegBankID &&
236 (isVectorRegisterBank(Bank: Src) || Src.getID() == AMDGPU::VCCRegBankID)) {
237 return std::numeric_limits<unsigned>::max();
238 }
239
240 // Bool values are tricky, because the meaning is based on context. The SCC
241 // and VCC banks are for the natural scalar and vector conditions produced by
242 // a compare.
243 //
244 // Legalization doesn't know about the necessary context, so an s1 use may
245 // have been a truncate from an arbitrary value, in which case a copy (lowered
246 // as a compare with 0) needs to be inserted.
247 if (Size == 1 &&
248 (Dst.getID() == AMDGPU::SGPRRegBankID) &&
249 (isVectorRegisterBank(Bank: Src) ||
250 Src.getID() == AMDGPU::SGPRRegBankID ||
251 Src.getID() == AMDGPU::VCCRegBankID))
252 return std::numeric_limits<unsigned>::max();
253
254 // There is no direct copy between AGPRs.
255 if (Dst.getID() == AMDGPU::AGPRRegBankID &&
256 Src.getID() == AMDGPU::AGPRRegBankID)
257 return 4;
258
259 return RegisterBankInfo::copyCost(A: Dst, B: Src, Size);
260}
261
262unsigned AMDGPURegisterBankInfo::getBreakDownCost(
263 const ValueMapping &ValMapping,
264 const RegisterBank *CurBank) const {
265 // Check if this is a breakdown for G_LOAD to move the pointer from SGPR to
266 // VGPR.
267 // FIXME: Is there a better way to do this?
268 if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64)
269 return 10; // This is expensive.
270
271 assert(ValMapping.NumBreakDowns == 2 &&
272 ValMapping.BreakDown[0].Length == 32 &&
273 ValMapping.BreakDown[0].StartIdx == 0 &&
274 ValMapping.BreakDown[1].Length == 32 &&
275 ValMapping.BreakDown[1].StartIdx == 32 &&
276 ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank);
277
278 // 32-bit extract of a 64-bit value is just access of a subregister, so free.
279 // TODO: Cost of 0 hits assert, though it's not clear it's what we really
280 // want.
281
282 // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
283 // alignment restrictions, but this probably isn't important.
284 return 1;
285}
286
287const RegisterBank &
288AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
289 LLT Ty) const {
290 // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a
291 // VCC-like use.
292 if (TRI->isSGPRClass(RC: &RC)) {
293 // FIXME: This probably came from a copy from a physical register, which
294 // should be inferable from the copied to-type. We don't have many boolean
295 // physical register constraints so just assume a normal SGPR for now.
296 if (!Ty.isValid())
297 return AMDGPU::SGPRRegBank;
298
299 return Ty == LLT::scalar(SizeInBits: 1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
300 }
301
302 return TRI->isAGPRClass(RC: &RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank;
303}
304
305template <unsigned NumOps>
306RegisterBankInfo::InstructionMappings
307AMDGPURegisterBankInfo::addMappingFromTable(
308 const MachineInstr &MI, const MachineRegisterInfo &MRI,
309 const std::array<unsigned, NumOps> RegSrcOpIdx,
310 ArrayRef<OpRegBankEntry<NumOps>> Table) const {
311
312 InstructionMappings AltMappings;
313
314 SmallVector<const ValueMapping *, 10> Operands(MI.getNumOperands());
315
316 unsigned Sizes[NumOps];
317 for (unsigned I = 0; I < NumOps; ++I) {
318 Register Reg = MI.getOperand(RegSrcOpIdx[I]).getReg();
319 Sizes[I] = getSizeInBits(Reg, MRI, TRI: *TRI);
320 }
321
322 for (unsigned I = 0, E = MI.getNumExplicitDefs(); I != E; ++I) {
323 unsigned SizeI = getSizeInBits(Reg: MI.getOperand(i: I).getReg(), MRI, TRI: *TRI);
324 Operands[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SizeI);
325 }
326
327 // getInstrMapping's default mapping uses ID 1, so start at 2.
328 unsigned MappingID = 2;
329 for (const auto &Entry : Table) {
330 for (unsigned I = 0; I < NumOps; ++I) {
331 int OpIdx = RegSrcOpIdx[I];
332 Operands[OpIdx] = AMDGPU::getValueMapping(BankID: Entry.RegBanks[I], Size: Sizes[I]);
333 }
334
335 AltMappings.push_back(Elt: &getInstructionMapping(ID: MappingID++, Cost: Entry.Cost,
336 OperandsMapping: getOperandsMapping(OpdsMapping: Operands),
337 NumOperands: Operands.size()));
338 }
339
340 return AltMappings;
341}
342
343RegisterBankInfo::InstructionMappings
344AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(
345 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
346 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
347 case Intrinsic::amdgcn_readlane: {
348 static const OpRegBankEntry<3> Table[2] = {
349 // Perfectly legal.
350 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1 },
351
352 // Need a readfirstlane for the index.
353 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 }
354 };
355
356 const std::array<unsigned, 3> RegSrcOpIdx = { ._M_elems: { 0, 2, 3 } };
357 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, Table);
358 }
359 case Intrinsic::amdgcn_writelane: {
360 static const OpRegBankEntry<4> Table[4] = {
361 // Perfectly legal.
362 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1 },
363
364 // Need readfirstlane of first op
365 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 },
366
367 // Need readfirstlane of second op
368 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 },
369
370 // Need readfirstlane of both ops
371 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 3 }
372 };
373
374 // rsrc, voffset, offset
375 const std::array<unsigned, 4> RegSrcOpIdx = { ._M_elems: { 0, 2, 3, 4 } };
376 return addMappingFromTable<4>(MI, MRI, RegSrcOpIdx, Table);
377 }
378 default:
379 return RegisterBankInfo::getInstrAlternativeMappings(MI);
380 }
381}
382
383RegisterBankInfo::InstructionMappings
384AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(
385 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
386
387 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
388 case Intrinsic::amdgcn_s_buffer_load:
389 case Intrinsic::amdgcn_ptr_s_buffer_load: {
390 static const OpRegBankEntry<2> Table[4] = {
391 // Perfectly legal.
392 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1 },
393
394 // Only need 1 register in loop
395 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 300 },
396
397 // Have to waterfall the resource.
398 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1000 },
399
400 // Have to waterfall the resource, and the offset.
401 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1500 }
402 };
403
404 // rsrc, offset
405 const std::array<unsigned, 2> RegSrcOpIdx = { ._M_elems: { 2, 3 } };
406 return addMappingFromTable<2>(MI, MRI, RegSrcOpIdx, Table);
407 }
408 case Intrinsic::amdgcn_ds_ordered_add:
409 case Intrinsic::amdgcn_ds_ordered_swap: {
410 // VGPR = M0, VGPR
411 static const OpRegBankEntry<3> Table[2] = {
412 // Perfectly legal.
413 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1 },
414
415 // Need a readfirstlane for m0
416 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 }
417 };
418
419 const std::array<unsigned, 3> RegSrcOpIdx = { ._M_elems: { 0, 2, 3 } };
420 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, Table);
421 }
422 case Intrinsic::amdgcn_s_sendmsg:
423 case Intrinsic::amdgcn_s_sendmsghalt: {
424 // FIXME: Should have no register for immediate
425 static const OpRegBankEntry<1> Table[2] = {
426 // Perfectly legal.
427 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 },
428
429 // Need readlane
430 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 3 }
431 };
432
433 const std::array<unsigned, 1> RegSrcOpIdx = { ._M_elems: { 2 } };
434 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx, Table);
435 }
436 default:
437 return RegisterBankInfo::getInstrAlternativeMappings(MI);
438 }
439}
440
441// FIXME: Returns uniform if there's no source value information. This is
442// probably wrong.
443bool AMDGPURegisterBankInfo::isScalarLoadLegal(const MachineInstr &MI) const {
444 if (!MI.hasOneMemOperand())
445 return false;
446
447 const MachineMemOperand *MMO = *MI.memoperands_begin();
448 const unsigned AS = MMO->getAddrSpace();
449 const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS ||
450 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
451 const unsigned MemSize = 8 * MMO->getSize().getValue();
452
453 // Require 4-byte alignment.
454 return (MMO->getAlign() >= Align(4) ||
455 (Subtarget.hasScalarSubwordLoads() &&
456 ((MemSize == 16 && MMO->getAlign() >= Align(2)) ||
457 (MemSize == 8 && MMO->getAlign() >= Align(1))))) &&
458 // Can't do a scalar atomic load.
459 !MMO->isAtomic() &&
460 // Don't use scalar loads for volatile accesses to non-constant address
461 // spaces.
462 (IsConst || !MMO->isVolatile()) &&
463 // Memory must be known constant, or not written before this load.
464 (IsConst || MMO->isInvariant() || (MMO->getFlags() & MONoClobber)) &&
465 AMDGPU::isUniformMMO(MMO);
466}
467
468RegisterBankInfo::InstructionMappings
469AMDGPURegisterBankInfo::getInstrAlternativeMappings(
470 const MachineInstr &MI) const {
471
472 const MachineFunction &MF = *MI.getMF();
473 const MachineRegisterInfo &MRI = MF.getRegInfo();
474
475
476 InstructionMappings AltMappings;
477 switch (MI.getOpcode()) {
478 case TargetOpcode::G_CONSTANT:
479 case TargetOpcode::G_IMPLICIT_DEF: {
480 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
481 if (Size == 1) {
482 static const OpRegBankEntry<1> Table[3] = {
483 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 1 },
484 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 },
485 { .RegBanks: { AMDGPU::VCCRegBankID }, .Cost: 1 }
486 };
487
488 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx: {._M_elems: { 0 }}, Table);
489 }
490
491 [[fallthrough]];
492 }
493 case TargetOpcode::G_FCONSTANT:
494 case TargetOpcode::G_FRAME_INDEX:
495 case TargetOpcode::G_GLOBAL_VALUE: {
496 static const OpRegBankEntry<1> Table[2] = {
497 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 1 },
498 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 }
499 };
500
501 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx: {._M_elems: { 0 }}, Table);
502 }
503 case TargetOpcode::G_AND:
504 case TargetOpcode::G_OR:
505 case TargetOpcode::G_XOR: {
506 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
507
508 if (Size == 1) {
509 // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0.
510 const InstructionMapping &SCCMapping = getInstructionMapping(
511 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
512 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32),
513 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32),
514 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32)}),
515 NumOperands: 3); // Num Operands
516 AltMappings.push_back(Elt: &SCCMapping);
517
518 const InstructionMapping &VCCMapping0 = getInstructionMapping(
519 ID: 2, Cost: 1, OperandsMapping: getOperandsMapping(
520 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size),
521 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size),
522 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size)}),
523 NumOperands: 3); // Num Operands
524 AltMappings.push_back(Elt: &VCCMapping0);
525 return AltMappings;
526 }
527
528 if (Size != 64)
529 break;
530
531 const InstructionMapping &SSMapping = getInstructionMapping(
532 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
533 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
534 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
535 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size)}),
536 NumOperands: 3); // Num Operands
537 AltMappings.push_back(Elt: &SSMapping);
538
539 const InstructionMapping &VVMapping = getInstructionMapping(
540 ID: 2, Cost: 2, OperandsMapping: getOperandsMapping(
541 OpdsMapping: {AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
542 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
543 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size)}),
544 NumOperands: 3); // Num Operands
545 AltMappings.push_back(Elt: &VVMapping);
546 break;
547 }
548 case TargetOpcode::G_LOAD:
549 case TargetOpcode::G_ZEXTLOAD:
550 case TargetOpcode::G_SEXTLOAD: {
551 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
552 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
553 unsigned PtrSize = PtrTy.getSizeInBits();
554 unsigned AS = PtrTy.getAddressSpace();
555
556 if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
557 AS != AMDGPUAS::PRIVATE_ADDRESS) &&
558 isScalarLoadLegal(MI)) {
559 const InstructionMapping &SSMapping = getInstructionMapping(
560 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
561 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
562 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: PtrSize)}),
563 NumOperands: 2); // Num Operands
564 AltMappings.push_back(Elt: &SSMapping);
565 }
566
567 const InstructionMapping &VVMapping = getInstructionMapping(
568 ID: 2, Cost: 1,
569 OperandsMapping: getOperandsMapping(
570 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
571 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize)}),
572 NumOperands: 2); // Num Operands
573 AltMappings.push_back(Elt: &VVMapping);
574
575 // It may be possible to have a vgpr = load sgpr mapping here, because
576 // the mubuf instructions support this kind of load, but probably for only
577 // gfx7 and older. However, the addressing mode matching in the instruction
578 // selector should be able to do a better job of detecting and selecting
579 // these kinds of loads from the vgpr = load vgpr mapping.
580
581 return AltMappings;
582
583 }
584 case TargetOpcode::G_SELECT: {
585 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
586 const InstructionMapping &SSMapping = getInstructionMapping(ID: 1, Cost: 1,
587 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
588 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1),
589 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
590 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size)}),
591 NumOperands: 4); // Num Operands
592 AltMappings.push_back(Elt: &SSMapping);
593
594 const InstructionMapping &VVMapping = getInstructionMapping(ID: 2, Cost: 1,
595 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
596 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1),
597 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
598 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size)}),
599 NumOperands: 4); // Num Operands
600 AltMappings.push_back(Elt: &VVMapping);
601
602 return AltMappings;
603 }
604 case TargetOpcode::G_UADDE:
605 case TargetOpcode::G_USUBE:
606 case TargetOpcode::G_SADDE:
607 case TargetOpcode::G_SSUBE: {
608 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
609 const InstructionMapping &SSMapping = getInstructionMapping(ID: 1, Cost: 1,
610 OperandsMapping: getOperandsMapping(
611 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
612 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1),
613 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
614 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
615 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1)}),
616 NumOperands: 5); // Num Operands
617 AltMappings.push_back(Elt: &SSMapping);
618
619 const InstructionMapping &VVMapping = getInstructionMapping(ID: 2, Cost: 1,
620 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
621 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1),
622 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
623 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
624 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1)}),
625 NumOperands: 5); // Num Operands
626 AltMappings.push_back(Elt: &VVMapping);
627 return AltMappings;
628 }
629 case AMDGPU::G_BRCOND: {
630 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
631
632 // TODO: Change type to 32 for scalar
633 const InstructionMapping &SMapping = getInstructionMapping(
634 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
635 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1), nullptr}),
636 NumOperands: 2); // Num Operands
637 AltMappings.push_back(Elt: &SMapping);
638
639 const InstructionMapping &VMapping = getInstructionMapping(
640 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
641 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1), nullptr }),
642 NumOperands: 2); // Num Operands
643 AltMappings.push_back(Elt: &VMapping);
644 return AltMappings;
645 }
646 case AMDGPU::G_INTRINSIC:
647 case AMDGPU::G_INTRINSIC_CONVERGENT:
648 return getInstrAlternativeMappingsIntrinsic(MI, MRI);
649 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
650 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
651 return getInstrAlternativeMappingsIntrinsicWSideEffects(MI, MRI);
652 default:
653 break;
654 }
655 return RegisterBankInfo::getInstrAlternativeMappings(MI);
656}
657
658void AMDGPURegisterBankInfo::split64BitValueForMapping(
659 MachineIRBuilder &B,
660 SmallVector<Register, 2> &Regs,
661 LLT HalfTy,
662 Register Reg) const {
663 assert(HalfTy.getSizeInBits() == 32);
664 MachineRegisterInfo *MRI = B.getMRI();
665 Register LoLHS = MRI->createGenericVirtualRegister(Ty: HalfTy);
666 Register HiLHS = MRI->createGenericVirtualRegister(Ty: HalfTy);
667 const RegisterBank *Bank = getRegBank(Reg, MRI: *MRI, TRI: *TRI);
668 MRI->setRegBank(Reg: LoLHS, RegBank: *Bank);
669 MRI->setRegBank(Reg: HiLHS, RegBank: *Bank);
670
671 Regs.push_back(Elt: LoLHS);
672 Regs.push_back(Elt: HiLHS);
673
674 B.buildInstr(Opcode: AMDGPU::G_UNMERGE_VALUES)
675 .addDef(RegNo: LoLHS)
676 .addDef(RegNo: HiLHS)
677 .addUse(RegNo: Reg);
678}
679
680/// Replace the current type each register in \p Regs has with \p NewTy
681static void setRegsToType(MachineRegisterInfo &MRI, ArrayRef<Register> Regs,
682 LLT NewTy) {
683 for (Register Reg : Regs) {
684 assert(MRI.getType(Reg).getSizeInBits() == NewTy.getSizeInBits());
685 MRI.setType(VReg: Reg, Ty: NewTy);
686 }
687}
688
689static LLT getHalfSizedType(LLT Ty) {
690 if (Ty.isVector()) {
691 assert(Ty.getElementCount().isKnownMultipleOf(2));
692 return LLT::scalarOrVector(EC: Ty.getElementCount().divideCoefficientBy(RHS: 2),
693 ScalarTy: Ty.getElementType());
694 }
695
696 assert(Ty.getScalarSizeInBits() % 2 == 0);
697 return LLT::scalar(SizeInBits: Ty.getScalarSizeInBits() / 2);
698}
699
700// Build one or more V_READFIRSTLANE_B32 instructions to move the given vector
701// source value into a scalar register.
702Register AMDGPURegisterBankInfo::buildReadFirstLane(MachineIRBuilder &B,
703 MachineRegisterInfo &MRI,
704 Register Src) const {
705 LLT Ty = MRI.getType(Reg: Src);
706 const RegisterBank *Bank = getRegBank(Reg: Src, MRI, TRI: *TRI);
707
708 if (Bank == &AMDGPU::SGPRRegBank)
709 return Src;
710
711 unsigned Bits = Ty.getSizeInBits();
712 assert(Bits % 32 == 0);
713
714 if (Bank != &AMDGPU::VGPRRegBank) {
715 // We need to copy from AGPR to VGPR
716 Src = B.buildCopy(Res: Ty, Op: Src).getReg(Idx: 0);
717 MRI.setRegBank(Reg: Src, RegBank: AMDGPU::VGPRRegBank);
718 }
719
720 LLT S32 = LLT::scalar(SizeInBits: 32);
721 unsigned NumParts = Bits / 32;
722 SmallVector<Register, 8> SrcParts;
723 SmallVector<Register, 8> DstParts;
724
725 if (Bits == 32) {
726 SrcParts.push_back(Elt: Src);
727 } else {
728 auto Unmerge = B.buildUnmerge(Res: S32, Op: Src);
729 for (unsigned i = 0; i < NumParts; ++i)
730 SrcParts.push_back(Elt: Unmerge.getReg(Idx: i));
731 }
732
733 for (unsigned i = 0; i < NumParts; ++i) {
734 Register SrcPart = SrcParts[i];
735 Register DstPart = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
736 MRI.setType(VReg: DstPart, Ty: NumParts == 1 ? Ty : S32);
737
738 const TargetRegisterClass *Constrained =
739 constrainGenericRegister(Reg: SrcPart, RC: AMDGPU::VGPR_32RegClass, MRI);
740 (void)Constrained;
741 assert(Constrained && "Failed to constrain readfirstlane src reg");
742
743 B.buildInstr(Opc: AMDGPU::V_READFIRSTLANE_B32, DstOps: {DstPart}, SrcOps: {SrcPart});
744
745 DstParts.push_back(Elt: DstPart);
746 }
747
748 if (Bits == 32)
749 return DstParts[0];
750
751 Register Dst = B.buildMergeLikeInstr(Res: Ty, Ops: DstParts).getReg(Idx: 0);
752 MRI.setRegBank(Reg: Dst, RegBank: AMDGPU::SGPRRegBank);
753 return Dst;
754}
755
756/// Legalize instruction \p MI where operands in \p OpIndices must be SGPRs. If
757/// any of the required SGPR operands are VGPRs, perform a waterfall loop to
758/// execute the instruction for each unique combination of values in all lanes
759/// in the wave. The block will be split such that rest of the instructions are
760/// moved to a new block.
761///
762/// Essentially performs this loop:
763//
764/// Save Execution Mask
765/// For (Lane : Wavefront) {
766/// Enable Lane, Disable all other lanes
767/// SGPR = read SGPR value for current lane from VGPR
768/// VGPRResult[Lane] = use_op SGPR
769/// }
770/// Restore Execution Mask
771///
772/// There is additional complexity to try for compare values to identify the
773/// unique values used.
774bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
775 MachineIRBuilder &B, iterator_range<MachineBasicBlock::iterator> Range,
776 SmallSet<Register, 4> &SGPROperandRegs) const {
777 // Track use registers which have already been expanded with a readfirstlane
778 // sequence. This may have multiple uses if moving a sequence.
779 DenseMap<Register, Register> WaterfalledRegMap;
780
781 MachineBasicBlock &MBB = B.getMBB();
782 MachineFunction *MF = &B.getMF();
783
784 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
785 const AMDGPU::LaneMaskConstants &LMC =
786 AMDGPU::LaneMaskConstants::get(ST: Subtarget);
787
788#ifndef NDEBUG
789 const int OrigRangeSize = std::distance(Range.begin(), Range.end());
790#endif
791
792 MachineRegisterInfo &MRI = *B.getMRI();
793 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
794 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
795
796 // Don't bother using generic instructions/registers for the exec mask.
797 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF)
798 .addDef(RegNo: InitSaveExecReg);
799
800 Register PhiExec = MRI.createVirtualRegister(RegClass: WaveRC);
801 Register NewExec = MRI.createVirtualRegister(RegClass: WaveRC);
802
803 // To insert the loop we need to split the block. Move everything before this
804 // point to a new block, and insert a new empty block before this instruction.
805 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
806 MachineBasicBlock *BodyBB = MF->CreateMachineBasicBlock();
807 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
808 MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock();
809 MachineFunction::iterator MBBI(MBB);
810 ++MBBI;
811 MF->insert(MBBI, MBB: LoopBB);
812 MF->insert(MBBI, MBB: BodyBB);
813 MF->insert(MBBI, MBB: RestoreExecBB);
814 MF->insert(MBBI, MBB: RemainderBB);
815
816 LoopBB->addSuccessor(Succ: BodyBB);
817 BodyBB->addSuccessor(Succ: RestoreExecBB);
818 BodyBB->addSuccessor(Succ: LoopBB);
819
820 // Move the rest of the block into a new block.
821 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
822 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Range.end(), To: MBB.end());
823
824 MBB.addSuccessor(Succ: LoopBB);
825 RestoreExecBB->addSuccessor(Succ: RemainderBB);
826
827 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
828
829 B.buildInstr(Opcode: TargetOpcode::PHI)
830 .addDef(RegNo: PhiExec)
831 .addReg(RegNo: InitSaveExecReg)
832 .addMBB(MBB: &MBB)
833 .addReg(RegNo: NewExec)
834 .addMBB(MBB: BodyBB);
835
836 const DebugLoc &DL = B.getDL();
837
838 MachineInstr &FirstInst = *Range.begin();
839
840 // Move the instruction into the loop body. Note we moved everything after
841 // Range.end() already into a new block, so Range.end() is no longer valid.
842 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: Range.begin(), To: MBB.end());
843
844 // Figure out the iterator range after splicing the instructions.
845 MachineBasicBlock::iterator NewBegin = FirstInst.getIterator();
846 auto NewEnd = BodyBB->end();
847
848 B.setMBB(*LoopBB);
849
850 LLT S1 = LLT::scalar(SizeInBits: 1);
851 Register CondReg;
852
853 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
854
855 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
856 for (MachineOperand &Op : MI.all_uses()) {
857 Register OldReg = Op.getReg();
858 if (!SGPROperandRegs.count(V: OldReg))
859 continue;
860
861 // See if we already processed this register in another instruction in the
862 // sequence.
863 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
864 if (OldVal != WaterfalledRegMap.end()) {
865 Op.setReg(OldVal->second);
866 continue;
867 }
868
869 Register OpReg = Op.getReg();
870 LLT OpTy = MRI.getType(Reg: OpReg);
871
872 const RegisterBank *OpBank = getRegBank(Reg: OpReg, MRI, TRI: *TRI);
873 if (OpBank != &AMDGPU::VGPRRegBank) {
874 // Insert copy from AGPR to VGPR before the loop.
875 B.setMBB(MBB);
876 OpReg = B.buildCopy(Res: OpTy, Op: OpReg).getReg(Idx: 0);
877 MRI.setRegBank(Reg: OpReg, RegBank: AMDGPU::VGPRRegBank);
878 B.setMBB(*LoopBB);
879 }
880
881 Register CurrentLaneReg = buildReadFirstLane(B, MRI, Src: OpReg);
882
883 // Build the comparison(s).
884 unsigned OpSize = OpTy.getSizeInBits();
885 bool Is64 = OpSize % 64 == 0;
886 unsigned PartSize = Is64 ? 64 : 32;
887 LLT PartTy = LLT::scalar(SizeInBits: PartSize);
888 unsigned NumParts = OpSize / PartSize;
889 SmallVector<Register, 8> OpParts;
890 SmallVector<Register, 8> CurrentLaneParts;
891
892 if (NumParts == 1) {
893 OpParts.push_back(Elt: OpReg);
894 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
895 } else {
896 auto UnmergeOp = B.buildUnmerge(Res: PartTy, Op: OpReg);
897 auto UnmergeCurrentLane = B.buildUnmerge(Res: PartTy, Op: CurrentLaneReg);
898 for (unsigned i = 0; i < NumParts; ++i) {
899 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
900 CurrentLaneParts.push_back(Elt: UnmergeCurrentLane.getReg(Idx: i));
901 MRI.setRegBank(Reg: OpParts[i], RegBank: AMDGPU::VGPRRegBank);
902 MRI.setRegBank(Reg: CurrentLaneParts[i], RegBank: AMDGPU::SGPRRegBank);
903 }
904 }
905
906 for (unsigned i = 0; i < NumParts; ++i) {
907 auto CmpReg = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: CurrentLaneParts[i],
908 Op1: OpParts[i]).getReg(Idx: 0);
909 MRI.setRegBank(Reg: CmpReg, RegBank: AMDGPU::VCCRegBank);
910
911 if (!CondReg) {
912 CondReg = CmpReg;
913 } else {
914 CondReg = B.buildAnd(Dst: S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
915 MRI.setRegBank(Reg: CondReg, RegBank: AMDGPU::VCCRegBank);
916 }
917 }
918
919 Op.setReg(CurrentLaneReg);
920
921 // Make sure we don't re-process this register again.
922 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
923 }
924 }
925
926 // The ballot becomes a no-op during instruction selection.
927 CondReg = B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot,
928 Res: {LLT::scalar(SizeInBits: Subtarget.isWave32() ? 32 : 64)})
929 .addReg(RegNo: CondReg)
930 .getReg(Idx: 0);
931 MRI.setRegClass(Reg: CondReg, RC: WaveRC);
932
933 // Update EXEC, save the original EXEC value to VCC.
934 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
935 .addDef(RegNo: NewExec)
936 .addReg(RegNo: CondReg, Flags: RegState::Kill);
937
938 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
939
940 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
941
942 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
943 B.buildInstr(Opcode: LMC.XorTermOpc)
944 .addDef(RegNo: LMC.ExecReg)
945 .addReg(RegNo: LMC.ExecReg)
946 .addReg(RegNo: NewExec);
947
948 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
949 // s_cbranch_scc0?
950
951 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
952 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
953
954 // Save the EXEC mask before the loop.
955 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExecReg)
956 .addReg(RegNo: LMC.ExecReg);
957
958 // Restore the EXEC mask after the loop.
959 B.setMBB(*RestoreExecBB);
960 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
961
962 // Set the insert point after the original instruction, so any new
963 // instructions will be in the remainder.
964 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
965
966 return true;
967}
968
969// Return any unique registers used by \p MI at \p OpIndices that need to be
970// handled in a waterfall loop. Returns these registers in \p
971// SGPROperandRegs. Returns true if there are any operands to handle and a
972// waterfall loop is necessary.
973bool AMDGPURegisterBankInfo::collectWaterfallOperands(
974 SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI,
975 MachineRegisterInfo &MRI, ArrayRef<unsigned> OpIndices) const {
976 for (unsigned Op : OpIndices) {
977 assert(MI.getOperand(Op).isUse());
978 Register Reg = MI.getOperand(i: Op).getReg();
979 const RegisterBank *OpBank = getRegBank(Reg, MRI, TRI: *TRI);
980 if (OpBank->getID() != AMDGPU::SGPRRegBankID)
981 SGPROperandRegs.insert(V: Reg);
982 }
983
984 // No operands need to be replaced, so no need to loop.
985 return !SGPROperandRegs.empty();
986}
987
988bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
989 MachineIRBuilder &B, MachineInstr &MI, ArrayRef<unsigned> OpIndices) const {
990 // Use a set to avoid extra readfirstlanes in the case where multiple operands
991 // are the same register.
992 SmallSet<Register, 4> SGPROperandRegs;
993
994 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI&: *B.getMRI(), OpIndices))
995 return false;
996
997 MachineBasicBlock::iterator I = MI.getIterator();
998 return executeInWaterfallLoop(B, Range: make_range(x: I, y: std::next(x: I)),
999 SGPROperandRegs);
1000}
1001
1002// Legalize an operand that must be an SGPR by inserting a readfirstlane.
1003void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane(
1004 MachineIRBuilder &B, MachineInstr &MI, unsigned OpIdx) const {
1005 Register Reg = MI.getOperand(i: OpIdx).getReg();
1006 MachineRegisterInfo &MRI = *B.getMRI();
1007 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
1008 if (Bank == &AMDGPU::SGPRRegBank)
1009 return;
1010
1011 Reg = buildReadFirstLane(B, MRI, Src: Reg);
1012 MI.getOperand(i: OpIdx).setReg(Reg);
1013}
1014
1015/// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
1016/// rest will be in the remainder.
1017static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) {
1018 unsigned TotalSize = Ty.getSizeInBits();
1019 if (!Ty.isVector())
1020 return {LLT::scalar(SizeInBits: FirstSize), LLT::scalar(SizeInBits: TotalSize - FirstSize)};
1021
1022 LLT EltTy = Ty.getElementType();
1023 unsigned EltSize = EltTy.getSizeInBits();
1024 assert(FirstSize % EltSize == 0);
1025
1026 unsigned FirstPartNumElts = FirstSize / EltSize;
1027 unsigned RemainderElts = (TotalSize - FirstSize) / EltSize;
1028
1029 return {LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: FirstPartNumElts), ScalarTy: EltTy),
1030 LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: RemainderElts), ScalarTy: EltTy)};
1031}
1032
1033static LLT widen96To128(LLT Ty) {
1034 if (!Ty.isVector())
1035 return LLT::scalar(SizeInBits: 128);
1036
1037 LLT EltTy = Ty.getElementType();
1038 assert(128 % EltTy.getSizeInBits() == 0);
1039 return LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1040}
1041
1042bool AMDGPURegisterBankInfo::applyMappingLoad(
1043 MachineIRBuilder &B,
1044 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1045 MachineInstr &MI) const {
1046 MachineRegisterInfo &MRI = *B.getMRI();
1047 Register DstReg = MI.getOperand(i: 0).getReg();
1048 const LLT LoadTy = MRI.getType(Reg: DstReg);
1049 unsigned LoadSize = LoadTy.getSizeInBits();
1050 MachineMemOperand *MMO = *MI.memoperands_begin();
1051 const unsigned MaxNonSmrdLoadSize = 128;
1052
1053 const RegisterBank *DstBank =
1054 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1055 if (DstBank == &AMDGPU::SGPRRegBank) {
1056 // There are some special cases that we need to look at for 32 bit and 96
1057 // bit SGPR loads otherwise we have nothing to do.
1058 if (LoadSize != 32 && (LoadSize != 96 || Subtarget.hasScalarDwordx3Loads()))
1059 return false;
1060
1061 const unsigned MemSize = 8 * MMO->getSize().getValue();
1062 // Scalar loads of size 8 or 16 bit with proper alignment may be widened to
1063 // 32 bit. Check to see if we need to widen the memory access, 8 or 16 bit
1064 // scalar loads should have a load size of 32 but memory access size of less
1065 // than 32.
1066 if (LoadSize == 32 &&
1067 (MemSize == 32 || LoadTy.isVector() || !isScalarLoadLegal(MI)))
1068 return false;
1069
1070 if (LoadSize == 32 &&
1071 ((MemSize == 8 && MMO->getAlign() >= Align(1)) ||
1072 (MemSize == 16 && MMO->getAlign() >= Align(2))) &&
1073 isScalarLoadLegal(MI) &&
1074 Subtarget.getGeneration() >= AMDGPUSubtarget::GFX12)
1075 return false;
1076
1077 Register PtrReg = MI.getOperand(i: 1).getReg();
1078
1079 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
1080
1081 if (LoadSize == 32) {
1082 // This is an extending load from a sub-dword size. Widen the memory
1083 // access size to 4 bytes and clear the extra high bits appropriately
1084 const LLT S32 = LLT::scalar(SizeInBits: 32);
1085 if (MI.getOpcode() == AMDGPU::G_SEXTLOAD) {
1086 // Must extend the sign bit into higher bits for a G_SEXTLOAD
1087 auto WideLoad = B.buildLoadFromOffset(Dst: S32, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1088 B.buildSExtInReg(Res: MI.getOperand(i: 0), Op: WideLoad, ImmOp: MemSize);
1089 } else if (MI.getOpcode() == AMDGPU::G_ZEXTLOAD) {
1090 // Must extend zero into higher bits with an AND for a G_ZEXTLOAD
1091 auto WideLoad = B.buildLoadFromOffset(Dst: S32, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1092 B.buildZExtInReg(Res: MI.getOperand(i: 0), Op: WideLoad, ImmOp: MemSize);
1093 } else
1094 // We do not need to touch the higher bits for regular loads.
1095 B.buildLoadFromOffset(Dst: MI.getOperand(i: 0), BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1096 } else {
1097 // 96-bit loads are only available for vector loads. We need to split this
1098 // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
1099 if (MMO->getAlign() < Align(16)) {
1100 LegalizerHelper Helper(B.getMF(), ApplyBank, B);
1101 LLT Part64, Part32;
1102 std::tie(args&: Part64, args&: Part32) = splitUnequalType(Ty: LoadTy, FirstSize: 64);
1103 if (Helper.reduceLoadStoreWidth(MI&: cast<GAnyLoad>(Val&: MI), TypeIdx: 0, NarrowTy: Part64) !=
1104 LegalizerHelper::Legalized)
1105 return false;
1106 return true;
1107 }
1108 LLT WiderTy = widen96To128(Ty: LoadTy);
1109 auto WideLoad = B.buildLoadFromOffset(Dst: WiderTy, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1110 if (WiderTy.isScalar()) {
1111 B.buildTrunc(Res: MI.getOperand(i: 0), Op: WideLoad);
1112 } else {
1113 B.buildDeleteTrailingVectorElements(Res: MI.getOperand(i: 0).getReg(),
1114 Op0: WideLoad);
1115 }
1116 }
1117
1118 MI.eraseFromParent();
1119 return true;
1120 }
1121
1122 // 128-bit loads are supported for all instruction types.
1123 if (LoadSize <= MaxNonSmrdLoadSize)
1124 return false;
1125
1126 SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
1127
1128 if (SrcRegs.empty())
1129 SrcRegs.push_back(Elt: MI.getOperand(i: 1).getReg());
1130
1131 // RegBankSelect only emits scalar types, so we need to reset the pointer
1132 // operand to a pointer type.
1133 Register BasePtrReg = SrcRegs[0];
1134 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1135 MRI.setType(VReg: BasePtrReg, Ty: PtrTy);
1136
1137 // The following are the loads not splitted enough during legalization
1138 // because it was not clear they are smem-load or vmem-load
1139 if (AMDGPU::isExtendedGlobalAddrSpace(AS: MMO->getAddrSpace()) ||
1140 MMO->getAddrSpace() == AMDGPUAS::BUFFER_RESOURCE) {
1141 assert(LoadSize % MaxNonSmrdLoadSize == 0);
1142 unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize;
1143 const LLT LoadSplitTy = LoadTy.divide(Factor: NumSplitParts);
1144 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank);
1145 LegalizerHelper Helper(B.getMF(), O, B);
1146 if (LoadTy.isVector()) {
1147 if (Helper.fewerElementsVector(MI, TypeIdx: 0, NarrowTy: LoadSplitTy) !=
1148 LegalizerHelper::Legalized)
1149 return false;
1150 } else {
1151 if (Helper.narrowScalar(MI, TypeIdx: 0, NarrowTy: LoadSplitTy) != LegalizerHelper::Legalized)
1152 return false;
1153 }
1154 }
1155
1156 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
1157 return true;
1158}
1159
1160bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
1161 MachineIRBuilder &B,
1162 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1163 MachineInstr &MI) const {
1164 MachineRegisterInfo &MRI = *B.getMRI();
1165 const MachineFunction &MF = B.getMF();
1166 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1167 const auto &TFI = *ST.getFrameLowering();
1168
1169 // Guard in case the stack growth direction ever changes with scratch
1170 // instructions.
1171 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1172 "Stack grows upwards for AMDGPU");
1173
1174 Register Dst = MI.getOperand(i: 0).getReg();
1175 Register AllocSize = MI.getOperand(i: 1).getReg();
1176 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1177
1178 // When using flat-scratch, the stack offset is unscaled.
1179 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1180 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1181
1182 const RegisterBank *SizeBank = getRegBank(Reg: AllocSize, MRI, TRI: *TRI);
1183
1184 if (SizeBank != &AMDGPU::SGPRRegBank) {
1185 auto WaveReduction =
1186 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {LLT::scalar(SizeInBits: 32)})
1187 .addUse(RegNo: AllocSize)
1188 .addImm(Val: 0);
1189 AllocSize = WaveReduction.getReg(Idx: 0);
1190 }
1191
1192 LLT PtrTy = MRI.getType(Reg: Dst);
1193 LLT IntPtrTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1194
1195 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1196 Register SPReg = Info->getStackPtrOffsetReg();
1197 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
1198
1199 Register ScaledSize = AllocSize;
1200 if (!HasFlatScratch) {
1201 auto WaveSize = B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: WavefrontSizeLog2);
1202 ScaledSize = B.buildShl(Dst: IntPtrTy, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1203 }
1204
1205 auto OldSP = B.buildCopy(Res: PtrTy, Op: SPReg);
1206 if (Alignment > TFI.getStackAlign()) {
1207 const uint64_t ScaledAlignment =
1208 HasFlatScratch ? Alignment.value()
1209 : (Alignment.value() << WavefrontSizeLog2);
1210 const uint64_t StackAlignMask = ScaledAlignment - 1;
1211 auto Tmp1 = B.buildPtrAdd(Res: PtrTy, Op0: OldSP,
1212 Op1: B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: StackAlignMask));
1213 B.buildMaskLowPtrBits(Res: Dst, Op0: Tmp1,
1214 NumBits: (HasFlatScratch
1215 ? Log2(A: Alignment)
1216 : Log2(A: Alignment) + WavefrontSizeLog2));
1217 } else {
1218 B.buildCopy(Res: Dst, Op: OldSP);
1219 }
1220 auto PtrAdd = B.buildPtrAdd(Res: PtrTy, Op0: Dst, Op1: ScaledSize);
1221 B.buildCopy(Res: SPReg, Op: PtrAdd);
1222 MI.eraseFromParent();
1223 return true;
1224}
1225
1226bool AMDGPURegisterBankInfo::applyMappingImage(
1227 MachineIRBuilder &B, MachineInstr &MI,
1228 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1229 int RsrcIdx) const {
1230 const int NumDefs = MI.getNumExplicitDefs();
1231
1232 // The reported argument index is relative to the IR intrinsic call arguments,
1233 // so we need to shift by the number of defs and the intrinsic ID.
1234 RsrcIdx += NumDefs + 1;
1235
1236 // Insert copies to VGPR arguments.
1237 applyDefaultMapping(OpdMapper);
1238
1239 // Fixup any SGPR arguments.
1240 SmallVector<unsigned, 4> SGPRIndexes;
1241 for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) {
1242 if (!MI.getOperand(i: I).isReg())
1243 continue;
1244
1245 // If this intrinsic has a sampler, it immediately follows rsrc.
1246 if (I == RsrcIdx || I == RsrcIdx + 1)
1247 SGPRIndexes.push_back(Elt: I);
1248 }
1249
1250 executeInWaterfallLoop(B, MI, OpIndices: SGPRIndexes);
1251 return true;
1252}
1253
1254// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
1255// the three offsets (voffset, soffset and instoffset)
1256unsigned AMDGPURegisterBankInfo::setBufferOffsets(
1257 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
1258 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) const {
1259 const LLT S32 = LLT::scalar(SizeInBits: 32);
1260 MachineRegisterInfo *MRI = B.getMRI();
1261
1262 if (std::optional<int64_t> Imm =
1263 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI: *MRI)) {
1264 uint32_t SOffset, ImmOffset;
1265 if (TII->splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
1266 VOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1267 SOffsetReg = B.buildConstant(Res: S32, Val: SOffset).getReg(Idx: 0);
1268 InstOffsetVal = ImmOffset;
1269
1270 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1271 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1272 return SOffset + ImmOffset;
1273 }
1274 }
1275
1276 const bool CheckNUW = Subtarget.hasGFX1250Insts();
1277 Register Base;
1278 unsigned Offset;
1279
1280 std::tie(args&: Base, args&: Offset) =
1281 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: CombinedOffset,
1282 /*KnownBits=*/ValueTracking: nullptr,
1283 /*CheckNUW=*/CheckNUW);
1284
1285 uint32_t SOffset, ImmOffset;
1286 if (static_cast<int32_t>(Offset) > 0 &&
1287 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
1288 if (getRegBank(Reg: Base, MRI: *MRI, TRI: *TRI) == &AMDGPU::VGPRRegBank) {
1289 VOffsetReg = Base;
1290 SOffsetReg = B.buildConstant(Res: S32, Val: SOffset).getReg(Idx: 0);
1291 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1292 InstOffsetVal = ImmOffset;
1293 return 0; // XXX - Why is this 0?
1294 }
1295
1296 // If we have SGPR base, we can use it for soffset.
1297 if (SOffset == 0) {
1298 VOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1299 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1300 SOffsetReg = Base;
1301 InstOffsetVal = ImmOffset;
1302 return 0; // XXX - Why is this 0?
1303 }
1304 }
1305
1306 // Handle the variable sgpr + vgpr case.
1307 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI: *MRI);
1308 if (Add && static_cast<int32_t>(Offset) >= 0 &&
1309 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
1310 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI: *MRI);
1311 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI: *MRI);
1312
1313 const RegisterBank *Src0Bank = getRegBank(Reg: Src0, MRI: *MRI, TRI: *TRI);
1314 const RegisterBank *Src1Bank = getRegBank(Reg: Src1, MRI: *MRI, TRI: *TRI);
1315
1316 if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) {
1317 VOffsetReg = Src0;
1318 SOffsetReg = Src1;
1319 return 0;
1320 }
1321
1322 if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) {
1323 VOffsetReg = Src1;
1324 SOffsetReg = Src0;
1325 return 0;
1326 }
1327 }
1328
1329 // Ensure we have a VGPR for the combined offset. This could be an issue if we
1330 // have an SGPR offset and a VGPR resource.
1331 if (getRegBank(Reg: CombinedOffset, MRI: *MRI, TRI: *TRI) == &AMDGPU::VGPRRegBank) {
1332 VOffsetReg = CombinedOffset;
1333 } else {
1334 VOffsetReg = B.buildCopy(Res: S32, Op: CombinedOffset).getReg(Idx: 0);
1335 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1336 }
1337
1338 SOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1339 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1340 return 0;
1341}
1342
1343static unsigned getSBufferLoadCorrespondingBufferLoadOpcode(unsigned Opc) {
1344 switch (Opc) {
1345 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
1346 return AMDGPU::G_AMDGPU_BUFFER_LOAD;
1347 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
1348 return AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
1349 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
1350 return AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE;
1351 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
1352 return AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
1353 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
1354 return AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT;
1355 default:
1356 break;
1357 }
1358 llvm_unreachable("Unexpected s_buffer_load opcode");
1359}
1360
1361bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
1362 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
1363 MachineInstr &MI = OpdMapper.getMI();
1364 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1365
1366 const LLT S32 = LLT::scalar(SizeInBits: 32);
1367 Register Dst = MI.getOperand(i: 0).getReg();
1368 LLT Ty = MRI.getType(Reg: Dst);
1369
1370 const RegisterBank *RSrcBank =
1371 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1372 const RegisterBank *OffsetBank =
1373 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
1374 if (RSrcBank == &AMDGPU::SGPRRegBank &&
1375 OffsetBank == &AMDGPU::SGPRRegBank)
1376 return true; // Legal mapping
1377
1378 // FIXME: 96-bit case was widened during legalize. We need to narrow it back
1379 // here but don't have an MMO.
1380
1381 unsigned LoadSize = Ty.getSizeInBits();
1382 int NumLoads = 1;
1383 if (LoadSize == 256 || LoadSize == 512) {
1384 NumLoads = LoadSize / 128;
1385 Ty = Ty.divide(Factor: NumLoads);
1386 }
1387
1388 // Use the alignment to ensure that the required offsets will fit into the
1389 // immediate offsets.
1390 const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
1391
1392 MachineFunction &MF = B.getMF();
1393
1394 Register SOffset;
1395 Register VOffset;
1396 int64_t ImmOffset = 0;
1397
1398 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
1399 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
1400
1401 // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
1402 // can, but we need to track an MMO for that.
1403 const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
1404 const Align MemAlign(4); // FIXME: ABI type alignment?
1405 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
1406 PtrInfo: MachinePointerInfo(),
1407 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1408 MachineMemOperand::MOInvariant,
1409 Size: MemSize, BaseAlignment: MemAlign);
1410 if (MMOOffset != 0)
1411 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
1412
1413 // If only the offset is divergent, emit a MUBUF buffer load instead. We can
1414 // assume that the buffer is unswizzled.
1415
1416 Register RSrc = MI.getOperand(i: 1).getReg();
1417 Register VIndex = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1418 B.getMRI()->setRegBank(Reg: VIndex, RegBank: AMDGPU::VGPRRegBank);
1419 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
1420
1421 SmallVector<Register, 4> LoadParts(NumLoads);
1422
1423 MachineBasicBlock::iterator MII = MI.getIterator();
1424 MachineInstrSpan Span(MII, &B.getMBB());
1425
1426 for (int i = 0; i < NumLoads; ++i) {
1427 if (NumLoads == 1) {
1428 LoadParts[i] = Dst;
1429 } else {
1430 LoadParts[i] = MRI.createGenericVirtualRegister(Ty);
1431 MRI.setRegBank(Reg: LoadParts[i], RegBank: AMDGPU::VGPRRegBank);
1432 }
1433
1434 if (i != 0)
1435 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16, Size: MemSize);
1436
1437 B.buildInstr(Opcode: getSBufferLoadCorrespondingBufferLoadOpcode(Opc: MI.getOpcode()))
1438 .addDef(RegNo: LoadParts[i]) // vdata
1439 .addUse(RegNo: RSrc) // rsrc
1440 .addUse(RegNo: VIndex) // vindex
1441 .addUse(RegNo: VOffset) // voffset
1442 .addUse(RegNo: SOffset) // soffset
1443 .addImm(Val: ImmOffset + 16 * i) // offset(imm)
1444 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
1445 .addImm(Val: 0) // idxen(imm)
1446 .addMemOperand(MMO: BaseMMO);
1447 }
1448
1449 // TODO: If only the resource is a VGPR, it may be better to execute the
1450 // scalar load in the waterfall loop if the resource is expected to frequently
1451 // be dynamically uniform.
1452 if (RSrcBank != &AMDGPU::SGPRRegBank) {
1453 // Remove the original instruction to avoid potentially confusing the
1454 // waterfall loop logic.
1455 B.setInstr(*Span.begin());
1456 MI.eraseFromParent();
1457
1458 SmallSet<Register, 4> OpsToWaterfall;
1459
1460 OpsToWaterfall.insert(V: RSrc);
1461 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
1462 SGPROperandRegs&: OpsToWaterfall);
1463 }
1464
1465 if (NumLoads != 1) {
1466 if (Ty.isVector())
1467 B.buildConcatVectors(Res: Dst, Ops: LoadParts);
1468 else
1469 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
1470 }
1471
1472 // We removed the instruction earlier with a waterfall loop.
1473 if (RSrcBank == &AMDGPU::SGPRRegBank)
1474 MI.eraseFromParent();
1475
1476 return true;
1477}
1478
1479bool AMDGPURegisterBankInfo::applyMappingBFE(MachineIRBuilder &B,
1480 const OperandsMapper &OpdMapper,
1481 bool Signed) const {
1482 MachineInstr &MI = OpdMapper.getMI();
1483 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1484
1485 // Insert basic copies
1486 applyDefaultMapping(OpdMapper);
1487
1488 Register DstReg = MI.getOperand(i: 0).getReg();
1489 LLT Ty = MRI.getType(Reg: DstReg);
1490
1491 const LLT S32 = LLT::scalar(SizeInBits: 32);
1492
1493 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
1494 Register SrcReg = MI.getOperand(i: FirstOpnd).getReg();
1495 Register OffsetReg = MI.getOperand(i: FirstOpnd + 1).getReg();
1496 Register WidthReg = MI.getOperand(i: FirstOpnd + 2).getReg();
1497
1498 const RegisterBank *DstBank =
1499 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1500 if (DstBank == &AMDGPU::VGPRRegBank) {
1501 if (Ty == S32)
1502 return true;
1503
1504 // There is no 64-bit vgpr bitfield extract instructions so the operation
1505 // is expanded to a sequence of instructions that implement the operation.
1506 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank);
1507
1508 const LLT S64 = LLT::scalar(SizeInBits: 64);
1509 // Shift the source operand so that extracted bits start at bit 0.
1510 auto ShiftOffset = Signed ? B.buildAShr(Dst: S64, Src0: SrcReg, Src1: OffsetReg)
1511 : B.buildLShr(Dst: S64, Src0: SrcReg, Src1: OffsetReg);
1512 auto UnmergeSOffset = B.buildUnmerge(Res: {S32, S32}, Op: ShiftOffset);
1513
1514 // A 64-bit bitfield extract uses the 32-bit bitfield extract instructions
1515 // if the width is a constant.
1516 if (auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: WidthReg, MRI)) {
1517 // Use the 32-bit bitfield extract instruction if the width is a constant.
1518 // Depending on the width size, use either the low or high 32-bits.
1519 auto Zero = B.buildConstant(Res: S32, Val: 0);
1520 auto WidthImm = ConstWidth->Value.getZExtValue();
1521 if (WidthImm <= 32) {
1522 // Use bitfield extract on the lower 32-bit source, and then sign-extend
1523 // or clear the upper 32-bits.
1524 auto Extract =
1525 Signed ? B.buildSbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 0), LSB: Zero, Width: WidthReg)
1526 : B.buildUbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 0), LSB: Zero, Width: WidthReg);
1527 auto Extend =
1528 Signed ? B.buildAShr(Dst: S32, Src0: Extract, Src1: B.buildConstant(Res: S32, Val: 31)) : Zero;
1529 B.buildMergeLikeInstr(Res: DstReg, Ops: {Extract, Extend});
1530 } else {
1531 // Use bitfield extract on upper 32-bit source, and combine with lower
1532 // 32-bit source.
1533 auto UpperWidth = B.buildConstant(Res: S32, Val: WidthImm - 32);
1534 auto Extract =
1535 Signed
1536 ? B.buildSbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 1), LSB: Zero, Width: UpperWidth)
1537 : B.buildUbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 1), LSB: Zero, Width: UpperWidth);
1538 B.buildMergeLikeInstr(Res: DstReg, Ops: {UnmergeSOffset.getReg(Idx: 0), Extract});
1539 }
1540 MI.eraseFromParent();
1541 return true;
1542 }
1543
1544 // Expand to Src >> Offset << (64 - Width) >> (64 - Width) using 64-bit
1545 // operations.
1546 auto ExtShift = B.buildSub(Dst: S32, Src0: B.buildConstant(Res: S32, Val: 64), Src1: WidthReg);
1547 auto SignBit = B.buildShl(Dst: S64, Src0: ShiftOffset, Src1: ExtShift);
1548 if (Signed)
1549 B.buildAShr(Dst: S64, Src0: SignBit, Src1: ExtShift);
1550 else
1551 B.buildLShr(Dst: S64, Src0: SignBit, Src1: ExtShift);
1552 MI.eraseFromParent();
1553 return true;
1554 }
1555
1556 // The scalar form packs the offset and width in a single operand.
1557
1558 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
1559
1560 // Ensure the high bits are clear to insert the offset.
1561 auto OffsetMask = B.buildConstant(Res: S32, Val: maskTrailingOnes<unsigned>(N: 6));
1562 auto ClampOffset = B.buildAnd(Dst: S32, Src0: OffsetReg, Src1: OffsetMask);
1563
1564 // Zeros out the low bits, so don't bother clamping the input value.
1565 auto ShiftWidth = B.buildShl(Dst: S32, Src0: WidthReg, Src1: B.buildConstant(Res: S32, Val: 16));
1566
1567 // Transformation function, pack the offset and width of a BFE into
1568 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1569 // source, bits [5:0] contain the offset and bits [22:16] the width.
1570 auto MergedInputs = B.buildOr(Dst: S32, Src0: ClampOffset, Src1: ShiftWidth);
1571
1572 // TODO: It might be worth using a pseudo here to avoid scc clobber and
1573 // register class constraints.
1574 unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) :
1575 (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64);
1576
1577 auto MIB = B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {SrcReg, MergedInputs});
1578 constrainSelectedInstRegOperands(I&: *MIB, TII: *TII, TRI: *TRI, RBI: *this);
1579
1580 MI.eraseFromParent();
1581 return true;
1582}
1583
1584bool AMDGPURegisterBankInfo::applyMappingMAD_64_32(
1585 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
1586 MachineInstr &MI = OpdMapper.getMI();
1587 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1588
1589 // Insert basic copies.
1590 applyDefaultMapping(OpdMapper);
1591
1592 Register Dst0 = MI.getOperand(i: 0).getReg();
1593 Register Dst1 = MI.getOperand(i: 1).getReg();
1594 Register Src0 = MI.getOperand(i: 2).getReg();
1595 Register Src1 = MI.getOperand(i: 3).getReg();
1596 Register Src2 = MI.getOperand(i: 4).getReg();
1597
1598 if (MRI.getRegBankOrNull(Reg: Src0) == &AMDGPU::VGPRRegBank)
1599 return true;
1600
1601 bool IsUnsigned = MI.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
1602 LLT S1 = LLT::scalar(SizeInBits: 1);
1603 LLT S32 = LLT::scalar(SizeInBits: 32);
1604
1605 bool DstOnValu = MRI.getRegBankOrNull(Reg: Src2) == &AMDGPU::VGPRRegBank;
1606 bool Accumulate = true;
1607
1608 if (!DstOnValu) {
1609 if (mi_match(R: Src2, MRI, P: m_ZeroInt()))
1610 Accumulate = false;
1611 }
1612
1613 // Keep the multiplication on the SALU.
1614 Register DstHi;
1615 Register DstLo = B.buildMul(Dst: S32, Src0, Src1).getReg(Idx: 0);
1616 bool MulHiInVgpr = false;
1617
1618 MRI.setRegBank(Reg: DstLo, RegBank: AMDGPU::SGPRRegBank);
1619
1620 if (Subtarget.hasSMulHi()) {
1621 DstHi = IsUnsigned ? B.buildUMulH(Dst: S32, Src0, Src1).getReg(Idx: 0)
1622 : B.buildSMulH(Dst: S32, Src0, Src1).getReg(Idx: 0);
1623 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::SGPRRegBank);
1624 } else {
1625 Register VSrc0 = B.buildCopy(Res: S32, Op: Src0).getReg(Idx: 0);
1626 Register VSrc1 = B.buildCopy(Res: S32, Op: Src1).getReg(Idx: 0);
1627
1628 MRI.setRegBank(Reg: VSrc0, RegBank: AMDGPU::VGPRRegBank);
1629 MRI.setRegBank(Reg: VSrc1, RegBank: AMDGPU::VGPRRegBank);
1630
1631 DstHi = IsUnsigned ? B.buildUMulH(Dst: S32, Src0: VSrc0, Src1: VSrc1).getReg(Idx: 0)
1632 : B.buildSMulH(Dst: S32, Src0: VSrc0, Src1: VSrc1).getReg(Idx: 0);
1633 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::VGPRRegBank);
1634
1635 if (!DstOnValu) {
1636 DstHi = buildReadFirstLane(B, MRI, Src: DstHi);
1637 } else {
1638 MulHiInVgpr = true;
1639 }
1640 }
1641
1642 // Accumulate and produce the "carry-out" bit.
1643 //
1644 // The "carry-out" is defined as bit 64 of the result when computed as a
1645 // big integer. For unsigned multiply-add, this matches the usual definition
1646 // of carry-out. For signed multiply-add, bit 64 is the sign bit of the
1647 // result, which is determined as:
1648 // sign(Src0 * Src1) + sign(Src2) + carry-out from unsigned 64-bit add
1649 LLT CarryType = DstOnValu ? S1 : S32;
1650 const RegisterBank &CarryBank =
1651 DstOnValu ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
1652 const RegisterBank &DstBank =
1653 DstOnValu ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank;
1654 Register Carry;
1655 Register Zero;
1656
1657 if (!IsUnsigned) {
1658 Zero = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1659 MRI.setRegBank(Reg: Zero,
1660 RegBank: MulHiInVgpr ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank);
1661
1662 Carry = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: MulHiInVgpr ? S1 : S32, Op0: DstHi, Op1: Zero)
1663 .getReg(Idx: 0);
1664 MRI.setRegBank(Reg: Carry, RegBank: MulHiInVgpr ? AMDGPU::VCCRegBank
1665 : AMDGPU::SGPRRegBank);
1666
1667 if (DstOnValu && !MulHiInVgpr) {
1668 Carry = B.buildTrunc(Res: S1, Op: Carry).getReg(Idx: 0);
1669 MRI.setRegBank(Reg: Carry, RegBank: AMDGPU::VCCRegBank);
1670 }
1671 }
1672
1673 if (Accumulate) {
1674 if (DstOnValu) {
1675 DstLo = B.buildCopy(Res: S32, Op: DstLo).getReg(Idx: 0);
1676 DstHi = B.buildCopy(Res: S32, Op: DstHi).getReg(Idx: 0);
1677 MRI.setRegBank(Reg: DstLo, RegBank: AMDGPU::VGPRRegBank);
1678 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::VGPRRegBank);
1679 }
1680
1681 auto Unmerge = B.buildUnmerge(Res: S32, Op: Src2);
1682 Register Src2Lo = Unmerge.getReg(Idx: 0);
1683 Register Src2Hi = Unmerge.getReg(Idx: 1);
1684 MRI.setRegBank(Reg: Src2Lo, RegBank: DstBank);
1685 MRI.setRegBank(Reg: Src2Hi, RegBank: DstBank);
1686
1687 if (!IsUnsigned) {
1688 auto Src2Sign = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: CarryType, Op0: Src2Hi, Op1: Zero);
1689 MRI.setRegBank(Reg: Src2Sign.getReg(Idx: 0), RegBank: CarryBank);
1690
1691 Carry = B.buildXor(Dst: CarryType, Src0: Carry, Src1: Src2Sign).getReg(Idx: 0);
1692 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1693 }
1694
1695 auto AddLo = B.buildUAddo(Res: S32, CarryOut: CarryType, Op0: DstLo, Op1: Src2Lo);
1696 DstLo = AddLo.getReg(Idx: 0);
1697 Register CarryLo = AddLo.getReg(Idx: 1);
1698 MRI.setRegBank(Reg: DstLo, RegBank: DstBank);
1699 MRI.setRegBank(Reg: CarryLo, RegBank: CarryBank);
1700
1701 auto AddHi = B.buildUAdde(Res: S32, CarryOut: CarryType, Op0: DstHi, Op1: Src2Hi, CarryIn: CarryLo);
1702 DstHi = AddHi.getReg(Idx: 0);
1703 MRI.setRegBank(Reg: DstHi, RegBank: DstBank);
1704
1705 Register CarryHi = AddHi.getReg(Idx: 1);
1706 MRI.setRegBank(Reg: CarryHi, RegBank: CarryBank);
1707
1708 if (IsUnsigned) {
1709 Carry = CarryHi;
1710 } else {
1711 Carry = B.buildXor(Dst: CarryType, Src0: Carry, Src1: CarryHi).getReg(Idx: 0);
1712 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1713 }
1714 } else {
1715 if (IsUnsigned) {
1716 Carry = B.buildConstant(Res: CarryType, Val: 0).getReg(Idx: 0);
1717 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1718 }
1719 }
1720
1721 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
1722
1723 if (DstOnValu) {
1724 B.buildCopy(Res: Dst1, Op: Carry);
1725 } else {
1726 B.buildTrunc(Res: Dst1, Op: Carry);
1727 }
1728
1729 MI.eraseFromParent();
1730 return true;
1731}
1732
1733// Return a suitable opcode for extending the operands of Opc when widening.
1734static unsigned getExtendOp(unsigned Opc) {
1735 switch (Opc) {
1736 case TargetOpcode::G_ASHR:
1737 case TargetOpcode::G_SMIN:
1738 case TargetOpcode::G_SMAX:
1739 return TargetOpcode::G_SEXT;
1740 case TargetOpcode::G_LSHR:
1741 case TargetOpcode::G_UMIN:
1742 case TargetOpcode::G_UMAX:
1743 return TargetOpcode::G_ZEXT;
1744 default:
1745 return TargetOpcode::G_ANYEXT;
1746 }
1747}
1748
1749// Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding
1750// any illegal vector extend or unmerge operations.
1751static std::pair<Register, Register>
1752unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) {
1753 const LLT S32 = LLT::scalar(SizeInBits: 32);
1754 auto Bitcast = B.buildBitcast(Dst: S32, Src);
1755
1756 if (ExtOpcode == TargetOpcode::G_SEXT) {
1757 auto ExtLo = B.buildSExtInReg(Res: S32, Op: Bitcast, ImmOp: 16);
1758 auto ShiftHi = B.buildAShr(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 16));
1759 return std::pair(ExtLo.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1760 }
1761
1762 auto ShiftHi = B.buildLShr(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 16));
1763 if (ExtOpcode == TargetOpcode::G_ZEXT) {
1764 auto ExtLo = B.buildAnd(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 0xffff));
1765 return std::pair(ExtLo.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1766 }
1767
1768 assert(ExtOpcode == TargetOpcode::G_ANYEXT);
1769 return std::pair(Bitcast.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1770}
1771
1772// For cases where only a single copy is inserted for matching register banks.
1773// Replace the register in the instruction operand
1774static bool substituteSimpleCopyRegs(
1775 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) {
1776 SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx));
1777 if (!SrcReg.empty()) {
1778 assert(SrcReg.size() == 1);
1779 OpdMapper.getMI().getOperand(i: OpIdx).setReg(SrcReg[0]);
1780 return true;
1781 }
1782
1783 return false;
1784}
1785
1786/// Handle register layout difference for f16 images for some subtargets.
1787Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
1788 MachineRegisterInfo &MRI,
1789 Register Reg) const {
1790 if (!Subtarget.hasUnpackedD16VMem())
1791 return Reg;
1792
1793 const LLT S16 = LLT::scalar(SizeInBits: 16);
1794 LLT StoreVT = MRI.getType(Reg);
1795 if (!StoreVT.isVector() || StoreVT.getElementType() != S16)
1796 return Reg;
1797
1798 auto Unmerge = B.buildUnmerge(Res: S16, Op: Reg);
1799
1800
1801 SmallVector<Register, 4> WideRegs;
1802 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
1803 WideRegs.push_back(Elt: Unmerge.getReg(Idx: I));
1804
1805 const LLT S32 = LLT::scalar(SizeInBits: 32);
1806 int NumElts = StoreVT.getNumElements();
1807
1808 return B.buildMergeLikeInstr(Res: LLT::fixed_vector(NumElements: NumElts, ScalarTy: S32), Ops: WideRegs)
1809 .getReg(Idx: 0);
1810}
1811
1812static std::pair<Register, unsigned>
1813getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
1814 int64_t Const;
1815 if (mi_match(R: Reg, MRI, P: m_ICst(Cst&: Const)))
1816 return std::pair(Register(), Const);
1817
1818 Register Base;
1819 if (mi_match(R: Reg, MRI, P: m_GAdd(L: m_Reg(R&: Base), R: m_ICst(Cst&: Const))))
1820 return std::pair(Base, Const);
1821
1822 // TODO: Handle G_OR used for add case
1823 return std::pair(Reg, 0);
1824}
1825
1826std::pair<Register, unsigned>
1827AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
1828 Register OrigOffset) const {
1829 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: Subtarget);
1830 Register BaseReg;
1831 unsigned ImmOffset;
1832 const LLT S32 = LLT::scalar(SizeInBits: 32);
1833
1834 // TODO: Use AMDGPU::getBaseWithConstantOffset() instead.
1835 std::tie(args&: BaseReg, args&: ImmOffset) = getBaseWithConstantOffset(MRI&: *B.getMRI(),
1836 Reg: OrigOffset);
1837
1838 unsigned C1 = 0;
1839 if (ImmOffset != 0) {
1840 // If the immediate value is too big for the immoffset field, put only bits
1841 // that would normally fit in the immoffset field. The remaining value that
1842 // is copied/added for the voffset field is a large power of 2, and it
1843 // stands more chance of being CSEd with the copy/add for another similar
1844 // load/store.
1845 // However, do not do that rounding down if that is a negative
1846 // number, as it appears to be illegal to have a negative offset in the
1847 // vgpr, even if adding the immediate offset makes it positive.
1848 unsigned Overflow = ImmOffset & ~MaxImm;
1849 ImmOffset -= Overflow;
1850 if (static_cast<int32_t>(Overflow) < 0) {
1851 Overflow += ImmOffset;
1852 ImmOffset = 0;
1853 }
1854
1855 C1 = ImmOffset;
1856 if (Overflow != 0) {
1857 if (!BaseReg)
1858 BaseReg = B.buildConstant(Res: S32, Val: Overflow).getReg(Idx: 0);
1859 else {
1860 auto OverflowVal = B.buildConstant(Res: S32, Val: Overflow);
1861 BaseReg = B.buildAdd(Dst: S32, Src0: BaseReg, Src1: OverflowVal).getReg(Idx: 0);
1862 }
1863 }
1864 }
1865
1866 if (!BaseReg)
1867 BaseReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1868
1869 return {BaseReg, C1};
1870}
1871
1872bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
1873 Register SrcReg) const {
1874 MachineRegisterInfo &MRI = *B.getMRI();
1875 LLT SrcTy = MRI.getType(Reg: SrcReg);
1876 if (SrcTy.getSizeInBits() == 32) {
1877 // Use a v_mov_b32 here to make the exec dependency explicit.
1878 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1879 .addDef(RegNo: DstReg)
1880 .addUse(RegNo: SrcReg);
1881 return constrainGenericRegister(Reg: DstReg, RC: AMDGPU::VGPR_32RegClass, MRI) &&
1882 constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI);
1883 }
1884
1885 Register TmpReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1886 Register TmpReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1887
1888 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1889 .addDef(RegNo: TmpReg0)
1890 .addUse(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
1891 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1892 .addDef(RegNo: TmpReg1)
1893 .addUse(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
1894 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
1895 .addDef(RegNo: DstReg)
1896 .addUse(RegNo: TmpReg0)
1897 .addImm(Val: AMDGPU::sub0)
1898 .addUse(RegNo: TmpReg1)
1899 .addImm(Val: AMDGPU::sub1);
1900
1901 return constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_64RegClass, MRI) &&
1902 constrainGenericRegister(Reg: DstReg, RC: AMDGPU::VReg_64RegClass, MRI);
1903}
1904
1905/// Utility function for pushing dynamic vector indexes with a constant offset
1906/// into waterfall loops.
1907static void reinsertVectorIndexAdd(MachineIRBuilder &B,
1908 MachineInstr &IdxUseInstr,
1909 unsigned OpIdx,
1910 unsigned ConstOffset) {
1911 MachineRegisterInfo &MRI = *B.getMRI();
1912 const LLT S32 = LLT::scalar(SizeInBits: 32);
1913 Register WaterfallIdx = IdxUseInstr.getOperand(i: OpIdx).getReg();
1914 B.setInsertPt(MBB&: *IdxUseInstr.getParent(), II: IdxUseInstr.getIterator());
1915
1916 auto MaterializedOffset = B.buildConstant(Res: S32, Val: ConstOffset);
1917
1918 auto Add = B.buildAdd(Dst: S32, Src0: WaterfallIdx, Src1: MaterializedOffset);
1919 MRI.setRegBank(Reg: MaterializedOffset.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
1920 MRI.setRegBank(Reg: Add.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
1921 IdxUseInstr.getOperand(i: OpIdx).setReg(Add.getReg(Idx: 0));
1922}
1923
1924/// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the
1925/// original 32-bit source value (to be inserted in the low part of the combined
1926/// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit
1927/// value.
1928static void extendLow32IntoHigh32(MachineIRBuilder &B,
1929 Register Hi32Reg, Register Lo32Reg,
1930 unsigned ExtOpc,
1931 const RegisterBank &RegBank,
1932 bool IsBooleanSrc = false) {
1933 if (ExtOpc == AMDGPU::G_ZEXT) {
1934 B.buildConstant(Res: Hi32Reg, Val: 0);
1935 } else if (ExtOpc == AMDGPU::G_SEXT) {
1936 if (IsBooleanSrc) {
1937 // If we know the original source was an s1, the high half is the same as
1938 // the low.
1939 B.buildCopy(Res: Hi32Reg, Op: Lo32Reg);
1940 } else {
1941 // Replicate sign bit from 32-bit extended part.
1942 auto ShiftAmt = B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 31);
1943 B.getMRI()->setRegBank(Reg: ShiftAmt.getReg(Idx: 0), RegBank);
1944 B.buildAShr(Dst: Hi32Reg, Src0: Lo32Reg, Src1: ShiftAmt);
1945 }
1946 } else {
1947 assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension");
1948 B.buildUndef(Res: Hi32Reg);
1949 }
1950}
1951
1952bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect(
1953 MachineIRBuilder &B, MachineInstr &MI,
1954 const OperandsMapper &OpdMapper) const {
1955 MachineRegisterInfo &MRI = *B.getMRI();
1956
1957 Register VecReg = MI.getOperand(i: 1).getReg();
1958 Register Idx = MI.getOperand(i: 2).getReg();
1959
1960 const RegisterBank &IdxBank =
1961 *OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
1962
1963 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
1964
1965 LLT VecTy = MRI.getType(Reg: VecReg);
1966 unsigned EltSize = VecTy.getScalarSizeInBits();
1967 unsigned NumElem = VecTy.getNumElements();
1968
1969 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
1970 IsDivergentIdx, Subtarget: &Subtarget))
1971 return false;
1972
1973 LLT S32 = LLT::scalar(SizeInBits: 32);
1974
1975 const RegisterBank &DstBank =
1976 *OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1977 const RegisterBank &SrcBank =
1978 *OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1979
1980 const RegisterBank &CCBank =
1981 (DstBank == AMDGPU::SGPRRegBank &&
1982 SrcBank == AMDGPU::SGPRRegBank &&
1983 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
1984 : AMDGPU::VCCRegBank;
1985 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(SizeInBits: 1);
1986
1987 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
1988 Idx = B.buildCopy(Res: S32, Op: Idx)->getOperand(i: 0).getReg();
1989 MRI.setRegBank(Reg: Idx, RegBank: AMDGPU::VGPRRegBank);
1990 }
1991
1992 LLT EltTy = VecTy.getScalarType();
1993 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
1994 unsigned NumLanes = DstRegs.size();
1995 if (!NumLanes)
1996 NumLanes = 1;
1997 else
1998 EltTy = MRI.getType(Reg: DstRegs[0]);
1999
2000 auto UnmergeToEltTy = B.buildUnmerge(Res: EltTy, Op: VecReg);
2001 SmallVector<Register, 2> Res(NumLanes);
2002 for (unsigned L = 0; L < NumLanes; ++L)
2003 Res[L] = UnmergeToEltTy.getReg(Idx: L);
2004
2005 for (unsigned I = 1; I < NumElem; ++I) {
2006 auto IC = B.buildConstant(Res: S32, Val: I);
2007 MRI.setRegBank(Reg: IC->getOperand(i: 0).getReg(), RegBank: AMDGPU::SGPRRegBank);
2008 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CCTy, Op0: Idx, Op1: IC);
2009 MRI.setRegBank(Reg: Cmp->getOperand(i: 0).getReg(), RegBank: CCBank);
2010
2011 for (unsigned L = 0; L < NumLanes; ++L) {
2012 auto S = B.buildSelect(Res: EltTy, Tst: Cmp,
2013 Op0: UnmergeToEltTy.getReg(Idx: I * NumLanes + L), Op1: Res[L]);
2014
2015 for (unsigned N : { 0, 2, 3 })
2016 MRI.setRegBank(Reg: S->getOperand(i: N).getReg(), RegBank: DstBank);
2017
2018 Res[L] = S->getOperand(i: 0).getReg();
2019 }
2020 }
2021
2022 for (unsigned L = 0; L < NumLanes; ++L) {
2023 Register DstReg = (NumLanes == 1) ? MI.getOperand(i: 0).getReg() : DstRegs[L];
2024 B.buildCopy(Res: DstReg, Op: Res[L]);
2025 MRI.setRegBank(Reg: DstReg, RegBank: DstBank);
2026 }
2027
2028 MRI.setRegBank(Reg: MI.getOperand(i: 0).getReg(), RegBank: DstBank);
2029 MI.eraseFromParent();
2030
2031 return true;
2032}
2033
2034// Insert a cross regbank copy for a register if it already has a bank that
2035// differs from the one we want to set.
2036static Register constrainRegToBank(MachineRegisterInfo &MRI,
2037 MachineIRBuilder &B, Register &Reg,
2038 const RegisterBank &Bank) {
2039 const RegisterBank *CurrBank = MRI.getRegBankOrNull(Reg);
2040 if (CurrBank && *CurrBank != Bank) {
2041 Register Copy = B.buildCopy(Res: MRI.getType(Reg), Op: Reg).getReg(Idx: 0);
2042 MRI.setRegBank(Reg: Copy, RegBank: Bank);
2043 return Copy;
2044 }
2045
2046 MRI.setRegBank(Reg, RegBank: Bank);
2047 return Reg;
2048}
2049
2050bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect(
2051 MachineIRBuilder &B, MachineInstr &MI,
2052 const OperandsMapper &OpdMapper) const {
2053
2054 MachineRegisterInfo &MRI = *B.getMRI();
2055 Register VecReg = MI.getOperand(i: 1).getReg();
2056 Register Idx = MI.getOperand(i: 3).getReg();
2057
2058 const RegisterBank &IdxBank =
2059 *OpdMapper.getInstrMapping().getOperandMapping(i: 3).BreakDown[0].RegBank;
2060
2061 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
2062
2063 LLT VecTy = MRI.getType(Reg: VecReg);
2064 unsigned EltSize = VecTy.getScalarSizeInBits();
2065 unsigned NumElem = VecTy.getNumElements();
2066
2067 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
2068 IsDivergentIdx, Subtarget: &Subtarget))
2069 return false;
2070
2071 LLT S32 = LLT::scalar(SizeInBits: 32);
2072
2073 const RegisterBank &DstBank =
2074 *OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2075 const RegisterBank &SrcBank =
2076 *OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2077 const RegisterBank &InsBank =
2078 *OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2079
2080 const RegisterBank &CCBank =
2081 (DstBank == AMDGPU::SGPRRegBank &&
2082 SrcBank == AMDGPU::SGPRRegBank &&
2083 InsBank == AMDGPU::SGPRRegBank &&
2084 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
2085 : AMDGPU::VCCRegBank;
2086 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(SizeInBits: 1);
2087
2088 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
2089 Idx = B.buildCopy(Res: S32, Op: Idx)->getOperand(i: 0).getReg();
2090 MRI.setRegBank(Reg: Idx, RegBank: AMDGPU::VGPRRegBank);
2091 }
2092
2093 LLT EltTy = VecTy.getScalarType();
2094 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(OpIdx: 2));
2095 unsigned NumLanes = InsRegs.size();
2096 if (!NumLanes) {
2097 NumLanes = 1;
2098 InsRegs.push_back(Elt: MI.getOperand(i: 2).getReg());
2099 } else {
2100 EltTy = MRI.getType(Reg: InsRegs[0]);
2101 }
2102
2103 auto UnmergeToEltTy = B.buildUnmerge(Res: EltTy, Op: VecReg);
2104 SmallVector<Register, 16> Ops(NumElem * NumLanes);
2105
2106 for (unsigned I = 0; I < NumElem; ++I) {
2107 auto IC = B.buildConstant(Res: S32, Val: I);
2108 MRI.setRegBank(Reg: IC->getOperand(i: 0).getReg(), RegBank: AMDGPU::SGPRRegBank);
2109 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CCTy, Op0: Idx, Op1: IC);
2110 MRI.setRegBank(Reg: Cmp->getOperand(i: 0).getReg(), RegBank: CCBank);
2111
2112 for (unsigned L = 0; L < NumLanes; ++L) {
2113 Register Op0 = constrainRegToBank(MRI, B, Reg&: InsRegs[L], Bank: DstBank);
2114 Register Op1 = UnmergeToEltTy.getReg(Idx: I * NumLanes + L);
2115 Op1 = constrainRegToBank(MRI, B, Reg&: Op1, Bank: DstBank);
2116
2117 Register Select = B.buildSelect(Res: EltTy, Tst: Cmp, Op0, Op1).getReg(Idx: 0);
2118 MRI.setRegBank(Reg: Select, RegBank: DstBank);
2119
2120 Ops[I * NumLanes + L] = Select;
2121 }
2122 }
2123
2124 LLT MergeTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: EltTy);
2125 if (MergeTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg())) {
2126 B.buildBuildVector(Res: MI.getOperand(i: 0), Ops);
2127 } else {
2128 auto Vec = B.buildBuildVector(Res: MergeTy, Ops);
2129 MRI.setRegBank(Reg: Vec->getOperand(i: 0).getReg(), RegBank: DstBank);
2130 B.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: Vec);
2131 }
2132
2133 MRI.setRegBank(Reg: MI.getOperand(i: 0).getReg(), RegBank: DstBank);
2134 MI.eraseFromParent();
2135
2136 return true;
2137}
2138
2139// Break s_mul_u64 into 32-bit vector operations.
2140void AMDGPURegisterBankInfo::applyMappingSMULU64(
2141 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
2142 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2143 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(OpIdx: 1));
2144 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2145
2146 // All inputs are SGPRs, nothing special to do.
2147 if (DefRegs.empty()) {
2148 assert(Src0Regs.empty() && Src1Regs.empty());
2149 applyDefaultMapping(OpdMapper);
2150 return;
2151 }
2152
2153 assert(DefRegs.size() == 2);
2154 assert(Src0Regs.size() == Src1Regs.size() &&
2155 (Src0Regs.empty() || Src0Regs.size() == 2));
2156
2157 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2158 MachineInstr &MI = OpdMapper.getMI();
2159 Register DstReg = MI.getOperand(i: 0).getReg();
2160 LLT HalfTy = LLT::scalar(SizeInBits: 32);
2161
2162 // Depending on where the source registers came from, the generic code may
2163 // have decided to split the inputs already or not. If not, we still need to
2164 // extract the values.
2165
2166 if (Src0Regs.empty())
2167 split64BitValueForMapping(B, Regs&: Src0Regs, HalfTy, Reg: MI.getOperand(i: 1).getReg());
2168 else
2169 setRegsToType(MRI, Regs: Src0Regs, NewTy: HalfTy);
2170
2171 if (Src1Regs.empty())
2172 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2173 else
2174 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2175
2176 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2177
2178 // The multiplication is done as follows:
2179 //
2180 // Op1H Op1L
2181 // * Op0H Op0L
2182 // --------------------
2183 // Op1H*Op0L Op1L*Op0L
2184 // + Op1H*Op0H Op1L*Op0H
2185 // -----------------------------------------
2186 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
2187 //
2188 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
2189 // value and that would overflow.
2190 // The low 32-bit value is Op1L*Op0L.
2191 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from
2192 // Op1L*Op0L).
2193
2194 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank);
2195
2196 Register Hi = B.buildUMulH(Dst: HalfTy, Src0: Src0Regs[0], Src1: Src1Regs[0]).getReg(Idx: 0);
2197 Register MulLoHi = B.buildMul(Dst: HalfTy, Src0: Src0Regs[0], Src1: Src1Regs[1]).getReg(Idx: 0);
2198 Register Add = B.buildAdd(Dst: HalfTy, Src0: Hi, Src1: MulLoHi).getReg(Idx: 0);
2199 Register MulHiLo = B.buildMul(Dst: HalfTy, Src0: Src0Regs[1], Src1: Src1Regs[0]).getReg(Idx: 0);
2200 B.buildAdd(Dst: DefRegs[1], Src0: Add, Src1: MulHiLo);
2201 B.buildMul(Dst: DefRegs[0], Src0: Src0Regs[0], Src1: Src1Regs[0]);
2202
2203 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2204 MI.eraseFromParent();
2205}
2206
2207void AMDGPURegisterBankInfo::applyMappingImpl(
2208 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
2209 MachineInstr &MI = OpdMapper.getMI();
2210 B.setInstrAndDebugLoc(MI);
2211 unsigned Opc = MI.getOpcode();
2212 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2213 switch (Opc) {
2214 case AMDGPU::G_CONSTANT:
2215 case AMDGPU::G_IMPLICIT_DEF: {
2216 Register DstReg = MI.getOperand(i: 0).getReg();
2217 LLT DstTy = MRI.getType(Reg: DstReg);
2218 if (DstTy != LLT::scalar(SizeInBits: 1))
2219 break;
2220
2221 const RegisterBank *DstBank =
2222 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2223 if (DstBank == &AMDGPU::VCCRegBank)
2224 break;
2225 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2226 if (DefRegs.empty())
2227 DefRegs.push_back(Elt: DstReg);
2228
2229 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
2230
2231 Register NewDstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2232 LLVMContext &Ctx = B.getMF().getFunction().getContext();
2233
2234 MI.getOperand(i: 0).setReg(NewDstReg);
2235 if (Opc != AMDGPU::G_IMPLICIT_DEF) {
2236 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
2237 MI.getOperand(i: 1).setCImm(
2238 ConstantInt::get(Ty: IntegerType::getInt32Ty(C&: Ctx), V: ConstVal));
2239 }
2240
2241 MRI.setRegBank(Reg: NewDstReg, RegBank: *DstBank);
2242 B.buildTrunc(Res: DefRegs[0], Op: NewDstReg);
2243 return;
2244 }
2245 case AMDGPU::G_PHI: {
2246 Register DstReg = MI.getOperand(i: 0).getReg();
2247 LLT DstTy = MRI.getType(Reg: DstReg);
2248 if (DstTy != LLT::scalar(SizeInBits: 1))
2249 break;
2250
2251 const LLT S32 = LLT::scalar(SizeInBits: 32);
2252 const RegisterBank *DstBank =
2253 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2254 if (DstBank == &AMDGPU::VCCRegBank) {
2255 applyDefaultMapping(OpdMapper);
2256 // The standard handling only considers the result register bank for
2257 // phis. For VCC, blindly inserting a copy when the phi is lowered will
2258 // produce an invalid copy. We can only copy with some kind of compare to
2259 // get a vector boolean result. Insert a register bank copy that will be
2260 // correctly lowered to a compare.
2261 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2262 Register SrcReg = MI.getOperand(i: I).getReg();
2263 const RegisterBank *SrcBank = getRegBank(Reg: SrcReg, MRI, TRI: *TRI);
2264
2265 if (SrcBank != &AMDGPU::VCCRegBank) {
2266 MachineBasicBlock *SrcMBB = MI.getOperand(i: I + 1).getMBB();
2267 B.setInsertPt(MBB&: *SrcMBB, II: SrcMBB->getFirstTerminator());
2268
2269 auto Copy = B.buildCopy(Res: LLT::scalar(SizeInBits: 1), Op: SrcReg);
2270 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: AMDGPU::VCCRegBank);
2271 MI.getOperand(i: I).setReg(Copy.getReg(Idx: 0));
2272 }
2273 }
2274
2275 return;
2276 }
2277
2278 // Phi handling is strange and only considers the bank of the destination.
2279 substituteSimpleCopyRegs(OpdMapper, OpIdx: 0);
2280
2281 // Promote SGPR/VGPR booleans to s32
2282 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
2283 B.setInsertPt(MBB&: B.getMBB(), II: MI);
2284 LegalizerHelper Helper(B.getMF(), ApplyBank, B);
2285
2286 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2287 llvm_unreachable("widen scalar should have succeeded");
2288
2289 return;
2290 }
2291 case AMDGPU::G_FCMP:
2292 if (!Subtarget.hasSALUFloatInsts())
2293 break;
2294 [[fallthrough]];
2295 case AMDGPU::G_ICMP:
2296 case AMDGPU::G_UADDO:
2297 case AMDGPU::G_USUBO:
2298 case AMDGPU::G_UADDE:
2299 case AMDGPU::G_SADDE:
2300 case AMDGPU::G_USUBE:
2301 case AMDGPU::G_SSUBE: {
2302 unsigned BoolDstOp =
2303 (Opc == AMDGPU::G_ICMP || Opc == AMDGPU::G_FCMP) ? 0 : 1;
2304 Register DstReg = MI.getOperand(i: BoolDstOp).getReg();
2305
2306 const RegisterBank *DstBank =
2307 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2308 if (DstBank != &AMDGPU::SGPRRegBank)
2309 break;
2310
2311 const bool HasCarryIn = MI.getNumOperands() == 5;
2312
2313 // If this is a scalar compare, promote the result to s32, as the selection
2314 // will end up using a copy to a 32-bit vreg.
2315 const LLT S32 = LLT::scalar(SizeInBits: 32);
2316 Register NewDstReg = MRI.createGenericVirtualRegister(Ty: S32);
2317 MRI.setRegBank(Reg: NewDstReg, RegBank: AMDGPU::SGPRRegBank);
2318 MI.getOperand(i: BoolDstOp).setReg(NewDstReg);
2319
2320 if (HasCarryIn) {
2321 Register NewSrcReg = MRI.createGenericVirtualRegister(Ty: S32);
2322 MRI.setRegBank(Reg: NewSrcReg, RegBank: AMDGPU::SGPRRegBank);
2323 B.buildZExt(Res: NewSrcReg, Op: MI.getOperand(i: 4).getReg());
2324 MI.getOperand(i: 4).setReg(NewSrcReg);
2325 }
2326
2327 MachineBasicBlock *MBB = MI.getParent();
2328 B.setInsertPt(MBB&: *MBB, II: std::next(x: MI.getIterator()));
2329
2330 // If we had a constrained VCC result register, a copy was inserted to VCC
2331 // from SGPR.
2332 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2333 if (DefRegs.empty())
2334 DefRegs.push_back(Elt: DstReg);
2335 B.buildTrunc(Res: DefRegs[0], Op: NewDstReg);
2336 return;
2337 }
2338 case AMDGPU::G_SELECT: {
2339 Register DstReg = MI.getOperand(i: 0).getReg();
2340 LLT DstTy = MRI.getType(Reg: DstReg);
2341
2342 SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(OpIdx: 1));
2343 if (CondRegs.empty())
2344 CondRegs.push_back(Elt: MI.getOperand(i: 1).getReg());
2345 else {
2346 assert(CondRegs.size() == 1);
2347 }
2348
2349 const RegisterBank *CondBank = getRegBank(Reg: CondRegs[0], MRI, TRI: *TRI);
2350 if (CondBank == &AMDGPU::SGPRRegBank) {
2351 const LLT S32 = LLT::scalar(SizeInBits: 32);
2352 Register NewCondReg = MRI.createGenericVirtualRegister(Ty: S32);
2353 MRI.setRegBank(Reg: NewCondReg, RegBank: AMDGPU::SGPRRegBank);
2354
2355 MI.getOperand(i: 1).setReg(NewCondReg);
2356 B.buildZExt(Res: NewCondReg, Op: CondRegs[0]);
2357 }
2358
2359 if (DstTy.getSizeInBits() != 64)
2360 break;
2361
2362 LLT HalfTy = getHalfSizedType(Ty: DstTy);
2363
2364 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2365 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2366 SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(OpIdx: 3));
2367
2368 // All inputs are SGPRs, nothing special to do.
2369 if (DefRegs.empty()) {
2370 assert(Src1Regs.empty() && Src2Regs.empty());
2371 break;
2372 }
2373
2374 if (Src1Regs.empty())
2375 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2376 else {
2377 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2378 }
2379
2380 if (Src2Regs.empty())
2381 split64BitValueForMapping(B, Regs&: Src2Regs, HalfTy, Reg: MI.getOperand(i: 3).getReg());
2382 else
2383 setRegsToType(MRI, Regs: Src2Regs, NewTy: HalfTy);
2384
2385 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2386
2387 auto Flags = MI.getFlags();
2388 B.buildSelect(Res: DefRegs[0], Tst: CondRegs[0], Op0: Src1Regs[0], Op1: Src2Regs[0], Flags);
2389 B.buildSelect(Res: DefRegs[1], Tst: CondRegs[0], Op0: Src1Regs[1], Op1: Src2Regs[1], Flags);
2390
2391 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2392 MI.eraseFromParent();
2393 return;
2394 }
2395 case AMDGPU::G_BRCOND: {
2396 Register CondReg = MI.getOperand(i: 0).getReg();
2397 // FIXME: Should use legalizer helper, but should change bool ext type.
2398 const RegisterBank *CondBank =
2399 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2400
2401 if (CondBank == &AMDGPU::SGPRRegBank) {
2402 const LLT S32 = LLT::scalar(SizeInBits: 32);
2403 Register NewCondReg = MRI.createGenericVirtualRegister(Ty: S32);
2404 MRI.setRegBank(Reg: NewCondReg, RegBank: AMDGPU::SGPRRegBank);
2405
2406 MI.getOperand(i: 0).setReg(NewCondReg);
2407 B.buildZExt(Res: NewCondReg, Op: CondReg);
2408 return;
2409 }
2410
2411 break;
2412 }
2413 case AMDGPU::G_AND:
2414 case AMDGPU::G_OR:
2415 case AMDGPU::G_XOR: {
2416 // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
2417 // there is a VGPR input.
2418 Register DstReg = MI.getOperand(i: 0).getReg();
2419 LLT DstTy = MRI.getType(Reg: DstReg);
2420
2421 const RegisterBank *DstBank =
2422 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2423
2424 if (DstTy.getSizeInBits() == 1) {
2425 if (DstBank == &AMDGPU::VCCRegBank)
2426 break;
2427
2428 MachineFunction *MF = MI.getMF();
2429 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
2430 LegalizerHelper Helper(*MF, ApplyBank, B);
2431
2432 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: LLT::scalar(SizeInBits: 32)) !=
2433 LegalizerHelper::Legalized)
2434 llvm_unreachable("widen scalar should have succeeded");
2435 return;
2436 }
2437
2438 if (DstTy.getSizeInBits() == 16 && DstBank == &AMDGPU::SGPRRegBank) {
2439 const LLT S32 = LLT::scalar(SizeInBits: 32);
2440 MachineBasicBlock *MBB = MI.getParent();
2441 MachineFunction *MF = MBB->getParent();
2442 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank);
2443 LegalizerHelper Helper(*MF, ApplySALU, B);
2444 // Widen to S32, but handle `G_XOR x, -1` differently. Legalizer widening
2445 // will use a G_ANYEXT to extend the -1 which prevents matching G_XOR -1
2446 // as "not".
2447 if (MI.getOpcode() == AMDGPU::G_XOR &&
2448 mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: -1))) {
2449 Helper.widenScalarSrc(MI, WideTy: S32, OpIdx: 1, ExtOpcode: AMDGPU::G_ANYEXT);
2450 Helper.widenScalarSrc(MI, WideTy: S32, OpIdx: 2, ExtOpcode: AMDGPU::G_SEXT);
2451 Helper.widenScalarDst(MI, WideTy: S32);
2452 } else {
2453 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2454 llvm_unreachable("widen scalar should have succeeded");
2455 }
2456 return;
2457 }
2458
2459 if (DstTy.getSizeInBits() != 64)
2460 break;
2461
2462 LLT HalfTy = getHalfSizedType(Ty: DstTy);
2463 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2464 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(OpIdx: 1));
2465 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2466
2467 // All inputs are SGPRs, nothing special to do.
2468 if (DefRegs.empty()) {
2469 assert(Src0Regs.empty() && Src1Regs.empty());
2470 break;
2471 }
2472
2473 assert(DefRegs.size() == 2);
2474 assert(Src0Regs.size() == Src1Regs.size() &&
2475 (Src0Regs.empty() || Src0Regs.size() == 2));
2476
2477 // Depending on where the source registers came from, the generic code may
2478 // have decided to split the inputs already or not. If not, we still need to
2479 // extract the values.
2480
2481 if (Src0Regs.empty())
2482 split64BitValueForMapping(B, Regs&: Src0Regs, HalfTy, Reg: MI.getOperand(i: 1).getReg());
2483 else
2484 setRegsToType(MRI, Regs: Src0Regs, NewTy: HalfTy);
2485
2486 if (Src1Regs.empty())
2487 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2488 else
2489 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2490
2491 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2492
2493 auto Flags = MI.getFlags();
2494 B.buildInstr(Opc, DstOps: {DefRegs[0]}, SrcOps: {Src0Regs[0], Src1Regs[0]}, Flags);
2495 B.buildInstr(Opc, DstOps: {DefRegs[1]}, SrcOps: {Src0Regs[1], Src1Regs[1]}, Flags);
2496
2497 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2498 MI.eraseFromParent();
2499 return;
2500 }
2501 case AMDGPU::G_ABS: {
2502 Register SrcReg = MI.getOperand(i: 1).getReg();
2503 const RegisterBank *SrcBank = MRI.getRegBankOrNull(Reg: SrcReg);
2504
2505 // There is no VALU abs instruction so we need to replace it with a sub and
2506 // max combination.
2507 if (SrcBank && SrcBank == &AMDGPU::VGPRRegBank) {
2508 MachineFunction *MF = MI.getMF();
2509 ApplyRegBankMapping Apply(B, *this, MRI, &AMDGPU::VGPRRegBank);
2510 LegalizerHelper Helper(*MF, Apply, B);
2511
2512 if (Helper.lowerAbsToMaxNeg(MI) != LegalizerHelper::Legalized)
2513 llvm_unreachable("lowerAbsToMaxNeg should have succeeded");
2514 return;
2515 }
2516 [[fallthrough]];
2517 }
2518 case AMDGPU::G_ADD:
2519 case AMDGPU::G_SUB:
2520 case AMDGPU::G_MUL:
2521 case AMDGPU::G_SHL:
2522 case AMDGPU::G_LSHR:
2523 case AMDGPU::G_ASHR:
2524 case AMDGPU::G_SMIN:
2525 case AMDGPU::G_SMAX:
2526 case AMDGPU::G_UMIN:
2527 case AMDGPU::G_UMAX: {
2528 Register DstReg = MI.getOperand(i: 0).getReg();
2529 LLT DstTy = MRI.getType(Reg: DstReg);
2530
2531 // Special case for s_mul_u64. There is not a vector equivalent of
2532 // s_mul_u64. Hence, we have to break down s_mul_u64 into 32-bit vector
2533 // multiplications.
2534 if (!Subtarget.hasVMulU64Inst() && Opc == AMDGPU::G_MUL &&
2535 DstTy.getSizeInBits() == 64) {
2536 applyMappingSMULU64(B, OpdMapper);
2537 return;
2538 }
2539
2540 // 16-bit operations are VALU only, but can be promoted to 32-bit SALU.
2541 // Packed 16-bit operations need to be scalarized and promoted.
2542 if (DstTy != LLT::scalar(SizeInBits: 16) && DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16))
2543 break;
2544
2545 const RegisterBank *DstBank =
2546 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2547 if (DstBank == &AMDGPU::VGPRRegBank)
2548 break;
2549
2550 const LLT S32 = LLT::scalar(SizeInBits: 32);
2551 MachineBasicBlock *MBB = MI.getParent();
2552 MachineFunction *MF = MBB->getParent();
2553 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank);
2554
2555 if (DstTy.isVector() && Opc == AMDGPU::G_ABS) {
2556 Register WideSrcLo, WideSrcHi;
2557
2558 std::tie(args&: WideSrcLo, args&: WideSrcHi) =
2559 unpackV2S16ToS32(B, Src: MI.getOperand(i: 1).getReg(), ExtOpcode: TargetOpcode::G_SEXT);
2560 auto Lo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {S32}, SrcOps: {WideSrcLo});
2561 auto Hi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {S32}, SrcOps: {WideSrcHi});
2562 B.buildBuildVectorTrunc(Res: DstReg, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
2563 MI.eraseFromParent();
2564 return;
2565 }
2566
2567 if (DstTy.isVector()) {
2568 Register WideSrc0Lo, WideSrc0Hi;
2569 Register WideSrc1Lo, WideSrc1Hi;
2570
2571 unsigned ExtendOp = getExtendOp(Opc: MI.getOpcode());
2572 std::tie(args&: WideSrc0Lo, args&: WideSrc0Hi)
2573 = unpackV2S16ToS32(B, Src: MI.getOperand(i: 1).getReg(), ExtOpcode: ExtendOp);
2574 std::tie(args&: WideSrc1Lo, args&: WideSrc1Hi)
2575 = unpackV2S16ToS32(B, Src: MI.getOperand(i: 2).getReg(), ExtOpcode: ExtendOp);
2576 auto Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {S32}, SrcOps: {WideSrc0Lo, WideSrc1Lo});
2577 auto Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {S32}, SrcOps: {WideSrc0Hi, WideSrc1Hi});
2578 B.buildBuildVectorTrunc(Res: DstReg, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
2579 MI.eraseFromParent();
2580 } else {
2581 LegalizerHelper Helper(*MF, ApplySALU, B);
2582
2583 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2584 llvm_unreachable("widen scalar should have succeeded");
2585
2586 // FIXME: s16 shift amounts should be legal.
2587 if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR ||
2588 Opc == AMDGPU::G_ASHR) {
2589 B.setInsertPt(MBB&: *MBB, II: MI.getIterator());
2590 if (Helper.widenScalar(MI, TypeIdx: 1, WideTy: S32) != LegalizerHelper::Legalized)
2591 llvm_unreachable("widen scalar should have succeeded");
2592 }
2593 }
2594
2595 return;
2596 }
2597 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
2598 case AMDGPU::G_AMDGPU_S_MUL_U64_U32: {
2599 // This is a special case for s_mul_u64. We use
2600 // G_AMDGPU_S_MUL_I64_I32 opcode to represent an s_mul_u64 operation
2601 // where the 33 higher bits are sign-extended and
2602 // G_AMDGPU_S_MUL_U64_U32 opcode to represent an s_mul_u64 operation
2603 // where the 32 higher bits are zero-extended. In case scalar registers are
2604 // selected, both opcodes are lowered as s_mul_u64. If the vector registers
2605 // are selected, then G_AMDGPU_S_MUL_I64_I32 and
2606 // G_AMDGPU_S_MUL_U64_U32 are lowered with a vector mad instruction.
2607
2608 // Insert basic copies.
2609 applyDefaultMapping(OpdMapper);
2610
2611 Register DstReg = MI.getOperand(i: 0).getReg();
2612 Register SrcReg0 = MI.getOperand(i: 1).getReg();
2613 Register SrcReg1 = MI.getOperand(i: 2).getReg();
2614 const LLT S32 = LLT::scalar(SizeInBits: 32);
2615 const LLT S64 = LLT::scalar(SizeInBits: 64);
2616 assert(MRI.getType(DstReg) == S64 && "This is a special case for s_mul_u64 "
2617 "that handles only 64-bit operands.");
2618 const RegisterBank *DstBank =
2619 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2620
2621 // Replace G_AMDGPU_S_MUL_I64_I32 and G_AMDGPU_S_MUL_U64_U32
2622 // with s_mul_u64 operation.
2623 if (DstBank == &AMDGPU::SGPRRegBank) {
2624 MI.setDesc(TII->get(Opcode: AMDGPU::S_MUL_U64));
2625 MRI.setRegClass(Reg: DstReg, RC: &AMDGPU::SGPR_64RegClass);
2626 MRI.setRegClass(Reg: SrcReg0, RC: &AMDGPU::SGPR_64RegClass);
2627 MRI.setRegClass(Reg: SrcReg1, RC: &AMDGPU::SGPR_64RegClass);
2628 return;
2629 }
2630
2631 // Replace G_AMDGPU_S_MUL_I64_I32 and G_AMDGPU_S_MUL_U64_U32
2632 // with a vector mad.
2633 assert(MRI.getRegBankOrNull(DstReg) == &AMDGPU::VGPRRegBank &&
2634 "The destination operand should be in vector registers.");
2635
2636 // Extract the lower subregister from the first operand.
2637 Register Op0L = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2638 MRI.setRegClass(Reg: Op0L, RC: &AMDGPU::VGPR_32RegClass);
2639 MRI.setType(VReg: Op0L, Ty: S32);
2640 B.buildTrunc(Res: Op0L, Op: SrcReg0);
2641
2642 // Extract the lower subregister from the second operand.
2643 Register Op1L = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2644 MRI.setRegClass(Reg: Op1L, RC: &AMDGPU::VGPR_32RegClass);
2645 MRI.setType(VReg: Op1L, Ty: S32);
2646 B.buildTrunc(Res: Op1L, Op: SrcReg1);
2647
2648 unsigned NewOpc = Opc == AMDGPU::G_AMDGPU_S_MUL_U64_U32
2649 ? AMDGPU::G_AMDGPU_MAD_U64_U32
2650 : AMDGPU::G_AMDGPU_MAD_I64_I32;
2651
2652 MachineIRBuilder B(MI);
2653 Register Zero64 = B.buildConstant(Res: S64, Val: 0).getReg(Idx: 0);
2654 MRI.setRegClass(Reg: Zero64, RC: &AMDGPU::VReg_64RegClass);
2655 Register CarryOut = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
2656 MRI.setRegClass(Reg: CarryOut, RC: &AMDGPU::VReg_64RegClass);
2657 B.buildInstr(Opc: NewOpc, DstOps: {DstReg, CarryOut}, SrcOps: {Op0L, Op1L, Zero64});
2658 MI.eraseFromParent();
2659 return;
2660 }
2661 case AMDGPU::G_SEXT_INREG: {
2662 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
2663 if (SrcRegs.empty())
2664 break; // Nothing to repair
2665
2666 const LLT S32 = LLT::scalar(SizeInBits: 32);
2667 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank);
2668
2669 // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs
2670 // we would need to further expand, and doesn't let us directly set the
2671 // result registers.
2672 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
2673
2674 int Amt = MI.getOperand(i: 2).getImm();
2675 if (Amt <= 32) {
2676 // Downstream users have expectations for the high bit behavior, so freeze
2677 // incoming undefined bits.
2678 if (Amt == 32) {
2679 // The low bits are unchanged.
2680 B.buildFreeze(Dst: DstRegs[0], Src: SrcRegs[0]);
2681 } else {
2682 auto Freeze = B.buildFreeze(Dst: S32, Src: SrcRegs[0]);
2683 // Extend in the low bits and propagate the sign bit to the high half.
2684 B.buildSExtInReg(Res: DstRegs[0], Op: Freeze, ImmOp: Amt);
2685 }
2686
2687 B.buildAShr(Dst: DstRegs[1], Src0: DstRegs[0], Src1: B.buildConstant(Res: S32, Val: 31));
2688 } else {
2689 // The low bits are unchanged, and extend in the high bits.
2690 // No freeze required
2691 B.buildCopy(Res: DstRegs[0], Op: SrcRegs[0]);
2692 B.buildSExtInReg(Res: DstRegs[1], Op: DstRegs[0], ImmOp: Amt - 32);
2693 }
2694
2695 Register DstReg = MI.getOperand(i: 0).getReg();
2696 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2697 MI.eraseFromParent();
2698 return;
2699 }
2700 case AMDGPU::G_CTPOP:
2701 case AMDGPU::G_BITREVERSE: {
2702 const RegisterBank *DstBank =
2703 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2704 if (DstBank == &AMDGPU::SGPRRegBank)
2705 break;
2706
2707 Register SrcReg = MI.getOperand(i: 1).getReg();
2708 const LLT S32 = LLT::scalar(SizeInBits: 32);
2709 LLT Ty = MRI.getType(Reg: SrcReg);
2710 if (Ty == S32)
2711 break;
2712
2713 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank);
2714
2715 MachineFunction &MF = B.getMF();
2716 LegalizerHelper Helper(MF, ApplyVALU, B);
2717
2718 if (Helper.narrowScalar(MI, TypeIdx: 1, NarrowTy: S32) != LegalizerHelper::Legalized)
2719 llvm_unreachable("narrowScalar should have succeeded");
2720 return;
2721 }
2722 case AMDGPU::G_AMDGPU_FFBH_U32:
2723 case AMDGPU::G_AMDGPU_FFBL_B32:
2724 case AMDGPU::G_CTLZ_ZERO_POISON:
2725 case AMDGPU::G_CTTZ_ZERO_POISON: {
2726 const RegisterBank *DstBank =
2727 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2728 if (DstBank == &AMDGPU::SGPRRegBank)
2729 break;
2730
2731 Register SrcReg = MI.getOperand(i: 1).getReg();
2732 const LLT S32 = LLT::scalar(SizeInBits: 32);
2733 LLT Ty = MRI.getType(Reg: SrcReg);
2734 if (Ty == S32)
2735 break;
2736
2737 // We can narrow this more efficiently than Helper can by using ffbh/ffbl
2738 // which return -1 when the input is zero:
2739 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
2740 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
2741 // (ffbh hi:lo) -> (umin (ffbh hi), (uaddsat (ffbh lo), 32))
2742 // (ffbl hi:lo) -> (umin (uaddsat (ffbh hi), 32), (ffbh lo))
2743 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank);
2744 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
2745 unsigned NewOpc = Opc == AMDGPU::G_CTLZ_ZERO_POISON
2746 ? (unsigned)AMDGPU::G_AMDGPU_FFBH_U32
2747 : Opc == AMDGPU::G_CTTZ_ZERO_POISON
2748 ? (unsigned)AMDGPU::G_AMDGPU_FFBL_B32
2749 : Opc;
2750 unsigned Idx = NewOpc == AMDGPU::G_AMDGPU_FFBH_U32;
2751 auto X = B.buildInstr(Opc: NewOpc, DstOps: {S32}, SrcOps: {SrcRegs[Idx]});
2752 auto Y = B.buildInstr(Opc: NewOpc, DstOps: {S32}, SrcOps: {SrcRegs[Idx ^ 1]});
2753 unsigned AddOpc =
2754 Opc == AMDGPU::G_CTLZ_ZERO_POISON || Opc == AMDGPU::G_CTTZ_ZERO_POISON
2755 ? AMDGPU::G_ADD
2756 : AMDGPU::G_UADDSAT;
2757 Y = B.buildInstr(Opc: AddOpc, DstOps: {S32}, SrcOps: {Y, B.buildConstant(Res: S32, Val: 32)});
2758 Register DstReg = MI.getOperand(i: 0).getReg();
2759 B.buildUMin(Dst: DstReg, Src0: X, Src1: Y);
2760 MI.eraseFromParent();
2761 return;
2762 }
2763 case AMDGPU::G_SEXT:
2764 case AMDGPU::G_ZEXT:
2765 case AMDGPU::G_ANYEXT: {
2766 Register SrcReg = MI.getOperand(i: 1).getReg();
2767 LLT SrcTy = MRI.getType(Reg: SrcReg);
2768 const bool Signed = Opc == AMDGPU::G_SEXT;
2769
2770 assert(OpdMapper.getVRegs(1).empty());
2771
2772 const RegisterBank *SrcBank =
2773 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2774
2775 Register DstReg = MI.getOperand(i: 0).getReg();
2776 LLT DstTy = MRI.getType(Reg: DstReg);
2777 if (DstTy.isScalar() &&
2778 SrcBank != &AMDGPU::SGPRRegBank &&
2779 SrcBank != &AMDGPU::VCCRegBank &&
2780 // FIXME: Should handle any type that round to s64 when irregular
2781 // breakdowns supported.
2782 DstTy.getSizeInBits() == 64 &&
2783 SrcTy.getSizeInBits() <= 32) {
2784 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2785
2786 // Extend to 32-bit, and then extend the low half.
2787 if (Signed) {
2788 // TODO: Should really be buildSExtOrCopy
2789 B.buildSExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2790 } else if (Opc == AMDGPU::G_ZEXT) {
2791 B.buildZExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2792 } else {
2793 B.buildAnyExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2794 }
2795
2796 extendLow32IntoHigh32(B, Hi32Reg: DefRegs[1], Lo32Reg: DefRegs[0], ExtOpc: Opc, RegBank: *SrcBank);
2797 MRI.setRegBank(Reg: DstReg, RegBank: *SrcBank);
2798 MI.eraseFromParent();
2799 return;
2800 }
2801
2802 if (SrcTy != LLT::scalar(SizeInBits: 1))
2803 return;
2804
2805 // It is not legal to have a legalization artifact with a VCC source. Rather
2806 // than introducing a copy, insert the select we would have to select the
2807 // copy to.
2808 if (SrcBank == &AMDGPU::VCCRegBank) {
2809 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2810
2811 const RegisterBank *DstBank = &AMDGPU::VGPRRegBank;
2812
2813 unsigned DstSize = DstTy.getSizeInBits();
2814 // 64-bit select is SGPR only
2815 const bool UseSel64 = DstSize > 32 &&
2816 SrcBank->getID() == AMDGPU::SGPRRegBankID;
2817
2818 // TODO: Should s16 select be legal?
2819 LLT SelType = UseSel64 ? LLT::scalar(SizeInBits: 64) : LLT::scalar(SizeInBits: 32);
2820 auto True = B.buildConstant(Res: SelType, Val: Signed ? -1 : 1);
2821 auto False = B.buildConstant(Res: SelType, Val: 0);
2822
2823 MRI.setRegBank(Reg: True.getReg(Idx: 0), RegBank: *DstBank);
2824 MRI.setRegBank(Reg: False.getReg(Idx: 0), RegBank: *DstBank);
2825 MRI.setRegBank(Reg: DstReg, RegBank: *DstBank);
2826
2827 if (DstSize > 32) {
2828 B.buildSelect(Res: DefRegs[0], Tst: SrcReg, Op0: True, Op1: False);
2829 extendLow32IntoHigh32(B, Hi32Reg: DefRegs[1], Lo32Reg: DefRegs[0], ExtOpc: Opc, RegBank: *SrcBank, IsBooleanSrc: true);
2830 } else if (DstSize < 32) {
2831 auto Sel = B.buildSelect(Res: SelType, Tst: SrcReg, Op0: True, Op1: False);
2832 MRI.setRegBank(Reg: Sel.getReg(Idx: 0), RegBank: *DstBank);
2833 B.buildTrunc(Res: DstReg, Op: Sel);
2834 } else {
2835 B.buildSelect(Res: DstReg, Tst: SrcReg, Op0: True, Op1: False);
2836 }
2837
2838 MI.eraseFromParent();
2839 return;
2840 }
2841
2842 break;
2843 }
2844 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
2845 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
2846
2847 assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty());
2848
2849 Register DstReg = MI.getOperand(i: 0).getReg();
2850 Register SrcReg = MI.getOperand(i: 1).getReg();
2851
2852 const LLT S32 = LLT::scalar(SizeInBits: 32);
2853 LLT DstTy = MRI.getType(Reg: DstReg);
2854 LLT SrcTy = MRI.getType(Reg: SrcReg);
2855
2856 if (foldExtractEltToCmpSelect(B, MI, OpdMapper))
2857 return;
2858
2859 const ValueMapping &DstMapping
2860 = OpdMapper.getInstrMapping().getOperandMapping(i: 0);
2861 const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank;
2862 const RegisterBank *SrcBank =
2863 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2864 const RegisterBank *IdxBank =
2865 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2866
2867 Register BaseIdxReg;
2868 unsigned ConstOffset;
2869 std::tie(args&: BaseIdxReg, args&: ConstOffset) =
2870 AMDGPU::getBaseWithConstantOffset(MRI, Reg: MI.getOperand(i: 2).getReg());
2871
2872 // See if the index is an add of a constant which will be foldable by moving
2873 // the base register of the index later if this is going to be executed in a
2874 // waterfall loop. This is essentially to reassociate the add of a constant
2875 // with the readfirstlane.
2876 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2877 ConstOffset > 0 &&
2878 ConstOffset < SrcTy.getNumElements();
2879
2880 // Move the base register. We'll re-insert the add later.
2881 if (ShouldMoveIndexIntoLoop)
2882 MI.getOperand(i: 2).setReg(BaseIdxReg);
2883
2884 // If this is a VGPR result only because the index was a VGPR result, the
2885 // actual indexing will be done on the SGPR source vector, which will
2886 // produce a scalar result. We need to copy to the VGPR result inside the
2887 // waterfall loop.
2888 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank &&
2889 SrcBank == &AMDGPU::SGPRRegBank;
2890 if (DstRegs.empty()) {
2891 applyDefaultMapping(OpdMapper);
2892
2893 executeInWaterfallLoop(B, MI, OpIndices: {2});
2894
2895 if (NeedCopyToVGPR) {
2896 // We don't want a phi for this temporary reg.
2897 Register TmpReg = MRI.createGenericVirtualRegister(Ty: DstTy);
2898 MRI.setRegBank(Reg: TmpReg, RegBank: AMDGPU::SGPRRegBank);
2899 MI.getOperand(i: 0).setReg(TmpReg);
2900 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
2901
2902 // Use a v_mov_b32 here to make the exec dependency explicit.
2903 buildVCopy(B, DstReg, SrcReg: TmpReg);
2904 }
2905
2906 // Re-insert the constant offset add inside the waterfall loop.
2907 if (ShouldMoveIndexIntoLoop)
2908 reinsertVectorIndexAdd(B, IdxUseInstr&: MI, OpIdx: 2, ConstOffset);
2909
2910 return;
2911 }
2912
2913 assert(DstTy.getSizeInBits() == 64);
2914
2915 LLT Vec32 = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
2916
2917 auto CastSrc = B.buildBitcast(Dst: Vec32, Src: SrcReg);
2918 auto One = B.buildConstant(Res: S32, Val: 1);
2919
2920 MachineBasicBlock::iterator MII = MI.getIterator();
2921
2922 // Split the vector index into 32-bit pieces. Prepare to move all of the
2923 // new instructions into a waterfall loop if necessary.
2924 //
2925 // Don't put the bitcast or constant in the loop.
2926 MachineInstrSpan Span(MII, &B.getMBB());
2927
2928 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2929 auto IdxLo = B.buildShl(Dst: S32, Src0: BaseIdxReg, Src1: One);
2930 auto IdxHi = B.buildAdd(Dst: S32, Src0: IdxLo, Src1: One);
2931
2932 auto Extract0 = B.buildExtractVectorElement(Res: DstRegs[0], Val: CastSrc, Idx: IdxLo);
2933 auto Extract1 = B.buildExtractVectorElement(Res: DstRegs[1], Val: CastSrc, Idx: IdxHi);
2934
2935 MRI.setRegBank(Reg: DstReg, RegBank: *DstBank);
2936 MRI.setRegBank(Reg: CastSrc.getReg(Idx: 0), RegBank: *SrcBank);
2937 MRI.setRegBank(Reg: One.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2938 MRI.setRegBank(Reg: IdxLo.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2939 MRI.setRegBank(Reg: IdxHi.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2940
2941 SmallSet<Register, 4> OpsToWaterfall;
2942 if (!collectWaterfallOperands(SGPROperandRegs&: OpsToWaterfall, MI, MRI, OpIndices: { 2 })) {
2943 MI.eraseFromParent();
2944 return;
2945 }
2946
2947 // Remove the original instruction to avoid potentially confusing the
2948 // waterfall loop logic.
2949 B.setInstr(*Span.begin());
2950 MI.eraseFromParent();
2951 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
2952 SGPROperandRegs&: OpsToWaterfall);
2953
2954 if (NeedCopyToVGPR) {
2955 MachineBasicBlock *LoopBB = Extract1->getParent();
2956 Register TmpReg0 = MRI.createGenericVirtualRegister(Ty: S32);
2957 Register TmpReg1 = MRI.createGenericVirtualRegister(Ty: S32);
2958 MRI.setRegBank(Reg: TmpReg0, RegBank: AMDGPU::SGPRRegBank);
2959 MRI.setRegBank(Reg: TmpReg1, RegBank: AMDGPU::SGPRRegBank);
2960
2961 Extract0->getOperand(i: 0).setReg(TmpReg0);
2962 Extract1->getOperand(i: 0).setReg(TmpReg1);
2963
2964 B.setInsertPt(MBB&: *LoopBB, II: ++Extract1->getIterator());
2965
2966 buildVCopy(B, DstReg: DstRegs[0], SrcReg: TmpReg0);
2967 buildVCopy(B, DstReg: DstRegs[1], SrcReg: TmpReg1);
2968 }
2969
2970 if (ShouldMoveIndexIntoLoop)
2971 reinsertVectorIndexAdd(B, IdxUseInstr&: *IdxLo, OpIdx: 1, ConstOffset);
2972
2973 return;
2974 }
2975 case AMDGPU::G_INSERT_VECTOR_ELT: {
2976 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(OpIdx: 2));
2977
2978 Register DstReg = MI.getOperand(i: 0).getReg();
2979 LLT VecTy = MRI.getType(Reg: DstReg);
2980
2981 assert(OpdMapper.getVRegs(0).empty());
2982 assert(OpdMapper.getVRegs(3).empty());
2983
2984 if (substituteSimpleCopyRegs(OpdMapper, OpIdx: 1))
2985 MRI.setType(VReg: MI.getOperand(i: 1).getReg(), Ty: VecTy);
2986
2987 if (foldInsertEltToCmpSelect(B, MI, OpdMapper))
2988 return;
2989
2990 const RegisterBank *IdxBank =
2991 OpdMapper.getInstrMapping().getOperandMapping(i: 3).BreakDown[0].RegBank;
2992
2993 Register SrcReg = MI.getOperand(i: 1).getReg();
2994 Register InsReg = MI.getOperand(i: 2).getReg();
2995 LLT InsTy = MRI.getType(Reg: InsReg);
2996 (void)InsTy;
2997
2998 Register BaseIdxReg;
2999 unsigned ConstOffset;
3000 std::tie(args&: BaseIdxReg, args&: ConstOffset) =
3001 AMDGPU::getBaseWithConstantOffset(MRI, Reg: MI.getOperand(i: 3).getReg());
3002
3003 // See if the index is an add of a constant which will be foldable by moving
3004 // the base register of the index later if this is going to be executed in a
3005 // waterfall loop. This is essentially to reassociate the add of a constant
3006 // with the readfirstlane.
3007 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
3008 ConstOffset > 0 &&
3009 ConstOffset < VecTy.getNumElements();
3010
3011 // Move the base register. We'll re-insert the add later.
3012 if (ShouldMoveIndexIntoLoop)
3013 MI.getOperand(i: 3).setReg(BaseIdxReg);
3014
3015
3016 if (InsRegs.empty()) {
3017 executeInWaterfallLoop(B, MI, OpIndices: {3});
3018
3019 // Re-insert the constant offset add inside the waterfall loop.
3020 if (ShouldMoveIndexIntoLoop) {
3021 reinsertVectorIndexAdd(B, IdxUseInstr&: MI, OpIdx: 3, ConstOffset);
3022 }
3023
3024 return;
3025 }
3026
3027 assert(InsTy.getSizeInBits() == 64);
3028
3029 const LLT S32 = LLT::scalar(SizeInBits: 32);
3030 LLT Vec32 = LLT::fixed_vector(NumElements: 2 * VecTy.getNumElements(), ScalarSizeInBits: 32);
3031
3032 auto CastSrc = B.buildBitcast(Dst: Vec32, Src: SrcReg);
3033 auto One = B.buildConstant(Res: S32, Val: 1);
3034
3035 // Split the vector index into 32-bit pieces. Prepare to move all of the
3036 // new instructions into a waterfall loop if necessary.
3037 //
3038 // Don't put the bitcast or constant in the loop.
3039 MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB());
3040
3041 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
3042 auto IdxLo = B.buildShl(Dst: S32, Src0: BaseIdxReg, Src1: One);
3043 auto IdxHi = B.buildAdd(Dst: S32, Src0: IdxLo, Src1: One);
3044
3045 auto InsLo = B.buildInsertVectorElement(Res: Vec32, Val: CastSrc, Elt: InsRegs[0], Idx: IdxLo);
3046 auto InsHi = B.buildInsertVectorElement(Res: Vec32, Val: InsLo, Elt: InsRegs[1], Idx: IdxHi);
3047
3048 const RegisterBank *DstBank =
3049 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
3050 const RegisterBank *SrcBank =
3051 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
3052 const RegisterBank *InsSrcBank =
3053 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
3054
3055 MRI.setRegBank(Reg: InsReg, RegBank: *InsSrcBank);
3056 MRI.setRegBank(Reg: CastSrc.getReg(Idx: 0), RegBank: *SrcBank);
3057 MRI.setRegBank(Reg: InsLo.getReg(Idx: 0), RegBank: *DstBank);
3058 MRI.setRegBank(Reg: InsHi.getReg(Idx: 0), RegBank: *DstBank);
3059 MRI.setRegBank(Reg: One.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
3060 MRI.setRegBank(Reg: IdxLo.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
3061 MRI.setRegBank(Reg: IdxHi.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
3062
3063
3064 SmallSet<Register, 4> OpsToWaterfall;
3065 if (!collectWaterfallOperands(SGPROperandRegs&: OpsToWaterfall, MI, MRI, OpIndices: { 3 })) {
3066 B.setInsertPt(MBB&: B.getMBB(), II: MI);
3067 B.buildBitcast(Dst: DstReg, Src: InsHi);
3068 MI.eraseFromParent();
3069 return;
3070 }
3071
3072 B.setInstr(*Span.begin());
3073 MI.eraseFromParent();
3074
3075 // Figure out the point after the waterfall loop before mangling the control
3076 // flow.
3077 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
3078 SGPROperandRegs&: OpsToWaterfall);
3079
3080 // The insertion point is now right after the original instruction.
3081 //
3082 // Keep the bitcast to the original vector type out of the loop. Doing this
3083 // saved an extra phi we don't need inside the loop.
3084 B.buildBitcast(Dst: DstReg, Src: InsHi);
3085
3086 // Re-insert the constant offset add inside the waterfall loop.
3087 if (ShouldMoveIndexIntoLoop)
3088 reinsertVectorIndexAdd(B, IdxUseInstr&: *IdxLo, OpIdx: 1, ConstOffset);
3089
3090 return;
3091 }
3092 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
3093 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
3094 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
3095 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
3096 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
3097 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
3098 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
3099 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
3100 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
3101 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
3102 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
3103 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
3104 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
3105 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
3106 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
3107 case AMDGPU::G_AMDGPU_BUFFER_STORE:
3108 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
3109 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
3110 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
3111 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16:
3112 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
3113 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: {
3114 applyDefaultMapping(OpdMapper);
3115 executeInWaterfallLoop(B, MI, OpIndices: {1, 4});
3116 return;
3117 }
3118 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
3119 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
3120 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
3121 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
3122 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
3123 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
3124 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
3125 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
3126 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
3127 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
3128 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
3129 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
3130 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
3131 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
3132 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
3133 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
3134 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
3135 applyDefaultMapping(OpdMapper);
3136 executeInWaterfallLoop(B, MI, OpIndices: {2, 5});
3137 return;
3138 }
3139 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
3140 applyDefaultMapping(OpdMapper);
3141 executeInWaterfallLoop(B, MI, OpIndices: {3, 6});
3142 return;
3143 }
3144 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
3145 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
3146 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
3147 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
3148 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
3149 applyMappingSBufferLoad(B, OpdMapper);
3150 return;
3151 }
3152 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
3153 constrainOpWithReadfirstlane(B, MI, OpIdx: 0);
3154 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3155 return;
3156 case AMDGPU::G_INTRINSIC:
3157 case AMDGPU::G_INTRINSIC_CONVERGENT: {
3158 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3159 case Intrinsic::amdgcn_readlane: {
3160 substituteSimpleCopyRegs(OpdMapper, OpIdx: 2);
3161
3162 assert(OpdMapper.getVRegs(0).empty());
3163 assert(OpdMapper.getVRegs(3).empty());
3164
3165 // Make sure the index is an SGPR. It doesn't make sense to run this in a
3166 // waterfall loop, so assume it's a uniform value.
3167 constrainOpWithReadfirstlane(B, MI, OpIdx: 3); // Index
3168 return;
3169 }
3170 case Intrinsic::amdgcn_writelane: {
3171 assert(OpdMapper.getVRegs(0).empty());
3172 assert(OpdMapper.getVRegs(2).empty());
3173 assert(OpdMapper.getVRegs(3).empty());
3174
3175 substituteSimpleCopyRegs(OpdMapper, OpIdx: 4); // VGPR input val
3176 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // Source value
3177 constrainOpWithReadfirstlane(B, MI, OpIdx: 3); // Index
3178 return;
3179 }
3180 case Intrinsic::amdgcn_interp_p1:
3181 case Intrinsic::amdgcn_interp_p2:
3182 case Intrinsic::amdgcn_interp_mov:
3183 case Intrinsic::amdgcn_interp_p1_f16:
3184 case Intrinsic::amdgcn_interp_p2_f16:
3185 case Intrinsic::amdgcn_lds_param_load: {
3186 applyDefaultMapping(OpdMapper);
3187
3188 // Readlane for m0 value, which is always the last operand.
3189 // FIXME: Should this be a waterfall loop instead?
3190 constrainOpWithReadfirstlane(B, MI, OpIdx: MI.getNumOperands() - 1); // Index
3191 return;
3192 }
3193 case Intrinsic::amdgcn_interp_inreg_p10:
3194 case Intrinsic::amdgcn_interp_inreg_p2:
3195 case Intrinsic::amdgcn_interp_inreg_p10_f16:
3196 case Intrinsic::amdgcn_interp_inreg_p2_f16:
3197 case Intrinsic::amdgcn_interp_p10_rtz_f16:
3198 case Intrinsic::amdgcn_interp_p2_rtz_f16:
3199 case Intrinsic::amdgcn_permlane16_swap:
3200 case Intrinsic::amdgcn_permlane32_swap:
3201 applyDefaultMapping(OpdMapper);
3202 return;
3203 case Intrinsic::amdgcn_permlane16:
3204 case Intrinsic::amdgcn_permlanex16: {
3205 // Doing a waterfall loop over these wouldn't make any sense.
3206 substituteSimpleCopyRegs(OpdMapper, OpIdx: 2);
3207 substituteSimpleCopyRegs(OpdMapper, OpIdx: 3);
3208 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3209 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3210 return;
3211 }
3212 case Intrinsic::amdgcn_permlane_bcast:
3213 case Intrinsic::amdgcn_permlane_up:
3214 case Intrinsic::amdgcn_permlane_down:
3215 case Intrinsic::amdgcn_permlane_xor:
3216 // Doing a waterfall loop over these wouldn't make any sense.
3217 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3218 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3219 return;
3220 case Intrinsic::amdgcn_permlane_idx_gen: {
3221 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3222 return;
3223 }
3224 case Intrinsic::amdgcn_sbfe:
3225 applyMappingBFE(B, OpdMapper, Signed: true);
3226 return;
3227 case Intrinsic::amdgcn_ubfe:
3228 applyMappingBFE(B, OpdMapper, Signed: false);
3229 return;
3230 case Intrinsic::amdgcn_inverse_ballot:
3231 case Intrinsic::amdgcn_s_bitreplicate:
3232 case Intrinsic::amdgcn_s_quadmask:
3233 case Intrinsic::amdgcn_s_wqm:
3234 applyDefaultMapping(OpdMapper);
3235 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // Mask
3236 return;
3237 case Intrinsic::amdgcn_ballot:
3238 // Use default handling and insert copy to vcc source.
3239 break;
3240 }
3241 break;
3242 }
3243 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3244 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
3245 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
3246 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
3247 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
3248 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3249 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
3250 assert(RSrcIntrin && RSrcIntrin->IsImage);
3251 // Non-images can have complications from operands that allow both SGPR
3252 // and VGPR. For now it's too complicated to figure out the final opcode
3253 // to derive the register bank from the MCInstrDesc.
3254 applyMappingImage(B, MI, OpdMapper, RsrcIdx: RSrcIntrin->RsrcArg);
3255 return;
3256 }
3257 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
3258 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
3259 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
3260 bool IsDualOrBVH8 =
3261 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
3262 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
3263 unsigned NumMods = IsDualOrBVH8 ? 0 : 1; // Has A16 modifier
3264 unsigned LastRegOpIdx = MI.getNumExplicitOperands() - 1 - NumMods;
3265 applyDefaultMapping(OpdMapper);
3266 executeInWaterfallLoop(B, MI, OpIndices: {LastRegOpIdx});
3267 return;
3268 }
3269 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
3270 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
3271 auto IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3272 switch (IntrID) {
3273 case Intrinsic::amdgcn_ds_ordered_add:
3274 case Intrinsic::amdgcn_ds_ordered_swap: {
3275 // This is only allowed to execute with 1 lane, so readfirstlane is safe.
3276 assert(OpdMapper.getVRegs(0).empty());
3277 substituteSimpleCopyRegs(OpdMapper, OpIdx: 3);
3278 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3279 return;
3280 }
3281 case Intrinsic::amdgcn_ds_gws_init:
3282 case Intrinsic::amdgcn_ds_gws_barrier:
3283 case Intrinsic::amdgcn_ds_gws_sema_br: {
3284 // Only the first lane is executes, so readfirstlane is safe.
3285 substituteSimpleCopyRegs(OpdMapper, OpIdx: 1);
3286 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3287 return;
3288 }
3289 case Intrinsic::amdgcn_ds_gws_sema_v:
3290 case Intrinsic::amdgcn_ds_gws_sema_p:
3291 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
3292 // Only the first lane is executes, so readfirstlane is safe.
3293 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // M0
3294 return;
3295 }
3296 case Intrinsic::amdgcn_ds_append:
3297 case Intrinsic::amdgcn_ds_consume: {
3298 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3299 return;
3300 }
3301 case Intrinsic::amdgcn_s_alloc_vgpr:
3302 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3303 return;
3304 case Intrinsic::amdgcn_s_sendmsg:
3305 case Intrinsic::amdgcn_s_sendmsghalt: {
3306 // FIXME: Should this use a waterfall loop?
3307 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3308 return;
3309 }
3310 case Intrinsic::amdgcn_s_setreg: {
3311 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3312 return;
3313 }
3314 case Intrinsic::amdgcn_s_ttracedata:
3315 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // M0
3316 return;
3317 case Intrinsic::amdgcn_raw_buffer_load_lds:
3318 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3319 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
3320 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: {
3321 applyDefaultMapping(OpdMapper);
3322 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // rsrc
3323 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3324 constrainOpWithReadfirstlane(B, MI, OpIdx: 5); // soffset
3325 return;
3326 }
3327 case Intrinsic::amdgcn_struct_buffer_load_lds:
3328 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3329 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
3330 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
3331 applyDefaultMapping(OpdMapper);
3332 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // rsrc
3333 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3334 constrainOpWithReadfirstlane(B, MI, OpIdx: 6); // soffset
3335 return;
3336 }
3337 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
3338 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
3339 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
3340 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
3341 applyDefaultMapping(OpdMapper);
3342 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3343 return;
3344 }
3345 case Intrinsic::amdgcn_load_to_lds:
3346 case Intrinsic::amdgcn_load_async_to_lds:
3347 case Intrinsic::amdgcn_global_load_lds:
3348 case Intrinsic::amdgcn_global_load_async_lds: {
3349 applyDefaultMapping(OpdMapper);
3350 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3351 return;
3352 }
3353 case Intrinsic::amdgcn_lds_direct_load: {
3354 applyDefaultMapping(OpdMapper);
3355 // Readlane for m0 value, which is always the last operand.
3356 constrainOpWithReadfirstlane(B, MI, OpIdx: MI.getNumOperands() - 1); // Index
3357 return;
3358 }
3359 case Intrinsic::amdgcn_exp_row:
3360 applyDefaultMapping(OpdMapper);
3361 constrainOpWithReadfirstlane(B, MI, OpIdx: 8); // M0
3362 return;
3363 case Intrinsic::amdgcn_cluster_load_b32:
3364 case Intrinsic::amdgcn_cluster_load_b64:
3365 case Intrinsic::amdgcn_cluster_load_b128: {
3366 applyDefaultMapping(OpdMapper);
3367 constrainOpWithReadfirstlane(B, MI, OpIdx: 4); // M0
3368 return;
3369 }
3370 case Intrinsic::amdgcn_s_sleep_var:
3371 assert(OpdMapper.getVRegs(1).empty());
3372 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3373 return;
3374 case Intrinsic::amdgcn_s_barrier_join:
3375 case Intrinsic::amdgcn_s_wakeup_barrier:
3376 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3377 return;
3378 case Intrinsic::amdgcn_s_barrier_init:
3379 case Intrinsic::amdgcn_s_barrier_signal_var:
3380 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3381 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3382 return;
3383 case Intrinsic::amdgcn_s_get_barrier_state:
3384 case Intrinsic::amdgcn_s_get_named_barrier_state: {
3385 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3386 return;
3387 }
3388 case Intrinsic::amdgcn_s_prefetch_data:
3389 case Intrinsic::amdgcn_s_prefetch_inst: {
3390 Register PtrReg = MI.getOperand(i: 1).getReg();
3391 unsigned AS = MRI.getType(Reg: PtrReg).getAddressSpace();
3392 if (AMDGPU::isFlatGlobalAddrSpace(AS)) {
3393 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3394 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3395 } else
3396 MI.eraseFromParent();
3397 return;
3398 }
3399 case Intrinsic::amdgcn_tensor_load_to_lds:
3400 case Intrinsic::amdgcn_tensor_store_from_lds: {
3401 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3402 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3403 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3404 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3405 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3406 return;
3407 }
3408 default: {
3409 if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3410 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
3411 // Non-images can have complications from operands that allow both SGPR
3412 // and VGPR. For now it's too complicated to figure out the final opcode
3413 // to derive the register bank from the MCInstrDesc.
3414 if (RSrcIntrin->IsImage) {
3415 applyMappingImage(B, MI, OpdMapper, RsrcIdx: RSrcIntrin->RsrcArg);
3416 return;
3417 }
3418 }
3419
3420 break;
3421 }
3422 }
3423 break;
3424 }
3425 case AMDGPU::G_SI_CALL: {
3426 // Use a set to avoid extra readfirstlanes in the case where multiple
3427 // operands are the same register.
3428 SmallSet<Register, 4> SGPROperandRegs;
3429
3430 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI, OpIndices: {1}))
3431 break;
3432
3433 // Move all copies to physical SGPRs that are used by the call instruction
3434 // into the loop block. Start searching for these copies until the
3435 // ADJCALLSTACKUP.
3436 unsigned FrameSetupOpcode = AMDGPU::ADJCALLSTACKUP;
3437 unsigned FrameDestroyOpcode = AMDGPU::ADJCALLSTACKDOWN;
3438
3439 // Move all non-copies before the copies, so that a complete range can be
3440 // moved into the waterfall loop.
3441 SmallVector<MachineInstr *, 4> NonCopyInstrs;
3442 // Count of NonCopyInstrs found until the current LastCopy.
3443 unsigned NonCopyInstrsLen = 0;
3444 MachineBasicBlock::iterator Start(&MI);
3445 MachineBasicBlock::iterator LastCopy = Start;
3446 MachineBasicBlock *MBB = MI.getParent();
3447 const SIMachineFunctionInfo *Info =
3448 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
3449 while (Start->getOpcode() != FrameSetupOpcode) {
3450 --Start;
3451 bool IsCopy = false;
3452 if (Start->getOpcode() == AMDGPU::COPY) {
3453 auto &Dst = Start->getOperand(i: 0);
3454 if (Dst.isReg()) {
3455 Register Reg = Dst.getReg();
3456 if (Reg.isPhysical() && MI.readsRegister(Reg, TRI)) {
3457 IsCopy = true;
3458 } else {
3459 // Also move the copy from the scratch rsrc descriptor into the loop
3460 // to allow it to be optimized away.
3461 auto &Src = Start->getOperand(i: 1);
3462 if (Src.isReg()) {
3463 Reg = Src.getReg();
3464 IsCopy = Info->getScratchRSrcReg() == Reg;
3465 }
3466 }
3467 }
3468 }
3469
3470 if (IsCopy) {
3471 LastCopy = Start;
3472 NonCopyInstrsLen = NonCopyInstrs.size();
3473 } else {
3474 NonCopyInstrs.push_back(Elt: &*Start);
3475 }
3476 }
3477 NonCopyInstrs.resize(N: NonCopyInstrsLen);
3478
3479 for (auto *NonCopy : reverse(C&: NonCopyInstrs)) {
3480 MBB->splice(Where: LastCopy, Other: MBB, From: NonCopy->getIterator());
3481 }
3482 Start = LastCopy;
3483
3484 // Do the same for copies after the loop
3485 NonCopyInstrs.clear();
3486 NonCopyInstrsLen = 0;
3487 MachineBasicBlock::iterator End(&MI);
3488 LastCopy = End;
3489 while (End->getOpcode() != FrameDestroyOpcode) {
3490 ++End;
3491 bool IsCopy = false;
3492 if (End->getOpcode() == AMDGPU::COPY) {
3493 auto &Src = End->getOperand(i: 1);
3494 if (Src.isReg()) {
3495 Register Reg = Src.getReg();
3496 IsCopy = Reg.isPhysical() && MI.modifiesRegister(Reg, TRI);
3497 }
3498 }
3499
3500 if (IsCopy) {
3501 LastCopy = End;
3502 NonCopyInstrsLen = NonCopyInstrs.size();
3503 } else {
3504 NonCopyInstrs.push_back(Elt: &*End);
3505 }
3506 }
3507 NonCopyInstrs.resize(N: NonCopyInstrsLen);
3508
3509 End = LastCopy;
3510 ++LastCopy;
3511 for (auto *NonCopy : reverse(C&: NonCopyInstrs)) {
3512 MBB->splice(Where: LastCopy, Other: MBB, From: NonCopy->getIterator());
3513 }
3514
3515 ++End;
3516 B.setInsertPt(MBB&: B.getMBB(), II: Start);
3517 executeInWaterfallLoop(B, Range: make_range(x: Start, y: End), SGPROperandRegs);
3518 break;
3519 }
3520 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
3521 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR:
3522 case AMDGPU::G_LOAD:
3523 case AMDGPU::G_ZEXTLOAD:
3524 case AMDGPU::G_SEXTLOAD: {
3525 if (applyMappingLoad(B, OpdMapper, MI))
3526 return;
3527 break;
3528 }
3529 case AMDGPU::G_DYN_STACKALLOC:
3530 applyMappingDynStackAlloc(B, OpdMapper, MI);
3531 return;
3532 case AMDGPU::G_STACKRESTORE: {
3533 applyDefaultMapping(OpdMapper);
3534 constrainOpWithReadfirstlane(B, MI, OpIdx: 0);
3535 return;
3536 }
3537 case AMDGPU::G_SBFX:
3538 applyMappingBFE(B, OpdMapper, /*Signed*/ true);
3539 return;
3540 case AMDGPU::G_UBFX:
3541 applyMappingBFE(B, OpdMapper, /*Signed*/ false);
3542 return;
3543 case AMDGPU::G_AMDGPU_MAD_U64_U32:
3544 case AMDGPU::G_AMDGPU_MAD_I64_I32:
3545 applyMappingMAD_64_32(B, OpdMapper);
3546 return;
3547 case AMDGPU::G_PREFETCH: {
3548 if (!Subtarget.hasSafeSmemPrefetch() && !Subtarget.hasVmemPrefInsts()) {
3549 MI.eraseFromParent();
3550 return;
3551 }
3552 Register PtrReg = MI.getOperand(i: 0).getReg();
3553 unsigned PtrBank = getRegBankID(Reg: PtrReg, MRI, Default: AMDGPU::SGPRRegBankID);
3554 if (PtrBank == AMDGPU::VGPRRegBankID &&
3555 (!Subtarget.hasVmemPrefInsts() || !MI.getOperand(i: 3).getImm())) {
3556 // Cannot do I$ prefetch with divergent pointer.
3557 MI.eraseFromParent();
3558 return;
3559 }
3560 unsigned AS = MRI.getType(Reg: PtrReg).getAddressSpace();
3561 if ((!AMDGPU::isFlatGlobalAddrSpace(AS) &&
3562 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
3563 (!Subtarget.hasSafeSmemPrefetch() &&
3564 (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
3565 !MI.getOperand(i: 3).getImm() /* I$ prefetch */))) {
3566 MI.eraseFromParent();
3567 return;
3568 }
3569 applyDefaultMapping(OpdMapper);
3570 return;
3571 }
3572 default:
3573 break;
3574 }
3575
3576 return applyDefaultMapping(OpdMapper);
3577}
3578
3579// vgpr, sgpr -> vgpr
3580// vgpr, agpr -> vgpr
3581// agpr, agpr -> agpr
3582// agpr, sgpr -> vgpr
3583static unsigned regBankUnion(unsigned RB0, unsigned RB1) {
3584 if (RB0 == AMDGPU::InvalidRegBankID)
3585 return RB1;
3586 if (RB1 == AMDGPU::InvalidRegBankID)
3587 return RB0;
3588
3589 if (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID)
3590 return AMDGPU::SGPRRegBankID;
3591
3592 if (RB0 == AMDGPU::AGPRRegBankID && RB1 == AMDGPU::AGPRRegBankID)
3593 return AMDGPU::AGPRRegBankID;
3594
3595 return AMDGPU::VGPRRegBankID;
3596}
3597
3598static unsigned regBankBoolUnion(unsigned RB0, unsigned RB1) {
3599 if (RB0 == AMDGPU::InvalidRegBankID)
3600 return RB1;
3601 if (RB1 == AMDGPU::InvalidRegBankID)
3602 return RB0;
3603
3604 // vcc, vcc -> vcc
3605 // vcc, sgpr -> vcc
3606 // vcc, vgpr -> vcc
3607 if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID)
3608 return AMDGPU::VCCRegBankID;
3609
3610 // vcc, vgpr -> vgpr
3611 return regBankUnion(RB0, RB1);
3612}
3613
3614unsigned AMDGPURegisterBankInfo::getMappingType(const MachineRegisterInfo &MRI,
3615 const MachineInstr &MI) const {
3616 unsigned RegBank = AMDGPU::InvalidRegBankID;
3617
3618 for (const MachineOperand &MO : MI.operands()) {
3619 if (!MO.isReg())
3620 continue;
3621 Register Reg = MO.getReg();
3622 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
3623 RegBank = regBankUnion(RB0: RegBank, RB1: Bank->getID());
3624 if (RegBank == AMDGPU::VGPRRegBankID)
3625 break;
3626 }
3627 }
3628
3629 return RegBank;
3630}
3631
3632bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const {
3633 const MachineFunction &MF = *MI.getMF();
3634 const MachineRegisterInfo &MRI = MF.getRegInfo();
3635 for (const MachineOperand &MO : MI.operands()) {
3636 if (!MO.isReg())
3637 continue;
3638 Register Reg = MO.getReg();
3639 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
3640 if (Bank->getID() != AMDGPU::SGPRRegBankID)
3641 return false;
3642 }
3643 }
3644 return true;
3645}
3646
3647const RegisterBankInfo::InstructionMapping &
3648AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const {
3649 const MachineFunction &MF = *MI.getMF();
3650 const MachineRegisterInfo &MRI = MF.getRegInfo();
3651 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3652
3653 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3654 const MachineOperand &SrcOp = MI.getOperand(i);
3655 if (!SrcOp.isReg())
3656 continue;
3657
3658 unsigned Size = getSizeInBits(Reg: SrcOp.getReg(), MRI, TRI: *TRI);
3659 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
3660 }
3661 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3662 NumOperands: MI.getNumOperands());
3663}
3664
3665const RegisterBankInfo::InstructionMapping &
3666AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
3667 const MachineFunction &MF = *MI.getMF();
3668 const MachineRegisterInfo &MRI = MF.getRegInfo();
3669 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3670
3671 // Even though we technically could use SGPRs, this would require knowledge of
3672 // the constant bus restriction. Force all sources to VGPR (except for VCC).
3673 //
3674 // TODO: Unary ops are trivially OK, so accept SGPRs?
3675 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3676 const MachineOperand &Src = MI.getOperand(i);
3677 if (!Src.isReg())
3678 continue;
3679
3680 unsigned Size = getSizeInBits(Reg: Src.getReg(), MRI, TRI: *TRI);
3681 unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
3682 OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
3683 }
3684
3685 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3686 NumOperands: MI.getNumOperands());
3687}
3688
3689const RegisterBankInfo::InstructionMapping &
3690AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
3691 const MachineFunction &MF = *MI.getMF();
3692 const MachineRegisterInfo &MRI = MF.getRegInfo();
3693 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3694
3695 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
3696 const MachineOperand &Op = MI.getOperand(i: I);
3697 if (!Op.isReg())
3698 continue;
3699
3700 unsigned Size = getSizeInBits(Reg: Op.getReg(), MRI, TRI: *TRI);
3701 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3702 }
3703
3704 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3705 NumOperands: MI.getNumOperands());
3706}
3707
3708const RegisterBankInfo::InstructionMapping &
3709AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
3710 const MachineInstr &MI,
3711 int RsrcIdx) const {
3712 // The reported argument index is relative to the IR intrinsic call arguments,
3713 // so we need to shift by the number of defs and the intrinsic ID.
3714 RsrcIdx += MI.getNumExplicitDefs() + 1;
3715
3716 const int NumOps = MI.getNumOperands();
3717 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps);
3718
3719 // TODO: Should packed/unpacked D16 difference be reported here as part of
3720 // the value mapping?
3721 for (int I = 0; I != NumOps; ++I) {
3722 if (!MI.getOperand(i: I).isReg())
3723 continue;
3724
3725 Register OpReg = MI.getOperand(i: I).getReg();
3726 // We replace some dead address operands with $noreg
3727 if (!OpReg)
3728 continue;
3729
3730 unsigned Size = getSizeInBits(Reg: OpReg, MRI, TRI: *TRI);
3731
3732 // FIXME: Probably need a new intrinsic register bank searchable table to
3733 // handle arbitrary intrinsics easily.
3734 //
3735 // If this has a sampler, it immediately follows rsrc.
3736 const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1;
3737
3738 if (MustBeSGPR) {
3739 // If this must be an SGPR, so we must report whatever it is as legal.
3740 unsigned NewBank = getRegBankID(Reg: OpReg, MRI, Default: AMDGPU::SGPRRegBankID);
3741 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: NewBank, Size);
3742 } else {
3743 // Some operands must be VGPR, and these are easy to copy to.
3744 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3745 }
3746 }
3747
3748 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping), NumOperands: NumOps);
3749}
3750
3751/// Return the mapping for a pointer argument.
3752const RegisterBankInfo::ValueMapping *
3753AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI,
3754 Register PtrReg) const {
3755 LLT PtrTy = MRI.getType(Reg: PtrReg);
3756 unsigned Size = PtrTy.getSizeInBits();
3757 if (Subtarget.useFlatForGlobal() ||
3758 !AMDGPU::isFlatGlobalAddrSpace(AS: PtrTy.getAddressSpace()))
3759 return AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3760
3761 // If we're using MUBUF instructions for global memory, an SGPR base register
3762 // is possible. Otherwise this needs to be a VGPR.
3763 const RegisterBank *PtrBank = getRegBank(Reg: PtrReg, MRI, TRI: *TRI);
3764 return AMDGPU::getValueMapping(BankID: PtrBank->getID(), Size);
3765}
3766
3767const RegisterBankInfo::InstructionMapping &
3768AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
3769
3770 const MachineFunction &MF = *MI.getMF();
3771 const MachineRegisterInfo &MRI = MF.getRegInfo();
3772 SmallVector<const ValueMapping*, 2> OpdsMapping(2);
3773 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3774 Register PtrReg = MI.getOperand(i: 1).getReg();
3775 LLT PtrTy = MRI.getType(Reg: PtrReg);
3776 unsigned AS = PtrTy.getAddressSpace();
3777 unsigned PtrSize = PtrTy.getSizeInBits();
3778
3779 const ValueMapping *ValMapping;
3780 const ValueMapping *PtrMapping;
3781
3782 const RegisterBank *PtrBank = getRegBank(Reg: PtrReg, MRI, TRI: *TRI);
3783
3784 if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) {
3785 if (isScalarLoadLegal(MI)) {
3786 // We have a uniform instruction so we want to use an SMRD load
3787 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
3788 PtrMapping = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: PtrSize);
3789 } else {
3790 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3791
3792 // If we're using MUBUF instructions for global memory, an SGPR base
3793 // register is possible. Otherwise this needs to be a VGPR.
3794 unsigned PtrBankID = Subtarget.useFlatForGlobal() ?
3795 AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID;
3796
3797 PtrMapping = AMDGPU::getValueMapping(BankID: PtrBankID, Size: PtrSize);
3798 }
3799 } else {
3800 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3801 PtrMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize);
3802 }
3803
3804 OpdsMapping[0] = ValMapping;
3805 OpdsMapping[1] = PtrMapping;
3806 const RegisterBankInfo::InstructionMapping &Mapping = getInstructionMapping(
3807 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping), NumOperands: MI.getNumOperands());
3808 return Mapping;
3809
3810 // FIXME: Do we want to add a mapping for FLAT load, or should we just
3811 // handle that during instruction selection?
3812}
3813
3814unsigned
3815AMDGPURegisterBankInfo::getRegBankID(Register Reg,
3816 const MachineRegisterInfo &MRI,
3817 unsigned Default) const {
3818 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
3819 return Bank ? Bank->getID() : Default;
3820}
3821
3822const RegisterBankInfo::ValueMapping *
3823AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
3824 const MachineRegisterInfo &MRI,
3825 const TargetRegisterInfo &TRI) const {
3826 // Lie and claim anything is legal, even though this needs to be an SGPR
3827 // applyMapping will have to deal with it as a waterfall loop.
3828 unsigned Bank = getRegBankID(Reg, MRI, Default: AMDGPU::SGPRRegBankID);
3829 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3830 return AMDGPU::getValueMapping(BankID: Bank, Size);
3831}
3832
3833const RegisterBankInfo::ValueMapping *
3834AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
3835 const MachineRegisterInfo &MRI,
3836 const TargetRegisterInfo &TRI) const {
3837 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3838 return AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3839}
3840
3841const RegisterBankInfo::ValueMapping *
3842AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg,
3843 const MachineRegisterInfo &MRI,
3844 const TargetRegisterInfo &TRI) const {
3845 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3846 return AMDGPU::getValueMapping(BankID: AMDGPU::AGPRRegBankID, Size);
3847}
3848
3849///
3850/// This function must return a legal mapping, because
3851/// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called
3852/// in RegBankSelect::Mode::Fast. Any mapping that would cause a
3853/// VGPR to SGPR generated is illegal.
3854///
3855// Operands that must be SGPRs must accept potentially divergent VGPRs as
3856// legal. These will be dealt with in applyMappingImpl.
3857//
3858const RegisterBankInfo::InstructionMapping &
3859AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
3860 const MachineFunction &MF = *MI.getMF();
3861 const MachineRegisterInfo &MRI = MF.getRegInfo();
3862
3863 if (MI.isCopy() || MI.getOpcode() == AMDGPU::G_FREEZE) {
3864 Register DstReg = MI.getOperand(i: 0).getReg();
3865 Register SrcReg = MI.getOperand(i: 1).getReg();
3866
3867 // The default logic bothers to analyze impossible alternative mappings. We
3868 // want the most straightforward mapping, so just directly handle this.
3869 const RegisterBank *DstBank = getRegBank(Reg: DstReg, MRI, TRI: *TRI);
3870 const RegisterBank *SrcBank = getRegBank(Reg: SrcReg, MRI, TRI: *TRI);
3871
3872 // For COPY between a physical reg and an s1, there is no type associated so
3873 // we need to take the virtual register's type as a hint on how to interpret
3874 // s1 values.
3875 unsigned Size;
3876 if (!SrcReg.isVirtual() && !DstBank &&
3877 MRI.getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1)) {
3878 DstBank = &AMDGPU::VCCRegBank;
3879 Size = 1;
3880 } else if (!DstReg.isVirtual() && MRI.getType(Reg: SrcReg) == LLT::scalar(SizeInBits: 1)) {
3881 DstBank = &AMDGPU::VCCRegBank;
3882 Size = 1;
3883 } else {
3884 Size = getSizeInBits(Reg: DstReg, MRI, TRI: *TRI);
3885 }
3886
3887 if (!DstBank)
3888 DstBank = SrcBank;
3889 else if (!SrcBank)
3890 SrcBank = DstBank;
3891
3892 if (MI.getOpcode() != AMDGPU::G_FREEZE &&
3893 cannotCopy(Dst: *DstBank, Src: *SrcBank, Size: TypeSize::getFixed(ExactSize: Size)))
3894 return getInvalidInstructionMapping();
3895
3896 const ValueMapping &ValMap = getValueMapping(StartIdx: 0, Length: Size, RegBank: *DstBank);
3897 unsigned OpdsMappingSize = MI.isCopy() ? 1 : 2;
3898 SmallVector<const ValueMapping *, 1> OpdsMapping(OpdsMappingSize);
3899 OpdsMapping[0] = &ValMap;
3900 if (MI.getOpcode() == AMDGPU::G_FREEZE)
3901 OpdsMapping[1] = &ValMap;
3902
3903 return getInstructionMapping(
3904 ID: 1, /*Cost*/ 1,
3905 /*OperandsMapping*/ getOperandsMapping(OpdsMapping), NumOperands: OpdsMappingSize);
3906 }
3907
3908 if (MI.isRegSequence()) {
3909 // If any input is a VGPR, the result must be a VGPR. The default handling
3910 // assumes any copy between banks is legal.
3911 unsigned BankID = AMDGPU::SGPRRegBankID;
3912
3913 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3914 auto OpBank = getRegBankID(Reg: MI.getOperand(i: I).getReg(), MRI);
3915 // It doesn't make sense to use vcc or scc banks here, so just ignore
3916 // them.
3917 if (OpBank != AMDGPU::SGPRRegBankID) {
3918 BankID = AMDGPU::VGPRRegBankID;
3919 break;
3920 }
3921 }
3922 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3923
3924 const ValueMapping &ValMap = getValueMapping(StartIdx: 0, Length: Size, RegBank: getRegBank(ID: BankID));
3925 return getInstructionMapping(
3926 ID: 1, /*Cost*/ 1,
3927 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: {&ValMap}), NumOperands: 1);
3928 }
3929
3930 // The default handling is broken and doesn't handle illegal SGPR->VGPR copies
3931 // properly.
3932 //
3933 // TODO: There are additional exec masking dependencies to analyze.
3934 if (auto *PHI = dyn_cast<GPhi>(Val: &MI)) {
3935 unsigned ResultBank = AMDGPU::InvalidRegBankID;
3936 Register DstReg = PHI->getReg(Idx: 0);
3937
3938 // Sometimes the result may have already been assigned a bank.
3939 if (const RegisterBank *DstBank = getRegBank(Reg: DstReg, MRI, TRI: *TRI))
3940 ResultBank = DstBank->getID();
3941
3942 for (unsigned I = 0; I < PHI->getNumIncomingValues(); ++I) {
3943 Register Reg = PHI->getIncomingValue(I);
3944 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
3945
3946 // FIXME: Assuming VGPR for any undetermined inputs.
3947 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) {
3948 ResultBank = AMDGPU::VGPRRegBankID;
3949 break;
3950 }
3951
3952 // FIXME: Need to promote SGPR case to s32
3953 unsigned OpBank = Bank->getID();
3954 ResultBank = regBankBoolUnion(RB0: ResultBank, RB1: OpBank);
3955 }
3956
3957 assert(ResultBank != AMDGPU::InvalidRegBankID);
3958
3959 unsigned Size = MRI.getType(Reg: DstReg).getSizeInBits();
3960
3961 const ValueMapping &ValMap =
3962 getValueMapping(StartIdx: 0, Length: Size, RegBank: getRegBank(ID: ResultBank));
3963 return getInstructionMapping(
3964 ID: 1, /*Cost*/ 1,
3965 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: {&ValMap}), NumOperands: 1);
3966 }
3967
3968 const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
3969 if (Mapping.isValid())
3970 return Mapping;
3971
3972 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3973
3974 switch (MI.getOpcode()) {
3975 default:
3976 return getInvalidInstructionMapping();
3977
3978 case AMDGPU::G_AND:
3979 case AMDGPU::G_OR:
3980 case AMDGPU::G_XOR:
3981 case AMDGPU::G_MUL: {
3982 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
3983 if (Size == 1) {
3984 const RegisterBank *DstBank
3985 = getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3986
3987 unsigned TargetBankID = AMDGPU::InvalidRegBankID;
3988 unsigned BankLHS = AMDGPU::InvalidRegBankID;
3989 unsigned BankRHS = AMDGPU::InvalidRegBankID;
3990 if (DstBank) {
3991 TargetBankID = DstBank->getID();
3992 if (DstBank == &AMDGPU::VCCRegBank) {
3993 TargetBankID = AMDGPU::VCCRegBankID;
3994 BankLHS = AMDGPU::VCCRegBankID;
3995 BankRHS = AMDGPU::VCCRegBankID;
3996 } else {
3997 BankLHS = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
3998 Default: AMDGPU::SGPRRegBankID);
3999 BankRHS = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
4000 Default: AMDGPU::SGPRRegBankID);
4001 }
4002 } else {
4003 BankLHS = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
4004 Default: AMDGPU::VCCRegBankID);
4005 BankRHS = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
4006 Default: AMDGPU::VCCRegBankID);
4007
4008 // Both inputs should be true booleans to produce a boolean result.
4009 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) {
4010 TargetBankID = AMDGPU::VGPRRegBankID;
4011 } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) {
4012 TargetBankID = AMDGPU::VCCRegBankID;
4013 BankLHS = AMDGPU::VCCRegBankID;
4014 BankRHS = AMDGPU::VCCRegBankID;
4015 } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) {
4016 TargetBankID = AMDGPU::SGPRRegBankID;
4017 }
4018 }
4019
4020 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: TargetBankID, Size);
4021 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: BankLHS, Size);
4022 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: BankRHS, Size);
4023 break;
4024 }
4025
4026 if (Size == 64) {
4027
4028 if (isSALUMapping(MI)) {
4029 OpdsMapping[0] = getValueMappingSGPR64Only(BankID: AMDGPU::SGPRRegBankID, Size);
4030 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
4031 } else {
4032 if (MI.getOpcode() == AMDGPU::G_MUL && Subtarget.hasVMulU64Inst())
4033 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4034 else
4035 OpdsMapping[0] =
4036 getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size);
4037 unsigned Bank1 = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI /*, DefaultBankID*/);
4038 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank1, Size);
4039
4040 unsigned Bank2 = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI /*, DefaultBankID*/);
4041 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank2, Size);
4042 }
4043
4044 break;
4045 }
4046
4047 [[fallthrough]];
4048 }
4049 case AMDGPU::G_PTR_ADD:
4050 case AMDGPU::G_PTRMASK:
4051 case AMDGPU::G_ADD:
4052 case AMDGPU::G_SUB:
4053 case AMDGPU::G_SHL:
4054 case AMDGPU::G_LSHR:
4055 case AMDGPU::G_ASHR:
4056 case AMDGPU::G_UADDO:
4057 case AMDGPU::G_USUBO:
4058 case AMDGPU::G_UADDE:
4059 case AMDGPU::G_SADDE:
4060 case AMDGPU::G_USUBE:
4061 case AMDGPU::G_SSUBE:
4062 case AMDGPU::G_ABS:
4063 case AMDGPU::G_SHUFFLE_VECTOR:
4064 case AMDGPU::G_SBFX:
4065 case AMDGPU::G_UBFX:
4066 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
4067 case AMDGPU::G_AMDGPU_S_MUL_U64_U32:
4068 if (isSALUMapping(MI)) {
4069 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4070 unsigned Size = Ty.getSizeInBits();
4071 // Packed add and sub are VALU only.
4072 if (Subtarget.hasAnyPackedU64Ops() && Ty.isVector() && Size == 128)
4073 return getDefaultMappingVOP(MI);
4074 return getDefaultMappingSOP(MI);
4075 }
4076 return getDefaultMappingVOP(MI);
4077 case AMDGPU::G_SMIN:
4078 case AMDGPU::G_SMAX:
4079 case AMDGPU::G_UMIN:
4080 case AMDGPU::G_UMAX:
4081 if (isSALUMapping(MI)) {
4082 // There are no scalar 64-bit min and max, use vector instruction instead.
4083 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() == 64 &&
4084 Subtarget.hasMinMaxI64Insts())
4085 return getDefaultMappingVOP(MI);
4086 return getDefaultMappingSOP(MI);
4087 }
4088 return getDefaultMappingVOP(MI);
4089 case AMDGPU::G_FADD:
4090 case AMDGPU::G_FSUB:
4091 case AMDGPU::G_FMUL:
4092 case AMDGPU::G_FMA:
4093 case AMDGPU::G_FFLOOR:
4094 case AMDGPU::G_FCEIL:
4095 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
4096 case AMDGPU::G_FMINNUM:
4097 case AMDGPU::G_FMAXNUM:
4098 case AMDGPU::G_FMINIMUMNUM:
4099 case AMDGPU::G_FMAXIMUMNUM:
4100 case AMDGPU::G_INTRINSIC_TRUNC:
4101 case AMDGPU::G_STRICT_FADD:
4102 case AMDGPU::G_STRICT_FSUB:
4103 case AMDGPU::G_STRICT_FMUL:
4104 case AMDGPU::G_STRICT_FMA: {
4105 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4106 unsigned Size = Ty.getSizeInBits();
4107 if (Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
4108 (Size == 32 || Size == 16) && isSALUMapping(MI))
4109 return getDefaultMappingSOP(MI);
4110 return getDefaultMappingVOP(MI);
4111 }
4112 case AMDGPU::G_FMINIMUM:
4113 case AMDGPU::G_FMAXIMUM: {
4114 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4115 unsigned Size = Ty.getSizeInBits();
4116 if (Subtarget.hasSALUMinimumMaximumInsts() && Ty.isScalar() &&
4117 (Size == 32 || Size == 16) && isSALUMapping(MI))
4118 return getDefaultMappingSOP(MI);
4119 return getDefaultMappingVOP(MI);
4120 }
4121 case AMDGPU::G_FPTOSI:
4122 case AMDGPU::G_FPTOUI:
4123 case AMDGPU::G_FPTOSI_SAT:
4124 case AMDGPU::G_FPTOUI_SAT:
4125 case AMDGPU::G_SITOFP:
4126 case AMDGPU::G_UITOFP: {
4127 unsigned SizeDst = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4128 unsigned SizeSrc = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4129 if (Subtarget.hasSALUFloatInsts() && SizeDst == 32 && SizeSrc == 32 &&
4130 isSALUMapping(MI))
4131 return getDefaultMappingSOP(MI);
4132 return getDefaultMappingVOP(MI);
4133 }
4134 case AMDGPU::G_FPTRUNC:
4135 case AMDGPU::G_FPEXT: {
4136 unsigned SizeDst = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4137 unsigned SizeSrc = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4138 if (Subtarget.hasSALUFloatInsts() && SizeDst != 64 && SizeSrc != 64 &&
4139 isSALUMapping(MI))
4140 return getDefaultMappingSOP(MI);
4141 return getDefaultMappingVOP(MI);
4142 }
4143 case AMDGPU::G_FSQRT:
4144 case AMDGPU::G_FEXP2:
4145 case AMDGPU::G_FLOG2: {
4146 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4147 if (Subtarget.hasPseudoScalarTrans() && (Size == 16 || Size == 32) &&
4148 isSALUMapping(MI))
4149 return getDefaultMappingSOP(MI);
4150 return getDefaultMappingVOP(MI);
4151 }
4152 case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU
4153 case AMDGPU::G_SSUBSAT:
4154 case AMDGPU::G_UADDSAT:
4155 case AMDGPU::G_USUBSAT:
4156 case AMDGPU::G_FMAD:
4157 case AMDGPU::G_FLDEXP:
4158 case AMDGPU::G_FMINNUM_IEEE:
4159 case AMDGPU::G_FMAXNUM_IEEE:
4160 case AMDGPU::G_FCANONICALIZE:
4161 case AMDGPU::G_STRICT_FLDEXP:
4162 case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar?
4163 case AMDGPU::G_FSHR: // TODO: Expand for scalar
4164 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
4165 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
4166 case AMDGPU::G_AMDGPU_RCP_IFLAG:
4167 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
4168 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
4169 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
4170 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
4171 case AMDGPU::G_AMDGPU_CVT_PK_I16_I32:
4172 case AMDGPU::G_AMDGPU_SMED3:
4173 case AMDGPU::G_AMDGPU_FMED3:
4174 return getDefaultMappingVOP(MI);
4175 case AMDGPU::G_UMULH:
4176 case AMDGPU::G_SMULH: {
4177 if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI))
4178 return getDefaultMappingSOP(MI);
4179 return getDefaultMappingVOP(MI);
4180 }
4181 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4182 case AMDGPU::G_AMDGPU_MAD_I64_I32: {
4183 // Three possible mappings:
4184 //
4185 // - Default SOP
4186 // - Default VOP
4187 // - Scalar multiply: src0 and src1 are SGPRs, the rest is VOP.
4188 //
4189 // This allows instruction selection to keep the multiplication part of the
4190 // instruction on the SALU.
4191 bool AllSalu = true;
4192 bool MulSalu = true;
4193 for (unsigned i = 0; i < 5; ++i) {
4194 Register Reg = MI.getOperand(i).getReg();
4195 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
4196 if (Bank->getID() != AMDGPU::SGPRRegBankID) {
4197 AllSalu = false;
4198 if (i == 2 || i == 3) {
4199 MulSalu = false;
4200 break;
4201 }
4202 }
4203 }
4204 }
4205
4206 if (AllSalu)
4207 return getDefaultMappingSOP(MI);
4208
4209 // If the multiply-add is full-rate in VALU, use that even if the
4210 // multiplication part is scalar. Accumulating separately on the VALU would
4211 // take two instructions.
4212 if (!MulSalu || Subtarget.hasFullRate64Ops())
4213 return getDefaultMappingVOP(MI);
4214
4215 // Keep the multiplication on the SALU, then accumulate on the VALU.
4216 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 64);
4217 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4218 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4219 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4220 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 64);
4221 break;
4222 }
4223 case AMDGPU::G_IMPLICIT_DEF: {
4224 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4225 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4226 break;
4227 }
4228 case AMDGPU::G_FCONSTANT:
4229 case AMDGPU::G_CONSTANT:
4230 case AMDGPU::G_GLOBAL_VALUE:
4231 case AMDGPU::G_FRAME_INDEX:
4232 case AMDGPU::G_BLOCK_ADDR:
4233 case AMDGPU::G_READSTEADYCOUNTER:
4234 case AMDGPU::G_READCYCLECOUNTER: {
4235 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4236 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4237 break;
4238 }
4239 case AMDGPU::G_DYN_STACKALLOC: {
4240 // Result is always uniform, and a wave reduction is needed for the source.
4241 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4242 unsigned SrcBankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4243 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: SrcBankID, Size: 32);
4244 break;
4245 }
4246 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: {
4247 // This case is weird because we expect a physical register in the source,
4248 // but need to set a bank anyway.
4249 //
4250 // TODO: We could select the result to SGPR or VGPR
4251 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4252 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4253 break;
4254 }
4255 case AMDGPU::G_INSERT: {
4256 unsigned BankID = getMappingType(MRI, MI);
4257 unsigned DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4258 unsigned SrcSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4259 unsigned EltSize = getSizeInBits(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4260 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: DstSize);
4261 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size: SrcSize);
4262 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, Size: EltSize);
4263 OpdsMapping[3] = nullptr;
4264 break;
4265 }
4266 case AMDGPU::G_EXTRACT: {
4267 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4268 unsigned DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4269 unsigned SrcSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4270 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: DstSize);
4271 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size: SrcSize);
4272 OpdsMapping[2] = nullptr;
4273 break;
4274 }
4275 case AMDGPU::G_BUILD_VECTOR:
4276 case AMDGPU::G_BUILD_VECTOR_TRUNC: {
4277 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4278 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16)) {
4279 unsigned DstSize = DstTy.getSizeInBits();
4280 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4281 unsigned Src0BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4282 unsigned Src1BankID = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4283 unsigned DstBankID = regBankUnion(RB0: Src0BankID, RB1: Src1BankID);
4284
4285 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: DstBankID, Size: DstSize);
4286 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Src0BankID, Size: SrcSize);
4287 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Src1BankID, Size: SrcSize);
4288 break;
4289 }
4290
4291 [[fallthrough]];
4292 }
4293 case AMDGPU::G_MERGE_VALUES:
4294 case AMDGPU::G_CONCAT_VECTORS: {
4295 unsigned Bank = getMappingType(MRI, MI);
4296 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4297 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4298
4299 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: DstSize);
4300 // Op1 and Dst should use the same register bank.
4301 for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i)
4302 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: Bank, Size: SrcSize);
4303 break;
4304 }
4305 case AMDGPU::G_BITREVERSE:
4306 case AMDGPU::G_BITCAST:
4307 case AMDGPU::G_INTTOPTR:
4308 case AMDGPU::G_PTRTOINT:
4309 case AMDGPU::G_FABS:
4310 case AMDGPU::G_FNEG: {
4311 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4312 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4313 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
4314 break;
4315 }
4316 case AMDGPU::G_AMDGPU_FFBH_U32:
4317 case AMDGPU::G_AMDGPU_FFBL_B32:
4318 case AMDGPU::G_CTLZ_ZERO_POISON:
4319 case AMDGPU::G_CTTZ_ZERO_POISON: {
4320 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4321 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4322 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: 32);
4323 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID, Size);
4324 break;
4325 }
4326 case AMDGPU::G_CTPOP: {
4327 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4328 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4329 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: 32);
4330
4331 // This should really be getValueMappingSGPR64Only, but allowing the generic
4332 // code to handle the register split just makes using LegalizerHelper more
4333 // difficult.
4334 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
4335 break;
4336 }
4337 case AMDGPU::G_TRUNC: {
4338 Register Dst = MI.getOperand(i: 0).getReg();
4339 Register Src = MI.getOperand(i: 1).getReg();
4340 unsigned Bank = getRegBankID(Reg: Src, MRI);
4341 unsigned DstSize = getSizeInBits(Reg: Dst, MRI, TRI: *TRI);
4342 unsigned SrcSize = getSizeInBits(Reg: Src, MRI, TRI: *TRI);
4343 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: DstSize);
4344 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: SrcSize);
4345 break;
4346 }
4347 case AMDGPU::G_ZEXT:
4348 case AMDGPU::G_SEXT:
4349 case AMDGPU::G_ANYEXT:
4350 case AMDGPU::G_SEXT_INREG: {
4351 Register Dst = MI.getOperand(i: 0).getReg();
4352 Register Src = MI.getOperand(i: 1).getReg();
4353 unsigned DstSize = getSizeInBits(Reg: Dst, MRI, TRI: *TRI);
4354 unsigned SrcSize = getSizeInBits(Reg: Src, MRI, TRI: *TRI);
4355
4356 unsigned DstBank;
4357 const RegisterBank *SrcBank = getRegBank(Reg: Src, MRI, TRI: *TRI);
4358 assert(SrcBank);
4359 switch (SrcBank->getID()) {
4360 case AMDGPU::SGPRRegBankID:
4361 DstBank = AMDGPU::SGPRRegBankID;
4362 break;
4363 default:
4364 DstBank = AMDGPU::VGPRRegBankID;
4365 break;
4366 }
4367
4368 // Scalar extend can use 64-bit BFE, but VGPRs require extending to
4369 // 32-bits, and then to 64.
4370 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: DstBank, Size: DstSize);
4371 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID: SrcBank->getID(),
4372 Size: SrcSize);
4373 break;
4374 }
4375 case AMDGPU::G_IS_FPCLASS: {
4376 Register SrcReg = MI.getOperand(i: 1).getReg();
4377 unsigned SrcSize = MRI.getType(Reg: SrcReg).getSizeInBits();
4378 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4379 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: DstSize);
4380 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4381 break;
4382 }
4383 case AMDGPU::G_STORE: {
4384 assert(MI.getOperand(0).isReg());
4385 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4386
4387 // FIXME: We need to specify a different reg bank once scalar stores are
4388 // supported.
4389 const ValueMapping *ValMapping =
4390 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4391 OpdsMapping[0] = ValMapping;
4392 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
4393 break;
4394 }
4395 case AMDGPU::G_ICMP:
4396 case AMDGPU::G_FCMP: {
4397 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4398
4399 // See if the result register has already been constrained to vcc, which may
4400 // happen due to control flow intrinsic lowering.
4401 unsigned DstBank = getRegBankID(Reg: MI.getOperand(i: 0).getReg(), MRI,
4402 Default: AMDGPU::SGPRRegBankID);
4403 unsigned Op2Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4404 unsigned Op3Bank = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI);
4405
4406 auto canUseSCCICMP = [&]() {
4407 auto Pred =
4408 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4409 return Size == 32 ||
4410 (Size == 64 &&
4411 (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
4412 Subtarget.hasScalarCompareEq64());
4413 };
4414 auto canUseSCCFCMP = [&]() {
4415 return Subtarget.hasSALUFloatInsts() && (Size == 32 || Size == 16);
4416 };
4417
4418 bool isICMP = MI.getOpcode() == AMDGPU::G_ICMP;
4419 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
4420 Op2Bank == AMDGPU::SGPRRegBankID &&
4421 Op3Bank == AMDGPU::SGPRRegBankID &&
4422 (isICMP ? canUseSCCICMP() : canUseSCCFCMP());
4423
4424 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4425 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4426
4427 // TODO: Use 32-bit for scalar output size.
4428 // SCC results will need to be copied to a 32-bit SGPR virtual register.
4429 const unsigned ResultSize = 1;
4430
4431 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: DstBank, Size: ResultSize);
4432 OpdsMapping[1] = nullptr; // Predicate Operand.
4433 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: SrcBank, Size);
4434 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: SrcBank, Size);
4435 break;
4436 }
4437 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
4438 // VGPR index can be used for waterfall when indexing a SGPR vector.
4439 unsigned SrcBankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4440 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4441 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4442 unsigned IdxSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4443 unsigned IdxBank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4444 unsigned OutputBankID = regBankUnion(RB0: SrcBankID, RB1: IdxBank);
4445
4446 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: OutputBankID, Size: DstSize);
4447 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: SrcBankID, Size: SrcSize);
4448
4449 // The index can be either if the source vector is VGPR.
4450 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4451 break;
4452 }
4453 case AMDGPU::G_INSERT_VECTOR_ELT: {
4454 unsigned OutputBankID = isSALUMapping(MI) ?
4455 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4456
4457 unsigned VecSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4458 unsigned InsertSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4459 unsigned IdxSize = MRI.getType(Reg: MI.getOperand(i: 3).getReg()).getSizeInBits();
4460 unsigned InsertEltBankID = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4461 unsigned IdxBankID = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI);
4462
4463 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: OutputBankID, Size: VecSize);
4464 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: OutputBankID, Size: VecSize);
4465
4466 // This is a weird case, because we need to break down the mapping based on
4467 // the register bank of a different operand.
4468 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) {
4469 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(BankID: InsertEltBankID,
4470 Size: InsertSize);
4471 } else {
4472 assert(InsertSize == 32 || InsertSize == 64);
4473 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: InsertEltBankID, Size: InsertSize);
4474 }
4475
4476 // The index can be either if the source vector is VGPR.
4477 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBankID, Size: IdxSize);
4478 break;
4479 }
4480 case AMDGPU::G_UNMERGE_VALUES: {
4481 unsigned Bank = getMappingType(MRI, MI);
4482
4483 // Op1 and Dst should use the same register bank.
4484 // FIXME: Shouldn't this be the default? Why do we need to handle this?
4485 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
4486 unsigned Size = getSizeInBits(Reg: MI.getOperand(i).getReg(), MRI, TRI: *TRI);
4487 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: Bank, Size);
4488 }
4489 break;
4490 }
4491 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
4492 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
4493 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
4494 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
4495 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
4496 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
4497 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
4498 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
4499 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
4500 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
4501 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
4502 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
4503 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
4504 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
4505 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
4506 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
4507 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
4508 case AMDGPU::G_AMDGPU_BUFFER_STORE:
4509 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
4510 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
4511 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
4512 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: {
4513 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4514
4515 // rsrc
4516 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4517
4518 // vindex
4519 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4520
4521 // voffset
4522 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4523
4524 // soffset
4525 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4526
4527 // Any remaining operands are immediates and were correctly null
4528 // initialized.
4529 break;
4530 }
4531 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
4532 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
4533 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
4534 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
4535 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
4536 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
4537 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
4538 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
4539 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
4540 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
4541 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
4542 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
4543 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
4544 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
4545 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
4546 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
4547 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
4548 // vdata_out
4549 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4550
4551 // vdata_in
4552 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4553
4554 // rsrc
4555 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4556
4557 // vindex
4558 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4559
4560 // voffset
4561 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4562
4563 // soffset
4564 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
4565
4566 // Any remaining operands are immediates and were correctly null
4567 // initialized.
4568 break;
4569 }
4570 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
4571 // vdata_out
4572 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4573
4574 // vdata_in
4575 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4576
4577 // cmp
4578 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4579
4580 // rsrc
4581 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4582
4583 // vindex
4584 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4585
4586 // voffset
4587 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
4588
4589 // soffset
4590 OpdsMapping[6] = getSGPROpMapping(Reg: MI.getOperand(i: 6).getReg(), MRI, TRI: *TRI);
4591
4592 // Any remaining operands are immediates and were correctly null
4593 // initialized.
4594 break;
4595 }
4596 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
4597 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
4598 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
4599 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
4600 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
4601 // Lie and claim everything is legal, even though some need to be
4602 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
4603 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4604 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4605
4606 // We need to convert this to a MUBUF if either the resource of offset is
4607 // VGPR.
4608 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
4609 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
4610 unsigned ResultBank = regBankUnion(RB0: RSrcBank, RB1: OffsetBank);
4611
4612 unsigned Size0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4613 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: ResultBank, Size: Size0);
4614 break;
4615 }
4616 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
4617 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4618 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4619 break;
4620 case AMDGPU::G_AMDGPU_SPONENTRY: {
4621 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4622 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4623 break;
4624 }
4625 case AMDGPU::G_INTRINSIC:
4626 case AMDGPU::G_INTRINSIC_CONVERGENT: {
4627 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
4628 default:
4629 return getInvalidInstructionMapping();
4630 case Intrinsic::amdgcn_div_fmas:
4631 case Intrinsic::amdgcn_div_fixup:
4632 case Intrinsic::amdgcn_trig_preop:
4633 case Intrinsic::amdgcn_sin:
4634 case Intrinsic::amdgcn_cos:
4635 case Intrinsic::amdgcn_log_clamp:
4636 case Intrinsic::amdgcn_rcp_legacy:
4637 case Intrinsic::amdgcn_rsq_legacy:
4638 case Intrinsic::amdgcn_rsq_clamp:
4639 case Intrinsic::amdgcn_tanh:
4640 case Intrinsic::amdgcn_fmul_legacy:
4641 case Intrinsic::amdgcn_fma_legacy:
4642 case Intrinsic::amdgcn_frexp_mant:
4643 case Intrinsic::amdgcn_frexp_exp:
4644 case Intrinsic::amdgcn_fract:
4645 case Intrinsic::amdgcn_cvt_pknorm_i16:
4646 case Intrinsic::amdgcn_cvt_pknorm_u16:
4647 case Intrinsic::amdgcn_cvt_pk_i16:
4648 case Intrinsic::amdgcn_cvt_pk_u16:
4649 case Intrinsic::amdgcn_cvt_sr_pk_f16_f32:
4650 case Intrinsic::amdgcn_cvt_sr_pk_bf16_f32:
4651 case Intrinsic::amdgcn_cvt_pk_f16_fp8:
4652 case Intrinsic::amdgcn_cvt_pk_f16_bf8:
4653 case Intrinsic::amdgcn_cvt_pk_fp8_f16:
4654 case Intrinsic::amdgcn_cvt_pk_bf8_f16:
4655 case Intrinsic::amdgcn_cvt_sr_fp8_f16:
4656 case Intrinsic::amdgcn_cvt_sr_bf8_f16:
4657 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
4658 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
4659 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
4660 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
4661 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
4662 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
4663 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
4664 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
4665 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
4666 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
4667 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
4668 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
4669 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
4670 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
4671 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
4672 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_bf16:
4673 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_bf16:
4674 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f16:
4675 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f16:
4676 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f32:
4677 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f32:
4678 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f32:
4679 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f16:
4680 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_bf16:
4681 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f32:
4682 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f32:
4683 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f16:
4684 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f16:
4685 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_bf16:
4686 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_bf16:
4687 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_bf16:
4688 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_bf16:
4689 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f16:
4690 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f16:
4691 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f32:
4692 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f32:
4693 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f32:
4694 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f16:
4695 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_bf16:
4696 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f32:
4697 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f32:
4698 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f16:
4699 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f16:
4700 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_bf16:
4701 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_bf16:
4702 case Intrinsic::amdgcn_sat_pk4_i4_i8:
4703 case Intrinsic::amdgcn_sat_pk4_u4_u8:
4704 case Intrinsic::amdgcn_fmed3:
4705 case Intrinsic::amdgcn_cubeid:
4706 case Intrinsic::amdgcn_cubema:
4707 case Intrinsic::amdgcn_cubesc:
4708 case Intrinsic::amdgcn_cubetc:
4709 case Intrinsic::amdgcn_sffbh:
4710 case Intrinsic::amdgcn_fmad_ftz:
4711 case Intrinsic::amdgcn_mbcnt_lo:
4712 case Intrinsic::amdgcn_mbcnt_hi:
4713 case Intrinsic::amdgcn_mul_u24:
4714 case Intrinsic::amdgcn_mul_i24:
4715 case Intrinsic::amdgcn_mulhi_u24:
4716 case Intrinsic::amdgcn_mulhi_i24:
4717 case Intrinsic::amdgcn_lerp:
4718 case Intrinsic::amdgcn_sad_u8:
4719 case Intrinsic::amdgcn_msad_u8:
4720 case Intrinsic::amdgcn_sad_hi_u8:
4721 case Intrinsic::amdgcn_sad_u16:
4722 case Intrinsic::amdgcn_qsad_pk_u16_u8:
4723 case Intrinsic::amdgcn_mqsad_pk_u16_u8:
4724 case Intrinsic::amdgcn_mqsad_u32_u8:
4725 case Intrinsic::amdgcn_cvt_pk_u8_f32:
4726 case Intrinsic::amdgcn_alignbyte:
4727 case Intrinsic::amdgcn_perm:
4728 case Intrinsic::amdgcn_prng_b32:
4729 case Intrinsic::amdgcn_fdot2:
4730 case Intrinsic::amdgcn_sdot2:
4731 case Intrinsic::amdgcn_udot2:
4732 case Intrinsic::amdgcn_sdot4:
4733 case Intrinsic::amdgcn_udot4:
4734 case Intrinsic::amdgcn_sdot8:
4735 case Intrinsic::amdgcn_udot8:
4736 case Intrinsic::amdgcn_fdot2_bf16_bf16:
4737 case Intrinsic::amdgcn_fdot2_f16_f16:
4738 case Intrinsic::amdgcn_fdot2_f32_bf16:
4739 case Intrinsic::amdgcn_fdot2c_f32_bf16:
4740 case Intrinsic::amdgcn_sudot4:
4741 case Intrinsic::amdgcn_sudot8:
4742 case Intrinsic::amdgcn_dot4_f32_fp8_bf8:
4743 case Intrinsic::amdgcn_dot4_f32_bf8_fp8:
4744 case Intrinsic::amdgcn_dot4_f32_fp8_fp8:
4745 case Intrinsic::amdgcn_dot4_f32_bf8_bf8:
4746 case Intrinsic::amdgcn_cvt_f32_fp8:
4747 case Intrinsic::amdgcn_cvt_f32_fp8_e5m3:
4748 case Intrinsic::amdgcn_cvt_f32_bf8:
4749 case Intrinsic::amdgcn_cvt_off_f32_i4:
4750 case Intrinsic::amdgcn_cvt_pk_f32_fp8:
4751 case Intrinsic::amdgcn_cvt_pk_f32_bf8:
4752 case Intrinsic::amdgcn_cvt_pk_fp8_f32:
4753 case Intrinsic::amdgcn_cvt_pk_fp8_f32_e5m3:
4754 case Intrinsic::amdgcn_cvt_pk_bf8_f32:
4755 case Intrinsic::amdgcn_cvt_sr_fp8_f32:
4756 case Intrinsic::amdgcn_cvt_sr_fp8_f32_e5m3:
4757 case Intrinsic::amdgcn_cvt_sr_bf8_f32:
4758 case Intrinsic::amdgcn_cvt_sr_bf16_f32:
4759 case Intrinsic::amdgcn_cvt_sr_f16_f32:
4760 case Intrinsic::amdgcn_cvt_f16_fp8:
4761 case Intrinsic::amdgcn_cvt_f16_bf8:
4762 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f16:
4763 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f16:
4764 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_bf16:
4765 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_bf16:
4766 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f32:
4767 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f32:
4768 case Intrinsic::amdgcn_cvt_scalef32_f16_fp8:
4769 case Intrinsic::amdgcn_cvt_scalef32_f16_bf8:
4770 case Intrinsic::amdgcn_cvt_scalef32_f32_fp8:
4771 case Intrinsic::amdgcn_cvt_scalef32_f32_bf8:
4772 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f32:
4773 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f32:
4774 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp8:
4775 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_bf8:
4776 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f16:
4777 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_bf16:
4778 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f16:
4779 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_bf16:
4780 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp4:
4781 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f32:
4782 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp4:
4783 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp4:
4784 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_fp6:
4785 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_bf6:
4786 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_bf6:
4787 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_bf6:
4788 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_fp6:
4789 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_fp6:
4790 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_bf8:
4791 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_bf8:
4792 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp8:
4793 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp8:
4794 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f16:
4795 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_bf16:
4796 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f16:
4797 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_bf16:
4798 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f32:
4799 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_bf16:
4800 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f16:
4801 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f32:
4802 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_bf16:
4803 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f16:
4804 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f32:
4805 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_bf16:
4806 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f16:
4807 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f32:
4808 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_bf16:
4809 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f16:
4810 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f32:
4811 case Intrinsic::amdgcn_ashr_pk_i8_i32:
4812 case Intrinsic::amdgcn_ashr_pk_u8_i32:
4813 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_fp6_f32:
4814 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_bf6_f32:
4815 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16:
4816 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16:
4817 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied:
4818 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied:
4819 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf16:
4820 case Intrinsic::amdgcn_wmma_f32_16x16x16_f16:
4821 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu4:
4822 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu8:
4823 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8:
4824 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8:
4825 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8:
4826 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8:
4827 case Intrinsic::amdgcn_wmma_i32_16x16x32_iu4:
4828 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
4829 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
4830 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
4831 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
4832 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
4833 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
4834 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4:
4835 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
4836 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
4837 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
4838 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8:
4839 case Intrinsic::amdgcn_wmma_f64_16x16x4_f64:
4840 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4841 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4842 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4843 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4844 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4845 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
4846 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_fp8:
4847 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_bf8:
4848 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_fp8:
4849 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_bf8:
4850 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_fp8:
4851 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_bf8:
4852 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_fp8:
4853 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_bf8:
4854 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_fp8:
4855 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_bf8:
4856 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_fp8:
4857 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_bf8:
4858 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_fp8:
4859 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_bf8:
4860 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_fp8:
4861 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_bf8:
4862 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
4863 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
4864 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
4865 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
4866 case Intrinsic::amdgcn_wmma_f32_32x16x128_f4:
4867 case Intrinsic::amdgcn_wmma_scale_f32_32x16x128_f4:
4868 case Intrinsic::amdgcn_wmma_scale16_f32_32x16x128_f4:
4869 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
4870 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
4871 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
4872 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
4873 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
4874 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
4875 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
4876 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
4877 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
4878 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
4879 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
4880 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
4881 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8:
4882 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
4883 case Intrinsic::amdgcn_perm_pk16_b4_u4:
4884 case Intrinsic::amdgcn_perm_pk16_b6_u4:
4885 case Intrinsic::amdgcn_perm_pk16_b8_u4:
4886 case Intrinsic::amdgcn_add_max_i32:
4887 case Intrinsic::amdgcn_add_max_u32:
4888 case Intrinsic::amdgcn_add_min_i32:
4889 case Intrinsic::amdgcn_add_min_u32:
4890 case Intrinsic::amdgcn_pk_add_max_i16:
4891 case Intrinsic::amdgcn_pk_add_max_u16:
4892 case Intrinsic::amdgcn_pk_add_min_i16:
4893 case Intrinsic::amdgcn_pk_add_min_u16:
4894 return getDefaultMappingVOP(MI);
4895 case Intrinsic::amdgcn_log:
4896 case Intrinsic::amdgcn_exp2:
4897 case Intrinsic::amdgcn_rcp:
4898 case Intrinsic::amdgcn_rsq:
4899 case Intrinsic::amdgcn_sqrt: {
4900 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4901 unsigned Size = Ty.getSizeInBits();
4902 // There is no pseudo scalar transcendental instruction for bf16.
4903 if (Subtarget.hasPseudoScalarTrans() && !Ty.isBFloat16() &&
4904 (Size == 16 || Size == 32) && isSALUMapping(MI))
4905 return getDefaultMappingSOP(MI);
4906 return getDefaultMappingVOP(MI);
4907 }
4908 case Intrinsic::amdgcn_sbfe:
4909 case Intrinsic::amdgcn_ubfe:
4910 if (isSALUMapping(MI))
4911 return getDefaultMappingSOP(MI);
4912 return getDefaultMappingVOP(MI);
4913 case Intrinsic::amdgcn_ds_swizzle:
4914 case Intrinsic::amdgcn_ds_permute:
4915 case Intrinsic::amdgcn_ds_bpermute:
4916 case Intrinsic::amdgcn_update_dpp:
4917 case Intrinsic::amdgcn_mov_dpp8:
4918 case Intrinsic::amdgcn_mov_dpp:
4919 case Intrinsic::amdgcn_strict_wwm:
4920 case Intrinsic::amdgcn_wwm:
4921 case Intrinsic::amdgcn_strict_wqm:
4922 case Intrinsic::amdgcn_wqm:
4923 case Intrinsic::amdgcn_softwqm:
4924 case Intrinsic::amdgcn_set_inactive:
4925 case Intrinsic::amdgcn_set_inactive_chain_arg:
4926 case Intrinsic::amdgcn_permlane64:
4927 case Intrinsic::amdgcn_ds_bpermute_fi_b32:
4928 return getDefaultMappingAllVGPR(MI);
4929 case Intrinsic::amdgcn_cvt_pkrtz:
4930 if (Subtarget.hasSALUFloatInsts() && isSALUMapping(MI))
4931 return getDefaultMappingSOP(MI);
4932 return getDefaultMappingVOP(MI);
4933 case Intrinsic::amdgcn_kernarg_segment_ptr:
4934 case Intrinsic::amdgcn_s_getpc:
4935 case Intrinsic::amdgcn_groupstaticsize:
4936 case Intrinsic::amdgcn_reloc_constant:
4937 case Intrinsic::returnaddress: {
4938 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4939 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4940 break;
4941 }
4942 case Intrinsic::amdgcn_wqm_vote: {
4943 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4944 OpdsMapping[0] = OpdsMapping[2]
4945 = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size);
4946 break;
4947 }
4948 case Intrinsic::amdgcn_ps_live: {
4949 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4950 break;
4951 }
4952 case Intrinsic::amdgcn_div_scale: {
4953 unsigned Dst0Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4954 unsigned Dst1Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4955 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Dst0Size);
4956 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: Dst1Size);
4957
4958 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 3).getReg()).getSizeInBits();
4959 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4960 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4961 break;
4962 }
4963 case Intrinsic::amdgcn_class: {
4964 Register Src0Reg = MI.getOperand(i: 2).getReg();
4965 Register Src1Reg = MI.getOperand(i: 3).getReg();
4966 unsigned Src0Size = MRI.getType(Reg: Src0Reg).getSizeInBits();
4967 unsigned Src1Size = MRI.getType(Reg: Src1Reg).getSizeInBits();
4968 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4969 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: DstSize);
4970 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Src0Size);
4971 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Src1Size);
4972 break;
4973 }
4974 case Intrinsic::amdgcn_readlane: {
4975 // This must be an SGPR, but accept a VGPR.
4976 Register IdxReg = MI.getOperand(i: 3).getReg();
4977 unsigned IdxSize = MRI.getType(Reg: IdxReg).getSizeInBits();
4978 unsigned IdxBank = getRegBankID(Reg: IdxReg, MRI, Default: AMDGPU::SGPRRegBankID);
4979 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4980 [[fallthrough]];
4981 }
4982 case Intrinsic::amdgcn_readfirstlane: {
4983 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4984 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4985 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
4986 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4987 break;
4988 }
4989 case Intrinsic::amdgcn_writelane: {
4990 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4991 Register SrcReg = MI.getOperand(i: 2).getReg();
4992 unsigned SrcSize = MRI.getType(Reg: SrcReg).getSizeInBits();
4993 unsigned SrcBank = getRegBankID(Reg: SrcReg, MRI, Default: AMDGPU::SGPRRegBankID);
4994 Register IdxReg = MI.getOperand(i: 3).getReg();
4995 unsigned IdxSize = MRI.getType(Reg: IdxReg).getSizeInBits();
4996 unsigned IdxBank = getRegBankID(Reg: IdxReg, MRI, Default: AMDGPU::SGPRRegBankID);
4997 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
4998
4999 // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
5000 // to legalize.
5001 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: SrcBank, Size: SrcSize);
5002 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
5003 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
5004 break;
5005 }
5006 case Intrinsic::amdgcn_if_break: {
5007 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5008 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5009 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5010 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5011 break;
5012 }
5013 case Intrinsic::amdgcn_permlane16:
5014 case Intrinsic::amdgcn_permlanex16: {
5015 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5016 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5017 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5018 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5019 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5020 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5021 break;
5022 }
5023 case Intrinsic::amdgcn_permlane_bcast:
5024 case Intrinsic::amdgcn_permlane_up:
5025 case Intrinsic::amdgcn_permlane_down:
5026 case Intrinsic::amdgcn_permlane_xor: {
5027 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5028 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5029 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5030 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5031 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5032 break;
5033 }
5034 case Intrinsic::amdgcn_permlane_idx_gen: {
5035 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5036 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5037 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5038 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5039 break;
5040 }
5041 case Intrinsic::amdgcn_permlane16_var:
5042 case Intrinsic::amdgcn_permlanex16_var: {
5043 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5044 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5045 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5046 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5047 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5048 break;
5049 }
5050 case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
5051 case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
5052 case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
5053 case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
5054 case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
5055 case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
5056 case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
5057 case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
5058 case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
5059 case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
5060 case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
5061 case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
5062 case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
5063 case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
5064 case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
5065 case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
5066 case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
5067 case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
5068 case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
5069 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
5070 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
5071 case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
5072 case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
5073 case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
5074 case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
5075 case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
5076 case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
5077 case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
5078 case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
5079 case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
5080 case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
5081 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
5082 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
5083 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
5084 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
5085 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
5086 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
5087 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
5088 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
5089 case Intrinsic::amdgcn_mfma_f32_16x16x32_f16:
5090 case Intrinsic::amdgcn_mfma_f32_32x32x16_f16:
5091 case Intrinsic::amdgcn_mfma_i32_16x16x64_i8:
5092 case Intrinsic::amdgcn_mfma_i32_32x32x32_i8:
5093 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf16: {
5094 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5095 unsigned MinNumRegsRequired = DstSize / 32;
5096
5097 // Default for MAI intrinsics.
5098 // srcC can also be an immediate which can be folded later.
5099 // FIXME: Should we eventually add an alternative mapping with AGPR src
5100 // for srcA/srcB?
5101 //
5102 // vdst, srcA, srcB, srcC
5103 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5104
5105 bool UseAGPRForm = !Subtarget.hasGFX90AInsts() ||
5106 Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5107
5108 OpdsMapping[0] =
5109 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI)
5110 : getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5111 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5112 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5113 OpdsMapping[4] =
5114 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5115 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5116 break;
5117 }
5118 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
5119 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
5120 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5121 unsigned MinNumRegsRequired = DstSize / 32;
5122
5123 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5124 bool UseAGPRForm = Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5125
5126 OpdsMapping[0] =
5127 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI)
5128 : getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5129
5130 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5131 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5132 OpdsMapping[4] =
5133 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5134 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5135
5136 OpdsMapping[8] = getVGPROpMapping(Reg: MI.getOperand(i: 8).getReg(), MRI, TRI: *TRI);
5137 OpdsMapping[10] = getVGPROpMapping(Reg: MI.getOperand(i: 10).getReg(), MRI, TRI: *TRI);
5138 break;
5139 }
5140 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
5141 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
5142 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
5143 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
5144 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
5145 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
5146 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
5147 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
5148 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
5149 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
5150 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
5151 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
5152 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
5153 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
5154 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
5155 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
5156 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
5157 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
5158 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
5159 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
5160 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
5161 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
5162 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
5163 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
5164 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
5165 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
5166 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
5167 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8: {
5168 Register DstReg = MI.getOperand(i: 0).getReg();
5169 unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
5170 unsigned MinNumRegsRequired = DstSize / 32;
5171 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5172 bool UseAGPRForm = Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5173
5174 // vdst, srcA, srcB, srcC, idx
5175 OpdsMapping[0] = UseAGPRForm ? getAGPROpMapping(Reg: DstReg, MRI, TRI: *TRI)
5176 : getVGPROpMapping(Reg: DstReg, MRI, TRI: *TRI);
5177
5178 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5179 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5180 OpdsMapping[4] =
5181 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5182 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5183 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5184 break;
5185 }
5186 case Intrinsic::amdgcn_interp_p1:
5187 case Intrinsic::amdgcn_interp_p2:
5188 case Intrinsic::amdgcn_interp_mov:
5189 case Intrinsic::amdgcn_interp_p1_f16:
5190 case Intrinsic::amdgcn_interp_p2_f16:
5191 case Intrinsic::amdgcn_lds_param_load: {
5192 const int M0Idx = MI.getNumOperands() - 1;
5193 Register M0Reg = MI.getOperand(i: M0Idx).getReg();
5194 unsigned M0Bank = getRegBankID(Reg: M0Reg, MRI, Default: AMDGPU::SGPRRegBankID);
5195 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5196
5197 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5198 for (int I = 2; I != M0Idx && MI.getOperand(i: I).isReg(); ++I)
5199 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5200
5201 // Must be SGPR, but we must take whatever the original bank is and fix it
5202 // later.
5203 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5204 break;
5205 }
5206 case Intrinsic::amdgcn_interp_inreg_p10:
5207 case Intrinsic::amdgcn_interp_inreg_p2:
5208 case Intrinsic::amdgcn_interp_inreg_p10_f16:
5209 case Intrinsic::amdgcn_interp_inreg_p2_f16:
5210 case Intrinsic::amdgcn_interp_p10_rtz_f16:
5211 case Intrinsic::amdgcn_interp_p2_rtz_f16: {
5212 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5213 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5214 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5215 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5216 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5217 break;
5218 }
5219 case Intrinsic::amdgcn_permlane16_swap:
5220 case Intrinsic::amdgcn_permlane32_swap: {
5221 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5222 OpdsMapping[0] = OpdsMapping[1] = OpdsMapping[3] = OpdsMapping[4] =
5223 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5224 break;
5225 }
5226 case Intrinsic::amdgcn_ballot: {
5227 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5228 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5229 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
5230 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: SrcSize);
5231 break;
5232 }
5233 case Intrinsic::amdgcn_inverse_ballot: {
5234 // This must be an SGPR, but accept a VGPR.
5235 Register MaskReg = MI.getOperand(i: 2).getReg();
5236 unsigned MaskSize = MRI.getType(Reg: MaskReg).getSizeInBits();
5237 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5238 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5239 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: MaskSize);
5240 break;
5241 }
5242 case Intrinsic::amdgcn_bitop3: {
5243 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5244 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5245 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5246 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5247 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5248 break;
5249 }
5250 case Intrinsic::amdgcn_s_quadmask:
5251 case Intrinsic::amdgcn_s_wqm: {
5252 Register MaskReg = MI.getOperand(i: 2).getReg();
5253 unsigned MaskSize = MRI.getType(Reg: MaskReg).getSizeInBits();
5254 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5255 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: MaskSize);
5256 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: MaskSize);
5257 break;
5258 }
5259 case Intrinsic::amdgcn_wave_reduce_add:
5260 case Intrinsic::amdgcn_wave_reduce_fadd:
5261 case Intrinsic::amdgcn_wave_reduce_sub:
5262 case Intrinsic::amdgcn_wave_reduce_fsub:
5263 case Intrinsic::amdgcn_wave_reduce_min:
5264 case Intrinsic::amdgcn_wave_reduce_umin:
5265 case Intrinsic::amdgcn_wave_reduce_fmin:
5266 case Intrinsic::amdgcn_wave_reduce_max:
5267 case Intrinsic::amdgcn_wave_reduce_umax:
5268 case Intrinsic::amdgcn_wave_reduce_fmax:
5269 case Intrinsic::amdgcn_wave_reduce_and:
5270 case Intrinsic::amdgcn_wave_reduce_or:
5271 case Intrinsic::amdgcn_wave_reduce_xor: {
5272 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5273 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
5274 unsigned OpSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5275 auto regBankID =
5276 isSALUMapping(MI) ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5277 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: regBankID, Size: OpSize);
5278 break;
5279 }
5280 case Intrinsic::amdgcn_s_bitreplicate: {
5281 Register MaskReg = MI.getOperand(i: 2).getReg();
5282 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5283 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 64);
5284 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: 32);
5285 break;
5286 }
5287 case Intrinsic::amdgcn_wave_shuffle: {
5288 unsigned OpSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5289 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5290 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5291 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5292 break;
5293 }
5294 }
5295 break;
5296 }
5297 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
5298 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
5299 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
5300 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
5301 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
5302 auto IntrID = AMDGPU::getIntrinsicID(I: MI);
5303 const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(Intr: IntrID);
5304 assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic");
5305 // Non-images can have complications from operands that allow both SGPR
5306 // and VGPR. For now it's too complicated to figure out the final opcode
5307 // to derive the register bank from the MCInstrDesc.
5308 assert(RSrcIntrin->IsImage);
5309 return getImageMapping(MRI, MI, RsrcIdx: RSrcIntrin->RsrcArg);
5310 }
5311 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
5312 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
5313 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
5314 bool IsDualOrBVH8 =
5315 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
5316 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
5317 unsigned NumMods = IsDualOrBVH8 ? 0 : 1; // Has A16 modifier
5318 unsigned LastRegOpIdx = MI.getNumExplicitOperands() - 1 - NumMods;
5319 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5320 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5321 if (IsDualOrBVH8) {
5322 OpdsMapping[1] = AMDGPU::getValueMapping(
5323 BankID: AMDGPU::VGPRRegBankID,
5324 Size: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits());
5325 OpdsMapping[2] = AMDGPU::getValueMapping(
5326 BankID: AMDGPU::VGPRRegBankID,
5327 Size: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
5328 }
5329 OpdsMapping[LastRegOpIdx] =
5330 getSGPROpMapping(Reg: MI.getOperand(i: LastRegOpIdx).getReg(), MRI, TRI: *TRI);
5331 if (LastRegOpIdx == 3) {
5332 // Sequential form: all operands combined into VGPR256/VGPR512
5333 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5334 if (Size > 256)
5335 Size = 512;
5336 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5337 } else {
5338 // NSA form
5339 unsigned FirstSrcOpIdx = IsDualOrBVH8 ? 4 : 2;
5340 for (unsigned I = FirstSrcOpIdx; I < LastRegOpIdx; ++I) {
5341 unsigned Size = MRI.getType(Reg: MI.getOperand(i: I).getReg()).getSizeInBits();
5342 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5343 }
5344 }
5345 break;
5346 }
5347 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
5348 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
5349 auto IntrID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
5350 switch (IntrID) {
5351 case Intrinsic::amdgcn_s_getreg:
5352 case Intrinsic::amdgcn_s_memtime:
5353 case Intrinsic::amdgcn_s_memrealtime:
5354 case Intrinsic::amdgcn_s_get_waveid_in_workgroup:
5355 case Intrinsic::amdgcn_s_sendmsg_rtn: {
5356 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5357 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5358 break;
5359 }
5360 case Intrinsic::amdgcn_global_atomic_fmin_num:
5361 case Intrinsic::amdgcn_global_atomic_fmax_num:
5362 case Intrinsic::amdgcn_flat_atomic_fmin_num:
5363 case Intrinsic::amdgcn_flat_atomic_fmax_num:
5364 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
5365 case Intrinsic::amdgcn_global_load_tr_b64:
5366 case Intrinsic::amdgcn_global_load_tr_b128:
5367 case Intrinsic::amdgcn_global_load_tr4_b64:
5368 case Intrinsic::amdgcn_global_load_tr6_b96:
5369 case Intrinsic::amdgcn_ds_load_tr8_b64:
5370 case Intrinsic::amdgcn_ds_load_tr16_b128:
5371 case Intrinsic::amdgcn_ds_load_tr4_b64:
5372 case Intrinsic::amdgcn_ds_load_tr6_b96:
5373 case Intrinsic::amdgcn_ds_read_tr4_b64:
5374 case Intrinsic::amdgcn_ds_read_tr6_b96:
5375 case Intrinsic::amdgcn_ds_read_tr8_b64:
5376 case Intrinsic::amdgcn_ds_read_tr16_b64:
5377 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
5378 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
5379 return getDefaultMappingAllVGPR(MI);
5380 case Intrinsic::amdgcn_ds_ordered_add:
5381 case Intrinsic::amdgcn_ds_ordered_swap: {
5382 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5383 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5384 unsigned M0Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5385 Default: AMDGPU::SGPRRegBankID);
5386 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5387 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5388 break;
5389 }
5390 case Intrinsic::amdgcn_ds_append:
5391 case Intrinsic::amdgcn_ds_consume: {
5392 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5393 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5394 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5395 break;
5396 }
5397 case Intrinsic::amdgcn_exp_compr:
5398 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5399 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5400 break;
5401 case Intrinsic::amdgcn_exp:
5402 // FIXME: Could we support packed types here?
5403 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5404 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5405 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5406 OpdsMapping[6] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5407 break;
5408 case Intrinsic::amdgcn_exp_row:
5409 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5410 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5411 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5412 OpdsMapping[6] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5413 OpdsMapping[8] = getSGPROpMapping(Reg: MI.getOperand(i: 8).getReg(), MRI, TRI: *TRI);
5414 break;
5415 case Intrinsic::amdgcn_s_alloc_vgpr:
5416 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1);
5417 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
5418 break;
5419 case Intrinsic::amdgcn_s_sendmsg:
5420 case Intrinsic::amdgcn_s_sendmsghalt: {
5421 // This must be an SGPR, but accept a VGPR.
5422 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5423 Default: AMDGPU::SGPRRegBankID);
5424 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5425 break;
5426 }
5427 case Intrinsic::amdgcn_s_setreg: {
5428 // This must be an SGPR, but accept a VGPR.
5429 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5430 Default: AMDGPU::SGPRRegBankID);
5431 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5432 break;
5433 }
5434 case Intrinsic::amdgcn_s_ttracedata: {
5435 // This must be an SGPR, but accept a VGPR.
5436 unsigned Bank =
5437 getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5438 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5439 break;
5440 }
5441 case Intrinsic::amdgcn_end_cf: {
5442 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5443 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5444 break;
5445 }
5446 case Intrinsic::amdgcn_else: {
5447 unsigned WaveSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5448 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5449 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: WaveSize);
5450 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: WaveSize);
5451 break;
5452 }
5453 case Intrinsic::amdgcn_init_whole_wave:
5454 case Intrinsic::amdgcn_live_mask: {
5455 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5456 break;
5457 }
5458 case Intrinsic::amdgcn_wqm_demote:
5459 case Intrinsic::amdgcn_kill: {
5460 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5461 break;
5462 }
5463 case Intrinsic::amdgcn_ptr_s_buffer_load: {
5464 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5465 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5466 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5467 break;
5468 }
5469 case Intrinsic::amdgcn_raw_buffer_load:
5470 case Intrinsic::amdgcn_raw_ptr_buffer_load:
5471 case Intrinsic::amdgcn_raw_atomic_buffer_load:
5472 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
5473 case Intrinsic::amdgcn_raw_tbuffer_load:
5474 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
5475 // FIXME: Should make intrinsic ID the last operand of the instruction,
5476 // then this would be the same as store
5477 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5478 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5479 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5480 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5481 break;
5482 }
5483 case Intrinsic::amdgcn_raw_buffer_load_lds:
5484 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
5485 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
5486 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: {
5487 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5488 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5489 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5490 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5491 break;
5492 }
5493 case Intrinsic::amdgcn_raw_buffer_store:
5494 case Intrinsic::amdgcn_raw_ptr_buffer_store:
5495 case Intrinsic::amdgcn_raw_buffer_store_format:
5496 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
5497 case Intrinsic::amdgcn_raw_tbuffer_store:
5498 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
5499 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5500 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5501 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5502 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5503 break;
5504 }
5505 case Intrinsic::amdgcn_struct_buffer_load:
5506 case Intrinsic::amdgcn_struct_ptr_buffer_load:
5507 case Intrinsic::amdgcn_struct_tbuffer_load:
5508 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
5509 case Intrinsic::amdgcn_struct_atomic_buffer_load:
5510 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
5511 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5512 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5513 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5514 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5515 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5516 break;
5517 }
5518 case Intrinsic::amdgcn_struct_buffer_load_lds:
5519 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
5520 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
5521 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
5522 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5523 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5524 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5525 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5526 OpdsMapping[6] = getSGPROpMapping(Reg: MI.getOperand(i: 6).getReg(), MRI, TRI: *TRI);
5527 break;
5528 }
5529 case Intrinsic::amdgcn_struct_buffer_store:
5530 case Intrinsic::amdgcn_struct_ptr_buffer_store:
5531 case Intrinsic::amdgcn_struct_tbuffer_store:
5532 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
5533 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5534 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5535 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5536 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5537 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5538 break;
5539 }
5540 case Intrinsic::amdgcn_init_exec_from_input: {
5541 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5542 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5543 break;
5544 }
5545 case Intrinsic::amdgcn_ds_gws_init:
5546 case Intrinsic::amdgcn_ds_gws_barrier:
5547 case Intrinsic::amdgcn_ds_gws_sema_br: {
5548 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5549
5550 // This must be an SGPR, but accept a VGPR.
5551 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5552 Default: AMDGPU::SGPRRegBankID);
5553 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5554 break;
5555 }
5556 case Intrinsic::amdgcn_ds_gws_sema_v:
5557 case Intrinsic::amdgcn_ds_gws_sema_p:
5558 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
5559 // This must be an SGPR, but accept a VGPR.
5560 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
5561 Default: AMDGPU::SGPRRegBankID);
5562 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5563 break;
5564 }
5565 case Intrinsic::amdgcn_cluster_load_b32:
5566 case Intrinsic::amdgcn_cluster_load_b64:
5567 case Intrinsic::amdgcn_cluster_load_b128: {
5568 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5569 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5570 unsigned M0Bank =
5571 getRegBankID(Reg: MI.getOperand(i: 4).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5572 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5573 break;
5574 }
5575 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
5576 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
5577 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
5578 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
5579 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5580 // LDS address goes into $vdst (VGPR).
5581 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5582 unsigned M0Bank =
5583 getRegBankID(Reg: MI.getOperand(i: 5).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5584 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5585 break;
5586 }
5587 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
5588 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
5589 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
5590 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
5591 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
5592 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
5593 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
5594 case Intrinsic::amdgcn_global_load_async_to_lds_b128: {
5595 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5596 // LDS address goes into $vdst/$vdata (VGPR).
5597 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5598 break;
5599 }
5600 case Intrinsic::amdgcn_load_to_lds:
5601 case Intrinsic::amdgcn_load_async_to_lds:
5602 case Intrinsic::amdgcn_global_load_lds:
5603 case Intrinsic::amdgcn_global_load_async_lds: {
5604 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5605 // LDS address goes into M0 (SGPR).
5606 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5607 break;
5608 }
5609 case Intrinsic::amdgcn_lds_direct_load: {
5610 const int M0Idx = MI.getNumOperands() - 1;
5611 Register M0Reg = MI.getOperand(i: M0Idx).getReg();
5612 unsigned M0Bank = getRegBankID(Reg: M0Reg, MRI, Default: AMDGPU::SGPRRegBankID);
5613 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5614
5615 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5616 for (int I = 2; I != M0Idx && MI.getOperand(i: I).isReg(); ++I)
5617 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5618
5619 // Must be SGPR, but we must take whatever the original bank is and fix it
5620 // later.
5621 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5622 break;
5623 }
5624 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
5625 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn:
5626 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5627 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5628 break;
5629 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
5630 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
5631 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
5632 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
5633 OpdsMapping[0] =
5634 getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI); // %vdst
5635 OpdsMapping[1] =
5636 getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI); // %addr
5637 OpdsMapping[3] =
5638 getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI); // %addr
5639 OpdsMapping[4] =
5640 getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI); // %data0
5641 OpdsMapping[5] =
5642 getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI); // %data1
5643 break;
5644 }
5645 case Intrinsic::amdgcn_s_sleep_var:
5646 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5647 break;
5648 case Intrinsic::amdgcn_s_barrier_join:
5649 case Intrinsic::amdgcn_s_wakeup_barrier:
5650 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5651 break;
5652 case Intrinsic::amdgcn_s_barrier_init:
5653 case Intrinsic::amdgcn_s_barrier_signal_var:
5654 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5655 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5656 break;
5657 case Intrinsic::amdgcn_s_barrier_signal_isfirst: {
5658 const unsigned ResultSize = 1;
5659 OpdsMapping[0] =
5660 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: ResultSize);
5661 break;
5662 }
5663 case Intrinsic::amdgcn_s_get_barrier_state:
5664 case Intrinsic::amdgcn_s_get_named_barrier_state: {
5665 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5666 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5667 break;
5668 }
5669 case Intrinsic::amdgcn_pops_exiting_wave_id:
5670 return getDefaultMappingSOP(MI);
5671 case Intrinsic::amdgcn_tensor_load_to_lds:
5672 case Intrinsic::amdgcn_tensor_store_from_lds: {
5673 // Lie and claim everything is legal, even all operands need to be
5674 // SGPRs. applyMapping will have to deal with it with readfirstlane.
5675 for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
5676 if (MI.getOperand(i: I).isReg()) {
5677 Register Reg = MI.getOperand(i: I).getReg();
5678 auto OpBank = getRegBankID(Reg, MRI);
5679 unsigned Size = getSizeInBits(Reg, MRI, TRI: *TRI);
5680 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: OpBank, Size);
5681 }
5682 }
5683 break;
5684 }
5685 case Intrinsic::amdgcn_s_prefetch_data:
5686 case Intrinsic::amdgcn_s_prefetch_inst: {
5687 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5688 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5689 break;
5690 }
5691 case Intrinsic::amdgcn_flat_prefetch:
5692 case Intrinsic::amdgcn_global_prefetch:
5693 return getDefaultMappingVOP(MI);
5694 default:
5695 return getInvalidInstructionMapping();
5696 }
5697 break;
5698 }
5699 case AMDGPU::G_SELECT: {
5700 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5701 unsigned Op2Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5702 Default: AMDGPU::SGPRRegBankID);
5703 unsigned Op3Bank = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI,
5704 Default: AMDGPU::SGPRRegBankID);
5705 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
5706 Op3Bank == AMDGPU::SGPRRegBankID;
5707
5708 unsigned CondBankDefault = SGPRSrcs ?
5709 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5710 unsigned CondBank = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
5711 Default: CondBankDefault);
5712 if (CondBank == AMDGPU::SGPRRegBankID)
5713 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5714 else if (CondBank == AMDGPU::VGPRRegBankID)
5715 CondBank = AMDGPU::VCCRegBankID;
5716
5717 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ?
5718 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5719
5720 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID);
5721
5722 // TODO: Should report 32-bit for scalar condition type.
5723 if (Size == 64) {
5724 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5725 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: CondBank, Size: 1);
5726 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5727 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5728 } else {
5729 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size);
5730 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: CondBank, Size: 1);
5731 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size);
5732 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: Bank, Size);
5733 }
5734
5735 break;
5736 }
5737
5738 case AMDGPU::G_SI_CALL: {
5739 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 64);
5740 // Lie and claim everything is legal, even though some need to be
5741 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
5742 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5743
5744 // Allow anything for implicit arguments
5745 for (unsigned I = 4; I < MI.getNumOperands(); ++I) {
5746 if (MI.getOperand(i: I).isReg()) {
5747 Register Reg = MI.getOperand(i: I).getReg();
5748 auto OpBank = getRegBankID(Reg, MRI);
5749 unsigned Size = getSizeInBits(Reg, MRI, TRI: *TRI);
5750 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: OpBank, Size);
5751 }
5752 }
5753 break;
5754 }
5755 case AMDGPU::G_LOAD:
5756 case AMDGPU::G_ZEXTLOAD:
5757 case AMDGPU::G_SEXTLOAD:
5758 return getInstrMappingForLoad(MI);
5759
5760 case AMDGPU::G_ATOMICRMW_XCHG:
5761 case AMDGPU::G_ATOMICRMW_ADD:
5762 case AMDGPU::G_ATOMICRMW_SUB:
5763 case AMDGPU::G_ATOMICRMW_AND:
5764 case AMDGPU::G_ATOMICRMW_OR:
5765 case AMDGPU::G_ATOMICRMW_XOR:
5766 case AMDGPU::G_ATOMICRMW_MAX:
5767 case AMDGPU::G_ATOMICRMW_MIN:
5768 case AMDGPU::G_ATOMICRMW_UMAX:
5769 case AMDGPU::G_ATOMICRMW_UMIN:
5770 case AMDGPU::G_ATOMICRMW_FADD:
5771 case AMDGPU::G_ATOMICRMW_FMIN:
5772 case AMDGPU::G_ATOMICRMW_FMAX:
5773 case AMDGPU::G_ATOMICRMW_UINC_WRAP:
5774 case AMDGPU::G_ATOMICRMW_UDEC_WRAP:
5775 case AMDGPU::G_ATOMICRMW_USUB_COND:
5776 case AMDGPU::G_ATOMICRMW_USUB_SAT:
5777 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: {
5778 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5779 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
5780 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5781 break;
5782 }
5783 case AMDGPU::G_ATOMIC_CMPXCHG: {
5784 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5785 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
5786 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5787 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5788 break;
5789 }
5790 case AMDGPU::G_BRCOND: {
5791 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 0).getReg(), MRI,
5792 Default: AMDGPU::SGPRRegBankID);
5793 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
5794 if (Bank != AMDGPU::SGPRRegBankID)
5795 Bank = AMDGPU::VCCRegBankID;
5796
5797 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: 1);
5798 break;
5799 }
5800 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
5801 return getDefaultMappingVOP(MI);
5802 case AMDGPU::G_PREFETCH:
5803 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5804 break;
5805 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP:
5806 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN:
5807 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5808 break;
5809 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
5810 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR: {
5811 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5812 unsigned PtrSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5813 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5814 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize);
5815 break;
5816 }
5817 }
5818
5819 return getInstructionMapping(/*ID*/1, /*Cost*/1,
5820 OperandsMapping: getOperandsMapping(OpdsMapping),
5821 NumOperands: MI.getNumOperands());
5822}
5823