1//===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AMDGPU.
11///
12/// \par
13///
14/// AMDGPU has unique register bank constraints that require special high level
15/// strategies to deal with. There are two main true physical register banks
16/// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a
17/// sort of pseudo-register bank needed to represent SGPRs used in a vector
18/// boolean context. There is also the AGPR bank, which is a special purpose
19/// physical register bank present on some subtargets.
20///
21/// Copying from VGPR to SGPR is generally illegal, unless the value is known to
22/// be uniform. It is generally not valid to legalize operands by inserting
23/// copies as on other targets. Operations which require uniform, SGPR operands
24/// generally require scalarization by repeatedly executing the instruction,
25/// activating each set of lanes using a unique set of input values. This is
26/// referred to as a waterfall loop.
27///
28/// \par Booleans
29///
30/// Booleans (s1 values) requires special consideration. A vector compare result
31/// is naturally a bitmask with one bit per lane, in a 32 or 64-bit
32/// register. These are represented with the VCC bank. During selection, we need
33/// to be able to unambiguously go back from a register class to a register
34/// bank. To distinguish whether an SGPR should use the SGPR or VCC register
35/// bank, we need to know the use context type. An SGPR s1 value always means a
36/// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets
37/// SCC, which is a 1-bit unaddressable register. This will need to be copied to
38/// a 32-bit virtual register. Taken together, this means we need to adjust the
39/// type of boolean operations to be regbank legal. All SALU booleans need to be
40/// widened to 32-bits, and all VALU booleans need to be s1 values.
41///
42/// A noteworthy exception to the s1-means-vcc rule is for legalization artifact
43/// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc
44/// bank. A non-boolean source (such as a truncate from a 1-bit load from
45/// memory) will require a copy to the VCC bank which will require clearing the
46/// high bits and inserting a compare.
47///
48/// \par Constant bus restriction
49///
50/// VALU instructions have a limitation known as the constant bus
51/// restriction. Most VALU instructions can use SGPR operands, but may read at
52/// most 1 SGPR or constant literal value (this to 2 in gfx10 for most
53/// instructions). This is one unique SGPR, so the same SGPR may be used for
54/// multiple operands. From a register bank perspective, any combination of
55/// operands should be legal as an SGPR, but this is contextually dependent on
56/// the SGPR operands all being the same register. There is therefore optimal to
57/// choose the SGPR with the most uses to minimize the number of copies.
58///
59/// We avoid trying to solve this problem in RegBankSelect. Any VALU G_*
60/// operation should have its source operands all mapped to VGPRs (except for
61/// VCC), inserting copies from any SGPR operands. This the most trivial legal
62/// mapping. Anything beyond the simplest 1:1 instruction selection would be too
63/// complicated to solve here. Every optimization pattern or instruction
64/// selected to multiple outputs would have to enforce this rule, and there
65/// would be additional complexity in tracking this rule for every G_*
66/// operation. By forcing all inputs to VGPRs, it also simplifies the task of
67/// picking the optimal operand combination from a post-isel optimization pass.
68///
69//===----------------------------------------------------------------------===//
70
71#include "AMDGPURegisterBankInfo.h"
72
73#include "AMDGPUGlobalISelUtils.h"
74#include "AMDGPUInstrInfo.h"
75#include "AMDGPULaneMaskUtils.h"
76#include "GCNSubtarget.h"
77#include "SIMachineFunctionInfo.h"
78#include "SIRegisterInfo.h"
79#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
80#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
81#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
82#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
83#include "llvm/CodeGen/RegisterBank.h"
84#include "llvm/IR/IntrinsicsAMDGPU.h"
85
86#define GET_TARGET_REGBANK_IMPL
87#include "AMDGPUGenRegisterBank.inc"
88
89// This file will be TableGen'ed at some point.
90#include "AMDGPUGenRegisterBankInfo.def"
91
92using namespace llvm;
93using namespace MIPatternMatch;
94
95namespace {
96
97// Observer to apply a register bank to new registers created by LegalizerHelper.
98class ApplyRegBankMapping final : public GISelChangeObserver {
99private:
100 MachineIRBuilder &B;
101 const AMDGPURegisterBankInfo &RBI;
102 MachineRegisterInfo &MRI;
103 const RegisterBank *NewBank;
104 SmallVector<MachineInstr *, 4> NewInsts;
105
106public:
107 ApplyRegBankMapping(MachineIRBuilder &B, const AMDGPURegisterBankInfo &RBI_,
108 MachineRegisterInfo &MRI_, const RegisterBank *RB)
109 : B(B), RBI(RBI_), MRI(MRI_), NewBank(RB) {
110 assert(!B.isObservingChanges());
111 B.setChangeObserver(*this);
112 }
113
114 ~ApplyRegBankMapping() override {
115 for (MachineInstr *MI : NewInsts)
116 applyBank(MI&: *MI);
117
118 B.stopObservingChanges();
119 }
120
121 /// Set any registers that don't have a set register class or bank to SALU.
122 void applyBank(MachineInstr &MI) {
123 const unsigned Opc = MI.getOpcode();
124 if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT ||
125 Opc == AMDGPU::G_SEXT) {
126 // LegalizerHelper wants to use the basic legalization artifacts when
127 // widening etc. We don't handle selection with vcc in artifact sources,
128 // so we need to use a select instead to handle these properly.
129 Register DstReg = MI.getOperand(i: 0).getReg();
130 Register SrcReg = MI.getOperand(i: 1).getReg();
131 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI, TRI: *RBI.TRI);
132 if (SrcBank == &AMDGPU::VCCRegBank) {
133 const LLT S32 = LLT::scalar(SizeInBits: 32);
134 assert(MRI.getType(SrcReg) == LLT::scalar(1));
135 assert(MRI.getType(DstReg) == S32);
136 assert(NewBank == &AMDGPU::VGPRRegBank);
137
138 // Replace the extension with a select, which really uses the boolean
139 // source.
140 B.setInsertPt(MBB&: *MI.getParent(), II: MI);
141
142 auto True = B.buildConstant(Res: S32, Val: Opc == AMDGPU::G_SEXT ? -1 : 1);
143 auto False = B.buildConstant(Res: S32, Val: 0);
144 B.buildSelect(Res: DstReg, Tst: SrcReg, Op0: True, Op1: False);
145 MRI.setRegBank(Reg: True.getReg(Idx: 0), RegBank: *NewBank);
146 MRI.setRegBank(Reg: False.getReg(Idx: 0), RegBank: *NewBank);
147 MI.eraseFromParent();
148 }
149
150 assert(!MRI.getRegClassOrRegBank(DstReg));
151 MRI.setRegBank(Reg: DstReg, RegBank: *NewBank);
152 return;
153 }
154
155#ifndef NDEBUG
156 if (Opc == AMDGPU::G_TRUNC) {
157 Register DstReg = MI.getOperand(0).getReg();
158 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI);
159 assert(DstBank != &AMDGPU::VCCRegBank);
160 }
161#endif
162
163 for (MachineOperand &Op : MI.operands()) {
164 if (!Op.isReg())
165 continue;
166
167 // We may see physical registers if building a real MI
168 Register Reg = Op.getReg();
169 if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg))
170 continue;
171
172 const RegisterBank *RB = NewBank;
173 if (MRI.getType(Reg) == LLT::scalar(SizeInBits: 1)) {
174 assert(NewBank == &AMDGPU::VGPRRegBank &&
175 "s1 operands should only be used for vector bools");
176 assert((MI.getOpcode() != AMDGPU::G_TRUNC &&
177 MI.getOpcode() != AMDGPU::G_ANYEXT) &&
178 "not expecting legalization artifacts here");
179 RB = &AMDGPU::VCCRegBank;
180 }
181
182 MRI.setRegBank(Reg, RegBank: *RB);
183 }
184 }
185
186 void erasingInstr(MachineInstr &MI) override {}
187
188 void createdInstr(MachineInstr &MI) override {
189 // At this point, the instruction was just inserted and has no operands.
190 NewInsts.push_back(Elt: &MI);
191 }
192
193 void changingInstr(MachineInstr &MI) override {}
194 void changedInstr(MachineInstr &MI) override {
195 // FIXME: In principle we should probably add the instruction to NewInsts,
196 // but the way the LegalizerHelper uses the observer, we will always see the
197 // registers we need to set the regbank on also referenced in a new
198 // instruction.
199 }
200};
201
202} // anonymous namespace
203
204AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
205 : Subtarget(ST), TRI(Subtarget.getRegisterInfo()),
206 TII(Subtarget.getInstrInfo()) {
207
208 // HACK: Until this is fully tablegen'd.
209 static llvm::once_flag InitializeRegisterBankFlag;
210
211 static auto InitializeRegisterBankOnce = [this]() {
212 assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank &&
213 &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank &&
214 &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank);
215 (void)this;
216 };
217
218 llvm::call_once(flag&: InitializeRegisterBankFlag, F&: InitializeRegisterBankOnce);
219}
220
221static bool isVectorRegisterBank(const RegisterBank &Bank) {
222 unsigned BankID = Bank.getID();
223 return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID;
224}
225
226bool AMDGPURegisterBankInfo::isDivergentRegBank(const RegisterBank *RB) const {
227 return RB != &AMDGPU::SGPRRegBank;
228}
229
230unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
231 const RegisterBank &Src,
232 TypeSize Size) const {
233 // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane?
234 if (Dst.getID() == AMDGPU::SGPRRegBankID &&
235 (isVectorRegisterBank(Bank: Src) || Src.getID() == AMDGPU::VCCRegBankID)) {
236 return std::numeric_limits<unsigned>::max();
237 }
238
239 // Bool values are tricky, because the meaning is based on context. The SCC
240 // and VCC banks are for the natural scalar and vector conditions produced by
241 // a compare.
242 //
243 // Legalization doesn't know about the necessary context, so an s1 use may
244 // have been a truncate from an arbitrary value, in which case a copy (lowered
245 // as a compare with 0) needs to be inserted.
246 if (Size == 1 &&
247 (Dst.getID() == AMDGPU::SGPRRegBankID) &&
248 (isVectorRegisterBank(Bank: Src) ||
249 Src.getID() == AMDGPU::SGPRRegBankID ||
250 Src.getID() == AMDGPU::VCCRegBankID))
251 return std::numeric_limits<unsigned>::max();
252
253 // There is no direct copy between AGPRs.
254 if (Dst.getID() == AMDGPU::AGPRRegBankID &&
255 Src.getID() == AMDGPU::AGPRRegBankID)
256 return 4;
257
258 return RegisterBankInfo::copyCost(A: Dst, B: Src, Size);
259}
260
261unsigned AMDGPURegisterBankInfo::getBreakDownCost(
262 const ValueMapping &ValMapping,
263 const RegisterBank *CurBank) const {
264 // Check if this is a breakdown for G_LOAD to move the pointer from SGPR to
265 // VGPR.
266 // FIXME: Is there a better way to do this?
267 if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64)
268 return 10; // This is expensive.
269
270 assert(ValMapping.NumBreakDowns == 2 &&
271 ValMapping.BreakDown[0].Length == 32 &&
272 ValMapping.BreakDown[0].StartIdx == 0 &&
273 ValMapping.BreakDown[1].Length == 32 &&
274 ValMapping.BreakDown[1].StartIdx == 32 &&
275 ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank);
276
277 // 32-bit extract of a 64-bit value is just access of a subregister, so free.
278 // TODO: Cost of 0 hits assert, though it's not clear it's what we really
279 // want.
280
281 // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
282 // alignment restrictions, but this probably isn't important.
283 return 1;
284}
285
286const RegisterBank &
287AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
288 LLT Ty) const {
289 // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a
290 // VCC-like use.
291 if (TRI->isSGPRClass(RC: &RC)) {
292 // FIXME: This probably came from a copy from a physical register, which
293 // should be inferable from the copied to-type. We don't have many boolean
294 // physical register constraints so just assume a normal SGPR for now.
295 if (!Ty.isValid())
296 return AMDGPU::SGPRRegBank;
297
298 return Ty == LLT::scalar(SizeInBits: 1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
299 }
300
301 return TRI->isAGPRClass(RC: &RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank;
302}
303
304template <unsigned NumOps>
305RegisterBankInfo::InstructionMappings
306AMDGPURegisterBankInfo::addMappingFromTable(
307 const MachineInstr &MI, const MachineRegisterInfo &MRI,
308 const std::array<unsigned, NumOps> RegSrcOpIdx,
309 ArrayRef<OpRegBankEntry<NumOps>> Table) const {
310
311 InstructionMappings AltMappings;
312
313 SmallVector<const ValueMapping *, 10> Operands(MI.getNumOperands());
314
315 unsigned Sizes[NumOps];
316 for (unsigned I = 0; I < NumOps; ++I) {
317 Register Reg = MI.getOperand(RegSrcOpIdx[I]).getReg();
318 Sizes[I] = TRI->getRegSizeInBits(Reg, MRI);
319 }
320
321 for (unsigned I = 0, E = MI.getNumExplicitDefs(); I != E; ++I) {
322 unsigned SizeI = TRI->getRegSizeInBits(Reg: MI.getOperand(i: I).getReg(), MRI);
323 Operands[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SizeI);
324 }
325
326 // getInstrMapping's default mapping uses ID 1, so start at 2.
327 unsigned MappingID = 2;
328 for (const auto &Entry : Table) {
329 for (unsigned I = 0; I < NumOps; ++I) {
330 int OpIdx = RegSrcOpIdx[I];
331 Operands[OpIdx] = AMDGPU::getValueMapping(BankID: Entry.RegBanks[I], Size: Sizes[I]);
332 }
333
334 AltMappings.push_back(Elt: &getInstructionMapping(ID: MappingID++, Cost: Entry.Cost,
335 OperandsMapping: getOperandsMapping(OpdsMapping: Operands),
336 NumOperands: Operands.size()));
337 }
338
339 return AltMappings;
340}
341
342RegisterBankInfo::InstructionMappings
343AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(
344 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
345 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
346 case Intrinsic::amdgcn_readlane: {
347 static const OpRegBankEntry<3> Table[2] = {
348 // Perfectly legal.
349 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1 },
350
351 // Need a readfirstlane for the index.
352 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 }
353 };
354
355 const std::array<unsigned, 3> RegSrcOpIdx = { ._M_elems: { 0, 2, 3 } };
356 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, Table);
357 }
358 case Intrinsic::amdgcn_writelane: {
359 static const OpRegBankEntry<4> Table[4] = {
360 // Perfectly legal.
361 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1 },
362
363 // Need readfirstlane of first op
364 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 },
365
366 // Need readfirstlane of second op
367 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 },
368
369 // Need readfirstlane of both ops
370 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 3 }
371 };
372
373 // rsrc, voffset, offset
374 const std::array<unsigned, 4> RegSrcOpIdx = { ._M_elems: { 0, 2, 3, 4 } };
375 return addMappingFromTable<4>(MI, MRI, RegSrcOpIdx, Table);
376 }
377 default:
378 return RegisterBankInfo::getInstrAlternativeMappings(MI);
379 }
380}
381
382RegisterBankInfo::InstructionMappings
383AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(
384 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
385
386 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
387 case Intrinsic::amdgcn_ds_ordered_add:
388 case Intrinsic::amdgcn_ds_ordered_swap: {
389 // VGPR = M0, VGPR
390 static const OpRegBankEntry<3> Table[2] = {
391 // Perfectly legal.
392 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1 },
393
394 // Need a readfirstlane for m0
395 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 }
396 };
397
398 const std::array<unsigned, 3> RegSrcOpIdx = { ._M_elems: { 0, 2, 3 } };
399 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, Table);
400 }
401 case Intrinsic::amdgcn_s_sendmsg:
402 case Intrinsic::amdgcn_s_sendmsghalt: {
403 // FIXME: Should have no register for immediate
404 static const OpRegBankEntry<1> Table[2] = {
405 // Perfectly legal.
406 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 },
407
408 // Need readlane
409 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 3 }
410 };
411
412 const std::array<unsigned, 1> RegSrcOpIdx = { ._M_elems: { 2 } };
413 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx, Table);
414 }
415 default:
416 return RegisterBankInfo::getInstrAlternativeMappings(MI);
417 }
418}
419
420// FIXME: Returns uniform if there's no source value information. This is
421// probably wrong.
422bool AMDGPURegisterBankInfo::isScalarLoadLegal(const MachineInstr &MI) const {
423 if (!MI.hasOneMemOperand())
424 return false;
425
426 const MachineMemOperand *MMO = *MI.memoperands_begin();
427 const unsigned AS = MMO->getAddrSpace();
428 const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS ||
429 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
430 const unsigned MemSize = 8 * MMO->getSize().getValue();
431
432 // Require 4-byte alignment.
433 return (MMO->getAlign() >= Align(4) ||
434 (Subtarget.hasScalarSubwordLoads() &&
435 ((MemSize == 16 && MMO->getAlign() >= Align(2)) ||
436 (MemSize == 8 && MMO->getAlign() >= Align(1))))) &&
437 // Can't do a scalar atomic load.
438 !MMO->isAtomic() &&
439 // Don't use scalar loads for volatile accesses to non-constant address
440 // spaces.
441 (IsConst || !MMO->isVolatile()) &&
442 // Memory must be known constant, or not written before this load.
443 (IsConst || MMO->isInvariant() || (MMO->getFlags() & MONoClobber)) &&
444 AMDGPU::isUniformMMO(MMO);
445}
446
447RegisterBankInfo::InstructionMappings
448AMDGPURegisterBankInfo::getInstrAlternativeMappings(
449 const MachineInstr &MI) const {
450
451 const MachineFunction &MF = *MI.getMF();
452 const MachineRegisterInfo &MRI = MF.getRegInfo();
453
454
455 InstructionMappings AltMappings;
456 switch (MI.getOpcode()) {
457 case TargetOpcode::G_CONSTANT:
458 case TargetOpcode::G_IMPLICIT_DEF: {
459 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
460 if (Size == 1) {
461 static const OpRegBankEntry<1> Table[3] = {
462 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 1 },
463 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 },
464 { .RegBanks: { AMDGPU::VCCRegBankID }, .Cost: 1 }
465 };
466
467 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx: {._M_elems: { 0 }}, Table);
468 }
469
470 [[fallthrough]];
471 }
472 case TargetOpcode::G_FCONSTANT:
473 case TargetOpcode::G_FRAME_INDEX:
474 case TargetOpcode::G_GLOBAL_VALUE: {
475 static const OpRegBankEntry<1> Table[2] = {
476 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 1 },
477 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 }
478 };
479
480 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx: {._M_elems: { 0 }}, Table);
481 }
482 case TargetOpcode::G_AND:
483 case TargetOpcode::G_OR:
484 case TargetOpcode::G_XOR: {
485 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
486
487 if (Size == 1) {
488 // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0.
489 const InstructionMapping &SCCMapping = getInstructionMapping(
490 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
491 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32),
492 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32),
493 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32)}),
494 NumOperands: 3); // Num Operands
495 AltMappings.push_back(Elt: &SCCMapping);
496
497 const InstructionMapping &VCCMapping0 = getInstructionMapping(
498 ID: 2, Cost: 1, OperandsMapping: getOperandsMapping(
499 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size),
500 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size),
501 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size)}),
502 NumOperands: 3); // Num Operands
503 AltMappings.push_back(Elt: &VCCMapping0);
504 return AltMappings;
505 }
506
507 if (Size != 64)
508 break;
509
510 const InstructionMapping &SSMapping = getInstructionMapping(
511 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
512 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
513 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
514 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size)}),
515 NumOperands: 3); // Num Operands
516 AltMappings.push_back(Elt: &SSMapping);
517
518 const InstructionMapping &VVMapping = getInstructionMapping(
519 ID: 2, Cost: 2, OperandsMapping: getOperandsMapping(
520 OpdsMapping: {AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
521 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
522 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size)}),
523 NumOperands: 3); // Num Operands
524 AltMappings.push_back(Elt: &VVMapping);
525 break;
526 }
527 case TargetOpcode::G_LOAD:
528 case TargetOpcode::G_ZEXTLOAD:
529 case TargetOpcode::G_SEXTLOAD: {
530 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
531 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
532 unsigned PtrSize = PtrTy.getSizeInBits();
533 unsigned AS = PtrTy.getAddressSpace();
534
535 if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
536 AS != AMDGPUAS::PRIVATE_ADDRESS) &&
537 isScalarLoadLegal(MI)) {
538 const InstructionMapping &SSMapping = getInstructionMapping(
539 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
540 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
541 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: PtrSize)}),
542 NumOperands: 2); // Num Operands
543 AltMappings.push_back(Elt: &SSMapping);
544 }
545
546 const InstructionMapping &VVMapping = getInstructionMapping(
547 ID: 2, Cost: 1,
548 OperandsMapping: getOperandsMapping(
549 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
550 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize)}),
551 NumOperands: 2); // Num Operands
552 AltMappings.push_back(Elt: &VVMapping);
553
554 // It may be possible to have a vgpr = load sgpr mapping here, because
555 // the mubuf instructions support this kind of load, but probably for only
556 // gfx7 and older. However, the addressing mode matching in the instruction
557 // selector should be able to do a better job of detecting and selecting
558 // these kinds of loads from the vgpr = load vgpr mapping.
559
560 return AltMappings;
561
562 }
563 case TargetOpcode::G_SELECT: {
564 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
565 const InstructionMapping &SSMapping = getInstructionMapping(ID: 1, Cost: 1,
566 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
567 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1),
568 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
569 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size)}),
570 NumOperands: 4); // Num Operands
571 AltMappings.push_back(Elt: &SSMapping);
572
573 const InstructionMapping &VVMapping = getInstructionMapping(ID: 2, Cost: 1,
574 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
575 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1),
576 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
577 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size)}),
578 NumOperands: 4); // Num Operands
579 AltMappings.push_back(Elt: &VVMapping);
580
581 return AltMappings;
582 }
583 case TargetOpcode::G_UADDE:
584 case TargetOpcode::G_USUBE:
585 case TargetOpcode::G_SADDE:
586 case TargetOpcode::G_SSUBE: {
587 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
588 const InstructionMapping &SSMapping = getInstructionMapping(ID: 1, Cost: 1,
589 OperandsMapping: getOperandsMapping(
590 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
591 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1),
592 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
593 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
594 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1)}),
595 NumOperands: 5); // Num Operands
596 AltMappings.push_back(Elt: &SSMapping);
597
598 const InstructionMapping &VVMapping = getInstructionMapping(ID: 2, Cost: 1,
599 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
600 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1),
601 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
602 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
603 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1)}),
604 NumOperands: 5); // Num Operands
605 AltMappings.push_back(Elt: &VVMapping);
606 return AltMappings;
607 }
608 case AMDGPU::G_BRCOND: {
609 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
610
611 // TODO: Change type to 32 for scalar
612 const InstructionMapping &SMapping = getInstructionMapping(
613 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
614 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1), nullptr}),
615 NumOperands: 2); // Num Operands
616 AltMappings.push_back(Elt: &SMapping);
617
618 const InstructionMapping &VMapping = getInstructionMapping(
619 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
620 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1), nullptr }),
621 NumOperands: 2); // Num Operands
622 AltMappings.push_back(Elt: &VMapping);
623 return AltMappings;
624 }
625 case AMDGPU::G_INTRINSIC:
626 case AMDGPU::G_INTRINSIC_CONVERGENT:
627 return getInstrAlternativeMappingsIntrinsic(MI, MRI);
628 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
629 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
630 return getInstrAlternativeMappingsIntrinsicWSideEffects(MI, MRI);
631 default:
632 break;
633 }
634 return RegisterBankInfo::getInstrAlternativeMappings(MI);
635}
636
637void AMDGPURegisterBankInfo::split64BitValueForMapping(
638 MachineIRBuilder &B,
639 SmallVector<Register, 2> &Regs,
640 LLT HalfTy,
641 Register Reg) const {
642 assert(HalfTy.getSizeInBits() == 32);
643 MachineRegisterInfo *MRI = B.getMRI();
644 Register LoLHS = MRI->createGenericVirtualRegister(Ty: HalfTy);
645 Register HiLHS = MRI->createGenericVirtualRegister(Ty: HalfTy);
646 const RegisterBank *Bank = getRegBank(Reg, MRI: *MRI, TRI: *TRI);
647 MRI->setRegBank(Reg: LoLHS, RegBank: *Bank);
648 MRI->setRegBank(Reg: HiLHS, RegBank: *Bank);
649
650 Regs.push_back(Elt: LoLHS);
651 Regs.push_back(Elt: HiLHS);
652
653 B.buildInstr(Opcode: AMDGPU::G_UNMERGE_VALUES)
654 .addDef(RegNo: LoLHS)
655 .addDef(RegNo: HiLHS)
656 .addUse(RegNo: Reg);
657}
658
659/// Replace the current type each register in \p Regs has with \p NewTy
660static void setRegsToType(MachineRegisterInfo &MRI, ArrayRef<Register> Regs,
661 LLT NewTy) {
662 for (Register Reg : Regs) {
663 assert(MRI.getType(Reg).getSizeInBits() == NewTy.getSizeInBits());
664 MRI.setType(VReg: Reg, Ty: NewTy);
665 }
666}
667
668static LLT getHalfSizedType(LLT Ty) {
669 if (Ty.isVector()) {
670 assert(Ty.getElementCount().isKnownMultipleOf(2));
671 return LLT::scalarOrVector(EC: Ty.getElementCount().divideCoefficientBy(RHS: 2),
672 ScalarTy: Ty.getElementType());
673 }
674
675 assert(Ty.getScalarSizeInBits() % 2 == 0);
676 return LLT::scalar(SizeInBits: Ty.getScalarSizeInBits() / 2);
677}
678
679// Build one or more V_READFIRSTLANE_B32 instructions to move the given vector
680// source value into a scalar register.
681Register AMDGPURegisterBankInfo::buildReadFirstLane(MachineIRBuilder &B,
682 MachineRegisterInfo &MRI,
683 Register Src) const {
684 LLT Ty = MRI.getType(Reg: Src);
685 const RegisterBank *Bank = getRegBank(Reg: Src, MRI, TRI: *TRI);
686
687 if (Bank == &AMDGPU::SGPRRegBank)
688 return Src;
689
690 unsigned Bits = Ty.getSizeInBits();
691 assert(Bits % 32 == 0);
692
693 if (Bank != &AMDGPU::VGPRRegBank) {
694 // We need to copy from AGPR to VGPR
695 Src = B.buildCopy(Res: Ty, Op: Src).getReg(Idx: 0);
696 MRI.setRegBank(Reg: Src, RegBank: AMDGPU::VGPRRegBank);
697 }
698
699 LLT S32 = LLT::scalar(SizeInBits: 32);
700 unsigned NumParts = Bits / 32;
701 SmallVector<Register, 8> SrcParts;
702 SmallVector<Register, 8> DstParts;
703
704 if (Bits == 32) {
705 SrcParts.push_back(Elt: Src);
706 } else {
707 auto Unmerge = B.buildUnmerge(Res: S32, Op: Src);
708 for (unsigned i = 0; i < NumParts; ++i)
709 SrcParts.push_back(Elt: Unmerge.getReg(Idx: i));
710 }
711
712 for (unsigned i = 0; i < NumParts; ++i) {
713 Register SrcPart = SrcParts[i];
714 Register DstPart = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
715 MRI.setType(VReg: DstPart, Ty: NumParts == 1 ? Ty : S32);
716
717 const TargetRegisterClass *Constrained =
718 constrainGenericRegister(Reg: SrcPart, RC: AMDGPU::VGPR_32RegClass, MRI);
719 (void)Constrained;
720 assert(Constrained && "Failed to constrain readfirstlane src reg");
721
722 B.buildInstr(Opc: AMDGPU::V_READFIRSTLANE_B32, DstOps: {DstPart}, SrcOps: {SrcPart});
723
724 DstParts.push_back(Elt: DstPart);
725 }
726
727 if (Bits == 32)
728 return DstParts[0];
729
730 Register Dst = B.buildMergeLikeInstr(Res: Ty, Ops: DstParts).getReg(Idx: 0);
731 MRI.setRegBank(Reg: Dst, RegBank: AMDGPU::SGPRRegBank);
732 return Dst;
733}
734
735/// Legalize instruction \p MI where operands in \p OpIndices must be SGPRs. If
736/// any of the required SGPR operands are VGPRs, perform a waterfall loop to
737/// execute the instruction for each unique combination of values in all lanes
738/// in the wave. The block will be split such that rest of the instructions are
739/// moved to a new block.
740///
741/// Essentially performs this loop:
742//
743/// Save Execution Mask
744/// For (Lane : Wavefront) {
745/// Enable Lane, Disable all other lanes
746/// SGPR = read SGPR value for current lane from VGPR
747/// VGPRResult[Lane] = use_op SGPR
748/// }
749/// Restore Execution Mask
750///
751/// There is additional complexity to try for compare values to identify the
752/// unique values used.
753bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
754 MachineIRBuilder &B, iterator_range<MachineBasicBlock::iterator> Range,
755 SmallSet<Register, 4> &SGPROperandRegs) const {
756 // Track use registers which have already been expanded with a readfirstlane
757 // sequence. This may have multiple uses if moving a sequence.
758 DenseMap<Register, Register> WaterfalledRegMap;
759
760 MachineBasicBlock &MBB = B.getMBB();
761 MachineFunction *MF = &B.getMF();
762
763 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
764 const AMDGPU::LaneMaskConstants &LMC =
765 AMDGPU::LaneMaskConstants::get(ST: Subtarget);
766
767#ifndef NDEBUG
768 const int OrigRangeSize = std::distance(Range.begin(), Range.end());
769#endif
770
771 MachineRegisterInfo &MRI = *B.getMRI();
772 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
773 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
774
775 // Don't bother using generic instructions/registers for the exec mask.
776 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF)
777 .addDef(RegNo: InitSaveExecReg);
778
779 Register PhiExec = MRI.createVirtualRegister(RegClass: WaveRC);
780 Register NewExec = MRI.createVirtualRegister(RegClass: WaveRC);
781
782 // To insert the loop we need to split the block. Move everything before this
783 // point to a new block, and insert a new empty block before this instruction.
784 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
785 MachineBasicBlock *BodyBB = MF->CreateMachineBasicBlock();
786 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
787 MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock();
788 MachineFunction::iterator MBBI(MBB);
789 ++MBBI;
790 MF->insert(MBBI, MBB: LoopBB);
791 MF->insert(MBBI, MBB: BodyBB);
792 MF->insert(MBBI, MBB: RestoreExecBB);
793 MF->insert(MBBI, MBB: RemainderBB);
794
795 LoopBB->addSuccessor(Succ: BodyBB);
796 BodyBB->addSuccessor(Succ: RestoreExecBB);
797 BodyBB->addSuccessor(Succ: LoopBB);
798
799 // Move the rest of the block into a new block.
800 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
801 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Range.end(), To: MBB.end());
802
803 MBB.addSuccessor(Succ: LoopBB);
804 RestoreExecBB->addSuccessor(Succ: RemainderBB);
805
806 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
807
808 B.buildInstr(Opcode: TargetOpcode::PHI)
809 .addDef(RegNo: PhiExec)
810 .addReg(RegNo: InitSaveExecReg)
811 .addMBB(MBB: &MBB)
812 .addReg(RegNo: NewExec)
813 .addMBB(MBB: BodyBB);
814
815 const DebugLoc &DL = B.getDL();
816
817 MachineInstr &FirstInst = *Range.begin();
818
819 // Move the instruction into the loop body. Note we moved everything after
820 // Range.end() already into a new block, so Range.end() is no longer valid.
821 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: Range.begin(), To: MBB.end());
822
823 // Figure out the iterator range after splicing the instructions.
824 MachineBasicBlock::iterator NewBegin = FirstInst.getIterator();
825 auto NewEnd = BodyBB->end();
826
827 B.setMBB(*LoopBB);
828
829 LLT S1 = LLT::scalar(SizeInBits: 1);
830 Register CondReg;
831
832 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
833
834 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
835 for (MachineOperand &Op : MI.all_uses()) {
836 Register OldReg = Op.getReg();
837 if (!SGPROperandRegs.count(V: OldReg))
838 continue;
839
840 // See if we already processed this register in another instruction in the
841 // sequence.
842 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
843 if (OldVal != WaterfalledRegMap.end()) {
844 Op.setReg(OldVal->second);
845 continue;
846 }
847
848 Register OpReg = Op.getReg();
849 LLT OpTy = MRI.getType(Reg: OpReg);
850
851 const RegisterBank *OpBank = getRegBank(Reg: OpReg, MRI, TRI: *TRI);
852 if (OpBank != &AMDGPU::VGPRRegBank) {
853 // Insert copy from AGPR to VGPR before the loop.
854 B.setMBB(MBB);
855 OpReg = B.buildCopy(Res: OpTy, Op: OpReg).getReg(Idx: 0);
856 MRI.setRegBank(Reg: OpReg, RegBank: AMDGPU::VGPRRegBank);
857 B.setMBB(*LoopBB);
858 }
859
860 Register CurrentLaneReg = buildReadFirstLane(B, MRI, Src: OpReg);
861
862 // Build the comparison(s).
863 unsigned OpSize = OpTy.getSizeInBits();
864 bool Is64 = OpSize % 64 == 0;
865 unsigned PartSize = Is64 ? 64 : 32;
866 LLT PartTy = LLT::scalar(SizeInBits: PartSize);
867 unsigned NumParts = OpSize / PartSize;
868 SmallVector<Register, 8> OpParts;
869 SmallVector<Register, 8> CurrentLaneParts;
870
871 if (NumParts == 1) {
872 OpParts.push_back(Elt: OpReg);
873 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
874 } else {
875 auto UnmergeOp = B.buildUnmerge(Res: PartTy, Op: OpReg);
876 auto UnmergeCurrentLane = B.buildUnmerge(Res: PartTy, Op: CurrentLaneReg);
877 for (unsigned i = 0; i < NumParts; ++i) {
878 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
879 CurrentLaneParts.push_back(Elt: UnmergeCurrentLane.getReg(Idx: i));
880 MRI.setRegBank(Reg: OpParts[i], RegBank: AMDGPU::VGPRRegBank);
881 MRI.setRegBank(Reg: CurrentLaneParts[i], RegBank: AMDGPU::SGPRRegBank);
882 }
883 }
884
885 for (unsigned i = 0; i < NumParts; ++i) {
886 auto CmpReg = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: CurrentLaneParts[i],
887 Op1: OpParts[i]).getReg(Idx: 0);
888 MRI.setRegBank(Reg: CmpReg, RegBank: AMDGPU::VCCRegBank);
889
890 if (!CondReg) {
891 CondReg = CmpReg;
892 } else {
893 CondReg = B.buildAnd(Dst: S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
894 MRI.setRegBank(Reg: CondReg, RegBank: AMDGPU::VCCRegBank);
895 }
896 }
897
898 Op.setReg(CurrentLaneReg);
899
900 // Make sure we don't re-process this register again.
901 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
902 }
903 }
904
905 // The ballot becomes a no-op during instruction selection.
906 CondReg = B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot,
907 Res: {LLT::scalar(SizeInBits: Subtarget.isWave32() ? 32 : 64)})
908 .addReg(RegNo: CondReg)
909 .getReg(Idx: 0);
910 MRI.setRegClass(Reg: CondReg, RC: WaveRC);
911
912 // Update EXEC, save the original EXEC value to VCC.
913 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
914 .addDef(RegNo: NewExec)
915 .addReg(RegNo: CondReg)
916 .setOperandDead(3);
917
918 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
919
920 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
921
922 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
923 B.buildInstr(Opcode: LMC.XorTermOpc)
924 .addDef(RegNo: LMC.ExecReg)
925 .addReg(RegNo: LMC.ExecReg)
926 .addReg(RegNo: NewExec)
927 .setOperandDead(3);
928
929 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
930 // s_cbranch_scc0?
931
932 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
933 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
934
935 // Save the EXEC mask before the loop.
936 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExecReg)
937 .addReg(RegNo: LMC.ExecReg);
938
939 // Restore the EXEC mask after the loop.
940 B.setMBB(*RestoreExecBB);
941 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
942
943 // Set the insert point after the original instruction, so any new
944 // instructions will be in the remainder.
945 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
946
947 return true;
948}
949
950// Return any unique registers used by \p MI at \p OpIndices that need to be
951// handled in a waterfall loop. Returns these registers in \p
952// SGPROperandRegs. Returns true if there are any operands to handle and a
953// waterfall loop is necessary.
954bool AMDGPURegisterBankInfo::collectWaterfallOperands(
955 SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI,
956 MachineRegisterInfo &MRI, ArrayRef<unsigned> OpIndices) const {
957 for (unsigned Op : OpIndices) {
958 assert(MI.getOperand(Op).isUse());
959 Register Reg = MI.getOperand(i: Op).getReg();
960 const RegisterBank *OpBank = getRegBank(Reg, MRI, TRI: *TRI);
961 if (OpBank->getID() != AMDGPU::SGPRRegBankID)
962 SGPROperandRegs.insert(V: Reg);
963 }
964
965 // No operands need to be replaced, so no need to loop.
966 return !SGPROperandRegs.empty();
967}
968
969bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
970 MachineIRBuilder &B, MachineInstr &MI, ArrayRef<unsigned> OpIndices) const {
971 // Use a set to avoid extra readfirstlanes in the case where multiple operands
972 // are the same register.
973 SmallSet<Register, 4> SGPROperandRegs;
974
975 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI&: *B.getMRI(), OpIndices))
976 return false;
977
978 MachineBasicBlock::iterator I = MI.getIterator();
979 return executeInWaterfallLoop(B, Range: make_range(x: I, y: std::next(x: I)),
980 SGPROperandRegs);
981}
982
983// Legalize an operand that must be an SGPR by inserting a readfirstlane.
984void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane(
985 MachineIRBuilder &B, MachineInstr &MI, unsigned OpIdx) const {
986 Register Reg = MI.getOperand(i: OpIdx).getReg();
987 MachineRegisterInfo &MRI = *B.getMRI();
988 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
989 if (Bank == &AMDGPU::SGPRRegBank)
990 return;
991
992 Reg = buildReadFirstLane(B, MRI, Src: Reg);
993 MI.getOperand(i: OpIdx).setReg(Reg);
994}
995
996/// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
997/// rest will be in the remainder.
998static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) {
999 unsigned TotalSize = Ty.getSizeInBits();
1000 if (!Ty.isVector())
1001 return {LLT::scalar(SizeInBits: FirstSize), LLT::scalar(SizeInBits: TotalSize - FirstSize)};
1002
1003 LLT EltTy = Ty.getElementType();
1004 unsigned EltSize = EltTy.getSizeInBits();
1005 assert(FirstSize % EltSize == 0);
1006
1007 unsigned FirstPartNumElts = FirstSize / EltSize;
1008 unsigned RemainderElts = (TotalSize - FirstSize) / EltSize;
1009
1010 return {LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: FirstPartNumElts), ScalarTy: EltTy),
1011 LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: RemainderElts), ScalarTy: EltTy)};
1012}
1013
1014static LLT widen96To128(LLT Ty) {
1015 if (!Ty.isVector())
1016 return LLT::scalar(SizeInBits: 128);
1017
1018 LLT EltTy = Ty.getElementType();
1019 assert(128 % EltTy.getSizeInBits() == 0);
1020 return LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1021}
1022
1023bool AMDGPURegisterBankInfo::applyMappingLoad(
1024 MachineIRBuilder &B,
1025 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1026 MachineInstr &MI) const {
1027 MachineRegisterInfo &MRI = *B.getMRI();
1028 Register DstReg = MI.getOperand(i: 0).getReg();
1029 const LLT LoadTy = MRI.getType(Reg: DstReg);
1030 unsigned LoadSize = LoadTy.getSizeInBits();
1031 MachineMemOperand *MMO = *MI.memoperands_begin();
1032 const unsigned MaxNonSmrdLoadSize = 128;
1033
1034 const RegisterBank *DstBank =
1035 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1036 if (DstBank == &AMDGPU::SGPRRegBank) {
1037 // There are some special cases that we need to look at for 32 bit and 96
1038 // bit SGPR loads otherwise we have nothing to do.
1039 if (LoadSize != 32 && (LoadSize != 96 || Subtarget.hasScalarDwordx3Loads()))
1040 return false;
1041
1042 const unsigned MemSize = 8 * MMO->getSize().getValue();
1043 // Scalar loads of size 8 or 16 bit with proper alignment may be widened to
1044 // 32 bit. Check to see if we need to widen the memory access, 8 or 16 bit
1045 // scalar loads should have a load size of 32 but memory access size of less
1046 // than 32.
1047 if (LoadSize == 32 &&
1048 (MemSize == 32 || LoadTy.isVector() || !isScalarLoadLegal(MI)))
1049 return false;
1050
1051 if (LoadSize == 32 &&
1052 ((MemSize == 8 && MMO->getAlign() >= Align(1)) ||
1053 (MemSize == 16 && MMO->getAlign() >= Align(2))) &&
1054 isScalarLoadLegal(MI) &&
1055 Subtarget.getGeneration() >= AMDGPUSubtarget::GFX12)
1056 return false;
1057
1058 Register PtrReg = MI.getOperand(i: 1).getReg();
1059
1060 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
1061
1062 if (LoadSize == 32) {
1063 // This is an extending load from a sub-dword size. Widen the memory
1064 // access size to 4 bytes and clear the extra high bits appropriately
1065 const LLT S32 = LLT::scalar(SizeInBits: 32);
1066 if (MI.getOpcode() == AMDGPU::G_SEXTLOAD) {
1067 // Must extend the sign bit into higher bits for a G_SEXTLOAD
1068 auto WideLoad = B.buildLoadFromOffset(Dst: S32, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1069 B.buildSExtInReg(Res: MI.getOperand(i: 0), Op: WideLoad, ImmOp: MemSize);
1070 } else if (MI.getOpcode() == AMDGPU::G_ZEXTLOAD) {
1071 // Must extend zero into higher bits with an AND for a G_ZEXTLOAD
1072 auto WideLoad = B.buildLoadFromOffset(Dst: S32, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1073 B.buildZExtInReg(Res: MI.getOperand(i: 0), Op: WideLoad, ImmOp: MemSize);
1074 } else
1075 // We do not need to touch the higher bits for regular loads.
1076 B.buildLoadFromOffset(Dst: MI.getOperand(i: 0), BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1077 } else {
1078 // 96-bit loads are only available for vector loads. We need to split this
1079 // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
1080 if (MMO->getAlign() < Align(16)) {
1081 LegalizerHelper Helper(B.getMF(), ApplyBank, B);
1082 LLT Part64, Part32;
1083 std::tie(args&: Part64, args&: Part32) = splitUnequalType(Ty: LoadTy, FirstSize: 64);
1084 if (Helper.reduceLoadStoreWidth(MI&: cast<GAnyLoad>(Val&: MI), TypeIdx: 0, NarrowTy: Part64) !=
1085 LegalizerHelper::Legalized)
1086 return false;
1087 return true;
1088 }
1089 LLT WiderTy = widen96To128(Ty: LoadTy);
1090 auto WideLoad = B.buildLoadFromOffset(Dst: WiderTy, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1091 if (WiderTy.isScalar()) {
1092 B.buildTrunc(Res: MI.getOperand(i: 0), Op: WideLoad);
1093 } else {
1094 B.buildDeleteTrailingVectorElements(Res: MI.getOperand(i: 0).getReg(),
1095 Op0: WideLoad);
1096 }
1097 }
1098
1099 MI.eraseFromParent();
1100 return true;
1101 }
1102
1103 // 128-bit loads are supported for all instruction types.
1104 if (LoadSize <= MaxNonSmrdLoadSize)
1105 return false;
1106
1107 SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
1108
1109 if (SrcRegs.empty())
1110 SrcRegs.push_back(Elt: MI.getOperand(i: 1).getReg());
1111
1112 // RegBankSelect only emits scalar types, so we need to reset the pointer
1113 // operand to a pointer type.
1114 Register BasePtrReg = SrcRegs[0];
1115 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1116 MRI.setType(VReg: BasePtrReg, Ty: PtrTy);
1117
1118 // The following are the loads not splitted enough during legalization
1119 // because it was not clear they are smem-load or vmem-load
1120 if (AMDGPU::isExtendedGlobalAddrSpace(AS: MMO->getAddrSpace()) ||
1121 MMO->getAddrSpace() == AMDGPUAS::BUFFER_RESOURCE) {
1122 assert(LoadSize % MaxNonSmrdLoadSize == 0);
1123 unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize;
1124 const LLT LoadSplitTy = LoadTy.divide(Factor: NumSplitParts);
1125 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank);
1126 LegalizerHelper Helper(B.getMF(), O, B);
1127 if (LoadTy.isVector()) {
1128 if (Helper.fewerElementsVector(MI, TypeIdx: 0, NarrowTy: LoadSplitTy) !=
1129 LegalizerHelper::Legalized)
1130 return false;
1131 } else {
1132 if (Helper.narrowScalar(MI, TypeIdx: 0, NarrowTy: LoadSplitTy) != LegalizerHelper::Legalized)
1133 return false;
1134 }
1135 }
1136
1137 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
1138 return true;
1139}
1140
1141bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
1142 MachineIRBuilder &B,
1143 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1144 MachineInstr &MI) const {
1145 MachineRegisterInfo &MRI = *B.getMRI();
1146 const MachineFunction &MF = B.getMF();
1147 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1148 const auto &TFI = *ST.getFrameLowering();
1149
1150 // Guard in case the stack growth direction ever changes with scratch
1151 // instructions.
1152 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1153 "Stack grows upwards for AMDGPU");
1154
1155 Register Dst = MI.getOperand(i: 0).getReg();
1156 Register AllocSize = MI.getOperand(i: 1).getReg();
1157 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1158
1159 // When using flat-scratch, the stack offset is unscaled.
1160 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1161 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1162
1163 const RegisterBank *SizeBank = getRegBank(Reg: AllocSize, MRI, TRI: *TRI);
1164
1165 if (SizeBank != &AMDGPU::SGPRRegBank) {
1166 auto WaveReduction =
1167 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {LLT::scalar(SizeInBits: 32)})
1168 .addUse(RegNo: AllocSize)
1169 .addImm(Val: 0);
1170 AllocSize = WaveReduction.getReg(Idx: 0);
1171 }
1172
1173 LLT PtrTy = MRI.getType(Reg: Dst);
1174 LLT IntPtrTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1175
1176 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1177 Register SPReg = Info->getStackPtrOffsetReg();
1178 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
1179
1180 Register ScaledSize = AllocSize;
1181 if (!HasFlatScratch) {
1182 auto WaveSize = B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: WavefrontSizeLog2);
1183 ScaledSize = B.buildShl(Dst: IntPtrTy, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1184 }
1185
1186 auto OldSP = B.buildCopy(Res: PtrTy, Op: SPReg);
1187 if (Alignment > TFI.getStackAlign()) {
1188 const uint64_t ScaledAlignment =
1189 HasFlatScratch ? Alignment.value()
1190 : (Alignment.value() << WavefrontSizeLog2);
1191 const uint64_t StackAlignMask = ScaledAlignment - 1;
1192 auto Tmp1 = B.buildPtrAdd(Res: PtrTy, Op0: OldSP,
1193 Op1: B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: StackAlignMask));
1194 B.buildMaskLowPtrBits(Res: Dst, Op0: Tmp1,
1195 NumBits: (HasFlatScratch
1196 ? Log2(A: Alignment)
1197 : Log2(A: Alignment) + WavefrontSizeLog2));
1198 } else {
1199 B.buildCopy(Res: Dst, Op: OldSP);
1200 }
1201 auto PtrAdd = B.buildPtrAdd(Res: PtrTy, Op0: Dst, Op1: ScaledSize);
1202 B.buildCopy(Res: SPReg, Op: PtrAdd);
1203 MI.eraseFromParent();
1204 return true;
1205}
1206
1207bool AMDGPURegisterBankInfo::applyMappingImage(
1208 MachineIRBuilder &B, MachineInstr &MI,
1209 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1210 int RsrcIdx) const {
1211 const int NumDefs = MI.getNumExplicitDefs();
1212
1213 // The reported argument index is relative to the IR intrinsic call arguments,
1214 // so we need to shift by the number of defs and the intrinsic ID.
1215 RsrcIdx += NumDefs + 1;
1216
1217 // Insert copies to VGPR arguments.
1218 applyDefaultMapping(OpdMapper);
1219
1220 // Fixup any SGPR arguments.
1221 SmallVector<unsigned, 4> SGPRIndexes;
1222 for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) {
1223 if (!MI.getOperand(i: I).isReg())
1224 continue;
1225
1226 // If this intrinsic has a sampler, it immediately follows rsrc.
1227 if (I == RsrcIdx || I == RsrcIdx + 1)
1228 SGPRIndexes.push_back(Elt: I);
1229 }
1230
1231 executeInWaterfallLoop(B, MI, OpIndices: SGPRIndexes);
1232 return true;
1233}
1234
1235// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
1236// the three offsets (voffset, soffset and instoffset)
1237unsigned AMDGPURegisterBankInfo::setBufferOffsets(
1238 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
1239 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) const {
1240 const LLT S32 = LLT::scalar(SizeInBits: 32);
1241 MachineRegisterInfo *MRI = B.getMRI();
1242
1243 if (std::optional<int64_t> Imm =
1244 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI: *MRI)) {
1245 uint32_t SOffset, ImmOffset;
1246 if (TII->splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
1247 VOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1248 SOffsetReg = B.buildConstant(Res: S32, Val: SOffset).getReg(Idx: 0);
1249 InstOffsetVal = ImmOffset;
1250
1251 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1252 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1253 return SOffset + ImmOffset;
1254 }
1255 }
1256
1257 const bool CheckNUW = Subtarget.hasGFX1250Insts();
1258 Register Base;
1259 unsigned Offset;
1260
1261 std::tie(args&: Base, args&: Offset) =
1262 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: CombinedOffset,
1263 /*KnownBits=*/ValueTracking: nullptr,
1264 /*CheckNUW=*/CheckNUW);
1265
1266 uint32_t SOffset, ImmOffset;
1267 if (static_cast<int32_t>(Offset) > 0 &&
1268 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
1269 if (getRegBank(Reg: Base, MRI: *MRI, TRI: *TRI) == &AMDGPU::VGPRRegBank) {
1270 VOffsetReg = Base;
1271 SOffsetReg = B.buildConstant(Res: S32, Val: SOffset).getReg(Idx: 0);
1272 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1273 InstOffsetVal = ImmOffset;
1274 return 0; // XXX - Why is this 0?
1275 }
1276
1277 // If we have SGPR base, we can use it for soffset.
1278 if (SOffset == 0) {
1279 VOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1280 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1281 SOffsetReg = Base;
1282 InstOffsetVal = ImmOffset;
1283 return 0; // XXX - Why is this 0?
1284 }
1285 }
1286
1287 // Handle the variable sgpr + vgpr case.
1288 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI: *MRI);
1289 if (Add && static_cast<int32_t>(Offset) >= 0 &&
1290 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
1291 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI: *MRI);
1292 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI: *MRI);
1293
1294 const RegisterBank *Src0Bank = getRegBank(Reg: Src0, MRI: *MRI, TRI: *TRI);
1295 const RegisterBank *Src1Bank = getRegBank(Reg: Src1, MRI: *MRI, TRI: *TRI);
1296
1297 if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) {
1298 VOffsetReg = Src0;
1299 SOffsetReg = Src1;
1300 return 0;
1301 }
1302
1303 if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) {
1304 VOffsetReg = Src1;
1305 SOffsetReg = Src0;
1306 return 0;
1307 }
1308 }
1309
1310 // Ensure we have a VGPR for the combined offset. This could be an issue if we
1311 // have an SGPR offset and a VGPR resource.
1312 if (getRegBank(Reg: CombinedOffset, MRI: *MRI, TRI: *TRI) == &AMDGPU::VGPRRegBank) {
1313 VOffsetReg = CombinedOffset;
1314 } else {
1315 VOffsetReg = B.buildCopy(Res: S32, Op: CombinedOffset).getReg(Idx: 0);
1316 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1317 }
1318
1319 SOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1320 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1321 return 0;
1322}
1323
1324static unsigned getSBufferLoadCorrespondingBufferLoadOpcode(unsigned Opc) {
1325 switch (Opc) {
1326 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
1327 return AMDGPU::G_AMDGPU_BUFFER_LOAD;
1328 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
1329 return AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
1330 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
1331 return AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE;
1332 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
1333 return AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
1334 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
1335 return AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT;
1336 default:
1337 break;
1338 }
1339 llvm_unreachable("Unexpected s_buffer_load opcode");
1340}
1341
1342bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
1343 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
1344 MachineInstr &MI = OpdMapper.getMI();
1345 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1346
1347 const LLT S32 = LLT::scalar(SizeInBits: 32);
1348 Register Dst = MI.getOperand(i: 0).getReg();
1349 LLT Ty = MRI.getType(Reg: Dst);
1350
1351 const RegisterBank *RSrcBank =
1352 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1353 const RegisterBank *OffsetBank =
1354 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
1355 if (RSrcBank == &AMDGPU::SGPRRegBank &&
1356 OffsetBank == &AMDGPU::SGPRRegBank)
1357 return true; // Legal mapping
1358
1359 // FIXME: 96-bit case was widened during legalize. We need to narrow it back
1360 // here but don't have an MMO.
1361
1362 unsigned LoadSize = Ty.getSizeInBits();
1363 int NumLoads = 1;
1364 if (LoadSize == 256 || LoadSize == 512) {
1365 NumLoads = LoadSize / 128;
1366 Ty = Ty.divide(Factor: NumLoads);
1367 }
1368
1369 // Use the alignment to ensure that the required offsets will fit into the
1370 // immediate offsets.
1371 const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
1372
1373 MachineFunction &MF = B.getMF();
1374
1375 Register SOffset;
1376 Register VOffset;
1377 int64_t ImmOffset = 0;
1378
1379 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
1380 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
1381
1382 // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
1383 // can, but we need to track an MMO for that.
1384 const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
1385 const Align MemAlign(4); // FIXME: ABI type alignment?
1386 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
1387 PtrInfo: MachinePointerInfo(),
1388 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1389 MachineMemOperand::MOInvariant,
1390 Size: MemSize, BaseAlignment: MemAlign);
1391 if (MMOOffset != 0)
1392 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
1393
1394 // If only the offset is divergent, emit a MUBUF buffer load instead. We can
1395 // assume that the buffer is unswizzled.
1396
1397 Register RSrc = MI.getOperand(i: 1).getReg();
1398 Register VIndex = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1399 B.getMRI()->setRegBank(Reg: VIndex, RegBank: AMDGPU::VGPRRegBank);
1400 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
1401
1402 SmallVector<Register, 4> LoadParts(NumLoads);
1403
1404 MachineBasicBlock::iterator MII = MI.getIterator();
1405 MachineInstrSpan Span(MII, &B.getMBB());
1406
1407 for (int i = 0; i < NumLoads; ++i) {
1408 if (NumLoads == 1) {
1409 LoadParts[i] = Dst;
1410 } else {
1411 LoadParts[i] = MRI.createGenericVirtualRegister(Ty);
1412 MRI.setRegBank(Reg: LoadParts[i], RegBank: AMDGPU::VGPRRegBank);
1413 }
1414
1415 if (i != 0)
1416 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16, Size: MemSize);
1417
1418 B.buildInstr(Opcode: getSBufferLoadCorrespondingBufferLoadOpcode(Opc: MI.getOpcode()))
1419 .addDef(RegNo: LoadParts[i]) // vdata
1420 .addUse(RegNo: RSrc) // rsrc
1421 .addUse(RegNo: VIndex) // vindex
1422 .addUse(RegNo: VOffset) // voffset
1423 .addUse(RegNo: SOffset) // soffset
1424 .addImm(Val: ImmOffset + 16 * i) // offset(imm)
1425 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
1426 .addImm(Val: 0) // idxen(imm)
1427 .addMemOperand(MMO: BaseMMO);
1428 }
1429
1430 // TODO: If only the resource is a VGPR, it may be better to execute the
1431 // scalar load in the waterfall loop if the resource is expected to frequently
1432 // be dynamically uniform.
1433 if (RSrcBank != &AMDGPU::SGPRRegBank) {
1434 // Remove the original instruction to avoid potentially confusing the
1435 // waterfall loop logic.
1436 B.setInstr(*Span.begin());
1437 MI.eraseFromParent();
1438
1439 SmallSet<Register, 4> OpsToWaterfall;
1440
1441 OpsToWaterfall.insert(V: RSrc);
1442 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
1443 SGPROperandRegs&: OpsToWaterfall);
1444 }
1445
1446 if (NumLoads != 1) {
1447 if (Ty.isVector())
1448 B.buildConcatVectors(Res: Dst, Ops: LoadParts);
1449 else
1450 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
1451 }
1452
1453 // We removed the instruction earlier with a waterfall loop.
1454 if (RSrcBank == &AMDGPU::SGPRRegBank)
1455 MI.eraseFromParent();
1456
1457 return true;
1458}
1459
1460bool AMDGPURegisterBankInfo::applyMappingBFE(MachineIRBuilder &B,
1461 const OperandsMapper &OpdMapper,
1462 bool Signed) const {
1463 MachineInstr &MI = OpdMapper.getMI();
1464 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1465
1466 // Insert basic copies
1467 applyDefaultMapping(OpdMapper);
1468
1469 Register DstReg = MI.getOperand(i: 0).getReg();
1470 LLT Ty = MRI.getType(Reg: DstReg);
1471
1472 const LLT S32 = LLT::scalar(SizeInBits: 32);
1473
1474 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
1475 Register SrcReg = MI.getOperand(i: FirstOpnd).getReg();
1476 Register OffsetReg = MI.getOperand(i: FirstOpnd + 1).getReg();
1477 Register WidthReg = MI.getOperand(i: FirstOpnd + 2).getReg();
1478
1479 const RegisterBank *DstBank =
1480 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1481 if (DstBank == &AMDGPU::VGPRRegBank) {
1482 if (Ty == S32)
1483 return true;
1484
1485 // There is no 64-bit vgpr bitfield extract instructions so the operation
1486 // is expanded to a sequence of instructions that implement the operation.
1487 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank);
1488
1489 const LLT S64 = LLT::scalar(SizeInBits: 64);
1490 // Shift the source operand so that extracted bits start at bit 0.
1491 auto ShiftOffset = Signed ? B.buildAShr(Dst: S64, Src0: SrcReg, Src1: OffsetReg)
1492 : B.buildLShr(Dst: S64, Src0: SrcReg, Src1: OffsetReg);
1493 auto UnmergeSOffset = B.buildUnmerge(Res: {S32, S32}, Op: ShiftOffset);
1494
1495 // A 64-bit bitfield extract uses the 32-bit bitfield extract instructions
1496 // if the width is a constant.
1497 if (auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: WidthReg, MRI)) {
1498 // Use the 32-bit bitfield extract instruction if the width is a constant.
1499 // Depending on the width size, use either the low or high 32-bits.
1500 auto Zero = B.buildConstant(Res: S32, Val: 0);
1501 auto WidthImm = ConstWidth->Value.getZExtValue();
1502 if (WidthImm <= 32) {
1503 // Use bitfield extract on the lower 32-bit source, and then sign-extend
1504 // or clear the upper 32-bits.
1505 auto Extract =
1506 Signed ? B.buildSbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 0), LSB: Zero, Width: WidthReg)
1507 : B.buildUbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 0), LSB: Zero, Width: WidthReg);
1508 auto Extend =
1509 Signed ? B.buildAShr(Dst: S32, Src0: Extract, Src1: B.buildConstant(Res: S32, Val: 31)) : Zero;
1510 B.buildMergeLikeInstr(Res: DstReg, Ops: {Extract, Extend});
1511 } else {
1512 // Use bitfield extract on upper 32-bit source, and combine with lower
1513 // 32-bit source.
1514 auto UpperWidth = B.buildConstant(Res: S32, Val: WidthImm - 32);
1515 auto Extract =
1516 Signed
1517 ? B.buildSbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 1), LSB: Zero, Width: UpperWidth)
1518 : B.buildUbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 1), LSB: Zero, Width: UpperWidth);
1519 B.buildMergeLikeInstr(Res: DstReg, Ops: {UnmergeSOffset.getReg(Idx: 0), Extract});
1520 }
1521 MI.eraseFromParent();
1522 return true;
1523 }
1524
1525 // Expand to Src >> Offset << (64 - Width) >> (64 - Width) using 64-bit
1526 // operations.
1527 auto ExtShift = B.buildSub(Dst: S32, Src0: B.buildConstant(Res: S32, Val: 64), Src1: WidthReg);
1528 auto SignBit = B.buildShl(Dst: S64, Src0: ShiftOffset, Src1: ExtShift);
1529 if (Signed)
1530 B.buildAShr(Dst: S64, Src0: SignBit, Src1: ExtShift);
1531 else
1532 B.buildLShr(Dst: S64, Src0: SignBit, Src1: ExtShift);
1533 MI.eraseFromParent();
1534 return true;
1535 }
1536
1537 // The scalar form packs the offset and width in a single operand.
1538
1539 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
1540
1541 // Ensure the high bits are clear to insert the offset.
1542 auto OffsetMask = B.buildConstant(Res: S32, Val: maskTrailingOnes<unsigned>(N: 6));
1543 auto ClampOffset = B.buildAnd(Dst: S32, Src0: OffsetReg, Src1: OffsetMask);
1544
1545 // Zeros out the low bits, so don't bother clamping the input value.
1546 auto ShiftWidth = B.buildShl(Dst: S32, Src0: WidthReg, Src1: B.buildConstant(Res: S32, Val: 16));
1547
1548 // Transformation function, pack the offset and width of a BFE into
1549 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1550 // source, bits [5:0] contain the offset and bits [22:16] the width.
1551 auto MergedInputs = B.buildOr(Dst: S32, Src0: ClampOffset, Src1: ShiftWidth);
1552
1553 // TODO: It might be worth using a pseudo here to avoid scc clobber and
1554 // register class constraints.
1555 unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) :
1556 (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64);
1557
1558 auto MIB = B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {SrcReg, MergedInputs});
1559 constrainSelectedInstRegOperands(I&: *MIB, TII: *TII, TRI: *TRI, RBI: *this);
1560
1561 MI.eraseFromParent();
1562 return true;
1563}
1564
1565bool AMDGPURegisterBankInfo::applyMappingMAD_64_32(
1566 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
1567 MachineInstr &MI = OpdMapper.getMI();
1568 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1569
1570 // Insert basic copies.
1571 applyDefaultMapping(OpdMapper);
1572
1573 Register Dst0 = MI.getOperand(i: 0).getReg();
1574 Register Dst1 = MI.getOperand(i: 1).getReg();
1575 Register Src0 = MI.getOperand(i: 2).getReg();
1576 Register Src1 = MI.getOperand(i: 3).getReg();
1577 Register Src2 = MI.getOperand(i: 4).getReg();
1578
1579 if (MRI.getRegBankOrNull(Reg: Src0) == &AMDGPU::VGPRRegBank)
1580 return true;
1581
1582 bool IsUnsigned = MI.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
1583 LLT S1 = LLT::scalar(SizeInBits: 1);
1584 LLT S32 = LLT::scalar(SizeInBits: 32);
1585
1586 bool DstOnValu = MRI.getRegBankOrNull(Reg: Src2) == &AMDGPU::VGPRRegBank;
1587 bool Accumulate = true;
1588
1589 if (!DstOnValu) {
1590 if (mi_match(R: Src2, MRI, P: m_ZeroInt()))
1591 Accumulate = false;
1592 }
1593
1594 // Keep the multiplication on the SALU.
1595 Register DstHi;
1596 Register DstLo = B.buildMul(Dst: S32, Src0, Src1).getReg(Idx: 0);
1597 bool MulHiInVgpr = false;
1598
1599 MRI.setRegBank(Reg: DstLo, RegBank: AMDGPU::SGPRRegBank);
1600
1601 if (Subtarget.hasSMulHi()) {
1602 DstHi = IsUnsigned ? B.buildUMulH(Dst: S32, Src0, Src1).getReg(Idx: 0)
1603 : B.buildSMulH(Dst: S32, Src0, Src1).getReg(Idx: 0);
1604 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::SGPRRegBank);
1605 } else {
1606 Register VSrc0 = B.buildCopy(Res: S32, Op: Src0).getReg(Idx: 0);
1607 Register VSrc1 = B.buildCopy(Res: S32, Op: Src1).getReg(Idx: 0);
1608
1609 MRI.setRegBank(Reg: VSrc0, RegBank: AMDGPU::VGPRRegBank);
1610 MRI.setRegBank(Reg: VSrc1, RegBank: AMDGPU::VGPRRegBank);
1611
1612 DstHi = IsUnsigned ? B.buildUMulH(Dst: S32, Src0: VSrc0, Src1: VSrc1).getReg(Idx: 0)
1613 : B.buildSMulH(Dst: S32, Src0: VSrc0, Src1: VSrc1).getReg(Idx: 0);
1614 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::VGPRRegBank);
1615
1616 if (!DstOnValu) {
1617 DstHi = buildReadFirstLane(B, MRI, Src: DstHi);
1618 } else {
1619 MulHiInVgpr = true;
1620 }
1621 }
1622
1623 // Accumulate and produce the "carry-out" bit.
1624 //
1625 // The "carry-out" is defined as bit 64 of the result when computed as a
1626 // big integer. For unsigned multiply-add, this matches the usual definition
1627 // of carry-out. For signed multiply-add, bit 64 is the sign bit of the
1628 // result, which is determined as:
1629 // sign(Src0 * Src1) + sign(Src2) + carry-out from unsigned 64-bit add
1630 LLT CarryType = DstOnValu ? S1 : S32;
1631 const RegisterBank &CarryBank =
1632 DstOnValu ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
1633 const RegisterBank &DstBank =
1634 DstOnValu ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank;
1635 Register Carry;
1636 Register Zero;
1637
1638 if (!IsUnsigned) {
1639 Zero = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1640 MRI.setRegBank(Reg: Zero,
1641 RegBank: MulHiInVgpr ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank);
1642
1643 Carry = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: MulHiInVgpr ? S1 : S32, Op0: DstHi, Op1: Zero)
1644 .getReg(Idx: 0);
1645 MRI.setRegBank(Reg: Carry, RegBank: MulHiInVgpr ? AMDGPU::VCCRegBank
1646 : AMDGPU::SGPRRegBank);
1647
1648 if (DstOnValu && !MulHiInVgpr) {
1649 Carry = B.buildTrunc(Res: S1, Op: Carry).getReg(Idx: 0);
1650 MRI.setRegBank(Reg: Carry, RegBank: AMDGPU::VCCRegBank);
1651 }
1652 }
1653
1654 if (Accumulate) {
1655 if (DstOnValu) {
1656 DstLo = B.buildCopy(Res: S32, Op: DstLo).getReg(Idx: 0);
1657 DstHi = B.buildCopy(Res: S32, Op: DstHi).getReg(Idx: 0);
1658 MRI.setRegBank(Reg: DstLo, RegBank: AMDGPU::VGPRRegBank);
1659 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::VGPRRegBank);
1660 }
1661
1662 auto Unmerge = B.buildUnmerge(Res: S32, Op: Src2);
1663 Register Src2Lo = Unmerge.getReg(Idx: 0);
1664 Register Src2Hi = Unmerge.getReg(Idx: 1);
1665 MRI.setRegBank(Reg: Src2Lo, RegBank: DstBank);
1666 MRI.setRegBank(Reg: Src2Hi, RegBank: DstBank);
1667
1668 if (!IsUnsigned) {
1669 auto Src2Sign = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: CarryType, Op0: Src2Hi, Op1: Zero);
1670 MRI.setRegBank(Reg: Src2Sign.getReg(Idx: 0), RegBank: CarryBank);
1671
1672 Carry = B.buildXor(Dst: CarryType, Src0: Carry, Src1: Src2Sign).getReg(Idx: 0);
1673 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1674 }
1675
1676 auto AddLo = B.buildUAddo(Res: S32, CarryOut: CarryType, Op0: DstLo, Op1: Src2Lo);
1677 DstLo = AddLo.getReg(Idx: 0);
1678 Register CarryLo = AddLo.getReg(Idx: 1);
1679 MRI.setRegBank(Reg: DstLo, RegBank: DstBank);
1680 MRI.setRegBank(Reg: CarryLo, RegBank: CarryBank);
1681
1682 auto AddHi = B.buildUAdde(Res: S32, CarryOut: CarryType, Op0: DstHi, Op1: Src2Hi, CarryIn: CarryLo);
1683 DstHi = AddHi.getReg(Idx: 0);
1684 MRI.setRegBank(Reg: DstHi, RegBank: DstBank);
1685
1686 Register CarryHi = AddHi.getReg(Idx: 1);
1687 MRI.setRegBank(Reg: CarryHi, RegBank: CarryBank);
1688
1689 if (IsUnsigned) {
1690 Carry = CarryHi;
1691 } else {
1692 Carry = B.buildXor(Dst: CarryType, Src0: Carry, Src1: CarryHi).getReg(Idx: 0);
1693 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1694 }
1695 } else {
1696 if (IsUnsigned) {
1697 Carry = B.buildConstant(Res: CarryType, Val: 0).getReg(Idx: 0);
1698 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1699 }
1700 }
1701
1702 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
1703
1704 if (DstOnValu) {
1705 B.buildCopy(Res: Dst1, Op: Carry);
1706 } else {
1707 B.buildTrunc(Res: Dst1, Op: Carry);
1708 }
1709
1710 MI.eraseFromParent();
1711 return true;
1712}
1713
1714// Return a suitable opcode for extending the operands of Opc when widening.
1715static unsigned getExtendOp(unsigned Opc) {
1716 switch (Opc) {
1717 case TargetOpcode::G_ASHR:
1718 case TargetOpcode::G_SMIN:
1719 case TargetOpcode::G_SMAX:
1720 return TargetOpcode::G_SEXT;
1721 case TargetOpcode::G_LSHR:
1722 case TargetOpcode::G_UMIN:
1723 case TargetOpcode::G_UMAX:
1724 return TargetOpcode::G_ZEXT;
1725 default:
1726 return TargetOpcode::G_ANYEXT;
1727 }
1728}
1729
1730// Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding
1731// any illegal vector extend or unmerge operations.
1732static std::pair<Register, Register>
1733unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) {
1734 const LLT S32 = LLT::scalar(SizeInBits: 32);
1735 auto Bitcast = B.buildBitcast(Dst: S32, Src);
1736
1737 if (ExtOpcode == TargetOpcode::G_SEXT) {
1738 auto ExtLo = B.buildSExtInReg(Res: S32, Op: Bitcast, ImmOp: 16);
1739 auto ShiftHi = B.buildAShr(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 16));
1740 return std::pair(ExtLo.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1741 }
1742
1743 auto ShiftHi = B.buildLShr(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 16));
1744 if (ExtOpcode == TargetOpcode::G_ZEXT) {
1745 auto ExtLo = B.buildAnd(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 0xffff));
1746 return std::pair(ExtLo.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1747 }
1748
1749 assert(ExtOpcode == TargetOpcode::G_ANYEXT);
1750 return std::pair(Bitcast.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1751}
1752
1753// For cases where only a single copy is inserted for matching register banks.
1754// Replace the register in the instruction operand
1755static bool substituteSimpleCopyRegs(
1756 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) {
1757 SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx));
1758 if (!SrcReg.empty()) {
1759 assert(SrcReg.size() == 1);
1760 OpdMapper.getMI().getOperand(i: OpIdx).setReg(SrcReg[0]);
1761 return true;
1762 }
1763
1764 return false;
1765}
1766
1767/// Handle register layout difference for f16 images for some subtargets.
1768Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
1769 MachineRegisterInfo &MRI,
1770 Register Reg) const {
1771 if (!Subtarget.hasUnpackedD16VMem())
1772 return Reg;
1773
1774 const LLT S16 = LLT::scalar(SizeInBits: 16);
1775 LLT StoreVT = MRI.getType(Reg);
1776 if (!StoreVT.isVector() || StoreVT.getElementType() != S16)
1777 return Reg;
1778
1779 auto Unmerge = B.buildUnmerge(Res: S16, Op: Reg);
1780
1781
1782 SmallVector<Register, 4> WideRegs;
1783 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
1784 WideRegs.push_back(Elt: Unmerge.getReg(Idx: I));
1785
1786 const LLT S32 = LLT::scalar(SizeInBits: 32);
1787 int NumElts = StoreVT.getNumElements();
1788
1789 return B.buildMergeLikeInstr(Res: LLT::fixed_vector(NumElements: NumElts, ScalarTy: S32), Ops: WideRegs)
1790 .getReg(Idx: 0);
1791}
1792
1793bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
1794 Register SrcReg) const {
1795 MachineRegisterInfo &MRI = *B.getMRI();
1796 LLT SrcTy = MRI.getType(Reg: SrcReg);
1797 if (SrcTy.getSizeInBits() == 32) {
1798 // Use a v_mov_b32 here to make the exec dependency explicit.
1799 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1800 .addDef(RegNo: DstReg)
1801 .addUse(RegNo: SrcReg);
1802 return constrainGenericRegister(Reg: DstReg, RC: AMDGPU::VGPR_32RegClass, MRI) &&
1803 constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI);
1804 }
1805
1806 Register TmpReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1807 Register TmpReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1808
1809 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1810 .addDef(RegNo: TmpReg0)
1811 .addUse(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
1812 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1813 .addDef(RegNo: TmpReg1)
1814 .addUse(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
1815 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
1816 .addDef(RegNo: DstReg)
1817 .addUse(RegNo: TmpReg0)
1818 .addImm(Val: AMDGPU::sub0)
1819 .addUse(RegNo: TmpReg1)
1820 .addImm(Val: AMDGPU::sub1);
1821
1822 return constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_64RegClass, MRI) &&
1823 constrainGenericRegister(Reg: DstReg, RC: AMDGPU::VReg_64RegClass, MRI);
1824}
1825
1826/// Utility function for pushing dynamic vector indexes with a constant offset
1827/// into waterfall loops.
1828static void reinsertVectorIndexAdd(MachineIRBuilder &B,
1829 MachineInstr &IdxUseInstr,
1830 unsigned OpIdx,
1831 unsigned ConstOffset) {
1832 MachineRegisterInfo &MRI = *B.getMRI();
1833 const LLT S32 = LLT::scalar(SizeInBits: 32);
1834 Register WaterfallIdx = IdxUseInstr.getOperand(i: OpIdx).getReg();
1835 B.setInsertPt(MBB&: *IdxUseInstr.getParent(), II: IdxUseInstr.getIterator());
1836
1837 auto MaterializedOffset = B.buildConstant(Res: S32, Val: ConstOffset);
1838
1839 auto Add = B.buildAdd(Dst: S32, Src0: WaterfallIdx, Src1: MaterializedOffset);
1840 MRI.setRegBank(Reg: MaterializedOffset.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
1841 MRI.setRegBank(Reg: Add.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
1842 IdxUseInstr.getOperand(i: OpIdx).setReg(Add.getReg(Idx: 0));
1843}
1844
1845/// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the
1846/// original 32-bit source value (to be inserted in the low part of the combined
1847/// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit
1848/// value.
1849static void extendLow32IntoHigh32(MachineIRBuilder &B,
1850 Register Hi32Reg, Register Lo32Reg,
1851 unsigned ExtOpc,
1852 const RegisterBank &RegBank,
1853 bool IsBooleanSrc = false) {
1854 if (ExtOpc == AMDGPU::G_ZEXT) {
1855 B.buildConstant(Res: Hi32Reg, Val: 0);
1856 } else if (ExtOpc == AMDGPU::G_SEXT) {
1857 if (IsBooleanSrc) {
1858 // If we know the original source was an s1, the high half is the same as
1859 // the low.
1860 B.buildCopy(Res: Hi32Reg, Op: Lo32Reg);
1861 } else {
1862 // Replicate sign bit from 32-bit extended part.
1863 auto ShiftAmt = B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 31);
1864 B.getMRI()->setRegBank(Reg: ShiftAmt.getReg(Idx: 0), RegBank);
1865 B.buildAShr(Dst: Hi32Reg, Src0: Lo32Reg, Src1: ShiftAmt);
1866 }
1867 } else {
1868 assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension");
1869 B.buildUndef(Res: Hi32Reg);
1870 }
1871}
1872
1873bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect(
1874 MachineIRBuilder &B, MachineInstr &MI,
1875 const OperandsMapper &OpdMapper) const {
1876 MachineRegisterInfo &MRI = *B.getMRI();
1877
1878 Register VecReg = MI.getOperand(i: 1).getReg();
1879 Register Idx = MI.getOperand(i: 2).getReg();
1880
1881 const RegisterBank &IdxBank =
1882 *OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
1883
1884 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
1885
1886 LLT VecTy = MRI.getType(Reg: VecReg);
1887 unsigned EltSize = VecTy.getScalarSizeInBits();
1888 unsigned NumElem = VecTy.getNumElements();
1889
1890 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
1891 IsDivergentIdx, Subtarget: &Subtarget))
1892 return false;
1893
1894 LLT S32 = LLT::scalar(SizeInBits: 32);
1895
1896 const RegisterBank &DstBank =
1897 *OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1898 const RegisterBank &SrcBank =
1899 *OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1900
1901 const RegisterBank &CCBank =
1902 (DstBank == AMDGPU::SGPRRegBank &&
1903 SrcBank == AMDGPU::SGPRRegBank &&
1904 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
1905 : AMDGPU::VCCRegBank;
1906 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(SizeInBits: 1);
1907
1908 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
1909 Idx = B.buildCopy(Res: S32, Op: Idx)->getOperand(i: 0).getReg();
1910 MRI.setRegBank(Reg: Idx, RegBank: AMDGPU::VGPRRegBank);
1911 }
1912
1913 LLT EltTy = VecTy.getScalarType();
1914 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
1915 unsigned NumLanes = DstRegs.size();
1916 if (!NumLanes)
1917 NumLanes = 1;
1918 else
1919 EltTy = MRI.getType(Reg: DstRegs[0]);
1920
1921 auto UnmergeToEltTy = B.buildUnmerge(Res: EltTy, Op: VecReg);
1922 SmallVector<Register, 2> Res(NumLanes);
1923 for (unsigned L = 0; L < NumLanes; ++L)
1924 Res[L] = UnmergeToEltTy.getReg(Idx: L);
1925
1926 for (unsigned I = 1; I < NumElem; ++I) {
1927 auto IC = B.buildConstant(Res: S32, Val: I);
1928 MRI.setRegBank(Reg: IC->getOperand(i: 0).getReg(), RegBank: AMDGPU::SGPRRegBank);
1929 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CCTy, Op0: Idx, Op1: IC);
1930 MRI.setRegBank(Reg: Cmp->getOperand(i: 0).getReg(), RegBank: CCBank);
1931
1932 for (unsigned L = 0; L < NumLanes; ++L) {
1933 auto S = B.buildSelect(Res: EltTy, Tst: Cmp,
1934 Op0: UnmergeToEltTy.getReg(Idx: I * NumLanes + L), Op1: Res[L]);
1935
1936 for (unsigned N : { 0, 2, 3 })
1937 MRI.setRegBank(Reg: S->getOperand(i: N).getReg(), RegBank: DstBank);
1938
1939 Res[L] = S->getOperand(i: 0).getReg();
1940 }
1941 }
1942
1943 for (unsigned L = 0; L < NumLanes; ++L) {
1944 Register DstReg = (NumLanes == 1) ? MI.getOperand(i: 0).getReg() : DstRegs[L];
1945 B.buildCopy(Res: DstReg, Op: Res[L]);
1946 MRI.setRegBank(Reg: DstReg, RegBank: DstBank);
1947 }
1948
1949 MRI.setRegBank(Reg: MI.getOperand(i: 0).getReg(), RegBank: DstBank);
1950 MI.eraseFromParent();
1951
1952 return true;
1953}
1954
1955// Insert a cross regbank copy for a register if it already has a bank that
1956// differs from the one we want to set.
1957static Register constrainRegToBank(MachineRegisterInfo &MRI,
1958 MachineIRBuilder &B, Register &Reg,
1959 const RegisterBank &Bank) {
1960 const RegisterBank *CurrBank = MRI.getRegBankOrNull(Reg);
1961 if (CurrBank && *CurrBank != Bank) {
1962 Register Copy = B.buildCopy(Res: MRI.getType(Reg), Op: Reg).getReg(Idx: 0);
1963 MRI.setRegBank(Reg: Copy, RegBank: Bank);
1964 return Copy;
1965 }
1966
1967 MRI.setRegBank(Reg, RegBank: Bank);
1968 return Reg;
1969}
1970
1971bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect(
1972 MachineIRBuilder &B, MachineInstr &MI,
1973 const OperandsMapper &OpdMapper) const {
1974
1975 MachineRegisterInfo &MRI = *B.getMRI();
1976 Register VecReg = MI.getOperand(i: 1).getReg();
1977 Register Idx = MI.getOperand(i: 3).getReg();
1978
1979 const RegisterBank &IdxBank =
1980 *OpdMapper.getInstrMapping().getOperandMapping(i: 3).BreakDown[0].RegBank;
1981
1982 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
1983
1984 LLT VecTy = MRI.getType(Reg: VecReg);
1985 unsigned EltSize = VecTy.getScalarSizeInBits();
1986 unsigned NumElem = VecTy.getNumElements();
1987
1988 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
1989 IsDivergentIdx, Subtarget: &Subtarget))
1990 return false;
1991
1992 LLT S32 = LLT::scalar(SizeInBits: 32);
1993
1994 const RegisterBank &DstBank =
1995 *OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1996 const RegisterBank &SrcBank =
1997 *OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1998 const RegisterBank &InsBank =
1999 *OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2000
2001 const RegisterBank &CCBank =
2002 (DstBank == AMDGPU::SGPRRegBank &&
2003 SrcBank == AMDGPU::SGPRRegBank &&
2004 InsBank == AMDGPU::SGPRRegBank &&
2005 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
2006 : AMDGPU::VCCRegBank;
2007 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(SizeInBits: 1);
2008
2009 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
2010 Idx = B.buildCopy(Res: S32, Op: Idx)->getOperand(i: 0).getReg();
2011 MRI.setRegBank(Reg: Idx, RegBank: AMDGPU::VGPRRegBank);
2012 }
2013
2014 LLT EltTy = VecTy.getScalarType();
2015 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(OpIdx: 2));
2016 unsigned NumLanes = InsRegs.size();
2017 if (!NumLanes) {
2018 NumLanes = 1;
2019 InsRegs.push_back(Elt: MI.getOperand(i: 2).getReg());
2020 } else {
2021 EltTy = MRI.getType(Reg: InsRegs[0]);
2022 }
2023
2024 auto UnmergeToEltTy = B.buildUnmerge(Res: EltTy, Op: VecReg);
2025 SmallVector<Register, 16> Ops(NumElem * NumLanes);
2026
2027 for (unsigned I = 0; I < NumElem; ++I) {
2028 auto IC = B.buildConstant(Res: S32, Val: I);
2029 MRI.setRegBank(Reg: IC->getOperand(i: 0).getReg(), RegBank: AMDGPU::SGPRRegBank);
2030 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CCTy, Op0: Idx, Op1: IC);
2031 MRI.setRegBank(Reg: Cmp->getOperand(i: 0).getReg(), RegBank: CCBank);
2032
2033 for (unsigned L = 0; L < NumLanes; ++L) {
2034 Register Op0 = constrainRegToBank(MRI, B, Reg&: InsRegs[L], Bank: DstBank);
2035 Register Op1 = UnmergeToEltTy.getReg(Idx: I * NumLanes + L);
2036 Op1 = constrainRegToBank(MRI, B, Reg&: Op1, Bank: DstBank);
2037
2038 Register Select = B.buildSelect(Res: EltTy, Tst: Cmp, Op0, Op1).getReg(Idx: 0);
2039 MRI.setRegBank(Reg: Select, RegBank: DstBank);
2040
2041 Ops[I * NumLanes + L] = Select;
2042 }
2043 }
2044
2045 LLT MergeTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: EltTy);
2046 if (MergeTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg())) {
2047 B.buildBuildVector(Res: MI.getOperand(i: 0), Ops);
2048 } else {
2049 auto Vec = B.buildBuildVector(Res: MergeTy, Ops);
2050 MRI.setRegBank(Reg: Vec->getOperand(i: 0).getReg(), RegBank: DstBank);
2051 B.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: Vec);
2052 }
2053
2054 MRI.setRegBank(Reg: MI.getOperand(i: 0).getReg(), RegBank: DstBank);
2055 MI.eraseFromParent();
2056
2057 return true;
2058}
2059
2060// Break s_mul_u64 into 32-bit vector operations.
2061void AMDGPURegisterBankInfo::applyMappingSMULU64(
2062 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
2063 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2064 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(OpIdx: 1));
2065 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2066
2067 // All inputs are SGPRs, nothing special to do.
2068 if (DefRegs.empty()) {
2069 assert(Src0Regs.empty() && Src1Regs.empty());
2070 applyDefaultMapping(OpdMapper);
2071 return;
2072 }
2073
2074 assert(DefRegs.size() == 2);
2075 assert(Src0Regs.size() == Src1Regs.size() &&
2076 (Src0Regs.empty() || Src0Regs.size() == 2));
2077
2078 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2079 MachineInstr &MI = OpdMapper.getMI();
2080 Register DstReg = MI.getOperand(i: 0).getReg();
2081 LLT HalfTy = LLT::scalar(SizeInBits: 32);
2082
2083 // Depending on where the source registers came from, the generic code may
2084 // have decided to split the inputs already or not. If not, we still need to
2085 // extract the values.
2086
2087 if (Src0Regs.empty())
2088 split64BitValueForMapping(B, Regs&: Src0Regs, HalfTy, Reg: MI.getOperand(i: 1).getReg());
2089 else
2090 setRegsToType(MRI, Regs: Src0Regs, NewTy: HalfTy);
2091
2092 if (Src1Regs.empty())
2093 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2094 else
2095 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2096
2097 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2098
2099 // The multiplication is done as follows:
2100 //
2101 // Op1H Op1L
2102 // * Op0H Op0L
2103 // --------------------
2104 // Op1H*Op0L Op1L*Op0L
2105 // + Op1H*Op0H Op1L*Op0H
2106 // -----------------------------------------
2107 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
2108 //
2109 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
2110 // value and that would overflow.
2111 // The low 32-bit value is Op1L*Op0L.
2112 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from
2113 // Op1L*Op0L).
2114
2115 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank);
2116
2117 Register Hi = B.buildUMulH(Dst: HalfTy, Src0: Src0Regs[0], Src1: Src1Regs[0]).getReg(Idx: 0);
2118 Register MulLoHi = B.buildMul(Dst: HalfTy, Src0: Src0Regs[0], Src1: Src1Regs[1]).getReg(Idx: 0);
2119 Register Add = B.buildAdd(Dst: HalfTy, Src0: Hi, Src1: MulLoHi).getReg(Idx: 0);
2120 Register MulHiLo = B.buildMul(Dst: HalfTy, Src0: Src0Regs[1], Src1: Src1Regs[0]).getReg(Idx: 0);
2121 B.buildAdd(Dst: DefRegs[1], Src0: Add, Src1: MulHiLo);
2122 B.buildMul(Dst: DefRegs[0], Src0: Src0Regs[0], Src1: Src1Regs[0]);
2123
2124 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2125 MI.eraseFromParent();
2126}
2127
2128void AMDGPURegisterBankInfo::applyMappingImpl(
2129 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
2130 MachineInstr &MI = OpdMapper.getMI();
2131 B.setInstrAndDebugLoc(MI);
2132 unsigned Opc = MI.getOpcode();
2133 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2134 switch (Opc) {
2135 case AMDGPU::G_CONSTANT:
2136 case AMDGPU::G_IMPLICIT_DEF: {
2137 Register DstReg = MI.getOperand(i: 0).getReg();
2138 LLT DstTy = MRI.getType(Reg: DstReg);
2139 if (DstTy != LLT::scalar(SizeInBits: 1))
2140 break;
2141
2142 const RegisterBank *DstBank =
2143 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2144 if (DstBank == &AMDGPU::VCCRegBank)
2145 break;
2146 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2147 if (DefRegs.empty())
2148 DefRegs.push_back(Elt: DstReg);
2149
2150 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
2151
2152 Register NewDstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2153 LLVMContext &Ctx = B.getMF().getFunction().getContext();
2154
2155 MI.getOperand(i: 0).setReg(NewDstReg);
2156 if (Opc != AMDGPU::G_IMPLICIT_DEF) {
2157 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
2158 MI.getOperand(i: 1).setCImm(
2159 ConstantInt::get(Ty: IntegerType::getInt32Ty(C&: Ctx), V: ConstVal));
2160 }
2161
2162 MRI.setRegBank(Reg: NewDstReg, RegBank: *DstBank);
2163 B.buildTrunc(Res: DefRegs[0], Op: NewDstReg);
2164 return;
2165 }
2166 case AMDGPU::G_PHI: {
2167 Register DstReg = MI.getOperand(i: 0).getReg();
2168 LLT DstTy = MRI.getType(Reg: DstReg);
2169 if (DstTy != LLT::scalar(SizeInBits: 1))
2170 break;
2171
2172 const LLT S32 = LLT::scalar(SizeInBits: 32);
2173 const RegisterBank *DstBank =
2174 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2175 if (DstBank == &AMDGPU::VCCRegBank) {
2176 applyDefaultMapping(OpdMapper);
2177 // The standard handling only considers the result register bank for
2178 // phis. For VCC, blindly inserting a copy when the phi is lowered will
2179 // produce an invalid copy. We can only copy with some kind of compare to
2180 // get a vector boolean result. Insert a register bank copy that will be
2181 // correctly lowered to a compare.
2182 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2183 Register SrcReg = MI.getOperand(i: I).getReg();
2184 const RegisterBank *SrcBank = getRegBank(Reg: SrcReg, MRI, TRI: *TRI);
2185
2186 if (SrcBank != &AMDGPU::VCCRegBank) {
2187 MachineBasicBlock *SrcMBB = MI.getOperand(i: I + 1).getMBB();
2188 B.setInsertPt(MBB&: *SrcMBB, II: SrcMBB->getFirstTerminator());
2189
2190 auto Copy = B.buildCopy(Res: LLT::scalar(SizeInBits: 1), Op: SrcReg);
2191 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: AMDGPU::VCCRegBank);
2192 MI.getOperand(i: I).setReg(Copy.getReg(Idx: 0));
2193 }
2194 }
2195
2196 return;
2197 }
2198
2199 // Phi handling is strange and only considers the bank of the destination.
2200 substituteSimpleCopyRegs(OpdMapper, OpIdx: 0);
2201
2202 // Promote SGPR/VGPR booleans to s32
2203 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
2204 B.setInsertPt(MBB&: B.getMBB(), II: MI);
2205 LegalizerHelper Helper(B.getMF(), ApplyBank, B);
2206
2207 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2208 llvm_unreachable("widen scalar should have succeeded");
2209
2210 return;
2211 }
2212 case AMDGPU::G_FCMP:
2213 if (!Subtarget.hasSALUFloatInsts())
2214 break;
2215 [[fallthrough]];
2216 case AMDGPU::G_ICMP:
2217 case AMDGPU::G_UADDO:
2218 case AMDGPU::G_USUBO:
2219 case AMDGPU::G_UADDE:
2220 case AMDGPU::G_SADDE:
2221 case AMDGPU::G_USUBE:
2222 case AMDGPU::G_SSUBE: {
2223 unsigned BoolDstOp =
2224 (Opc == AMDGPU::G_ICMP || Opc == AMDGPU::G_FCMP) ? 0 : 1;
2225 Register DstReg = MI.getOperand(i: BoolDstOp).getReg();
2226
2227 const RegisterBank *DstBank =
2228 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2229 if (DstBank != &AMDGPU::SGPRRegBank)
2230 break;
2231
2232 const bool HasCarryIn = MI.getNumOperands() == 5;
2233
2234 // If this is a scalar compare, promote the result to s32, as the selection
2235 // will end up using a copy to a 32-bit vreg.
2236 const LLT S32 = LLT::scalar(SizeInBits: 32);
2237 Register NewDstReg = MRI.createGenericVirtualRegister(Ty: S32);
2238 MRI.setRegBank(Reg: NewDstReg, RegBank: AMDGPU::SGPRRegBank);
2239 MI.getOperand(i: BoolDstOp).setReg(NewDstReg);
2240
2241 if (HasCarryIn) {
2242 Register NewSrcReg = MRI.createGenericVirtualRegister(Ty: S32);
2243 MRI.setRegBank(Reg: NewSrcReg, RegBank: AMDGPU::SGPRRegBank);
2244 B.buildZExt(Res: NewSrcReg, Op: MI.getOperand(i: 4).getReg());
2245 MI.getOperand(i: 4).setReg(NewSrcReg);
2246 }
2247
2248 MachineBasicBlock *MBB = MI.getParent();
2249 B.setInsertPt(MBB&: *MBB, II: std::next(x: MI.getIterator()));
2250
2251 // If we had a constrained VCC result register, a copy was inserted to VCC
2252 // from SGPR.
2253 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2254 if (DefRegs.empty())
2255 DefRegs.push_back(Elt: DstReg);
2256 B.buildTrunc(Res: DefRegs[0], Op: NewDstReg);
2257 return;
2258 }
2259 case AMDGPU::G_SELECT: {
2260 Register DstReg = MI.getOperand(i: 0).getReg();
2261 LLT DstTy = MRI.getType(Reg: DstReg);
2262
2263 SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(OpIdx: 1));
2264 if (CondRegs.empty())
2265 CondRegs.push_back(Elt: MI.getOperand(i: 1).getReg());
2266 else {
2267 assert(CondRegs.size() == 1);
2268 }
2269
2270 const RegisterBank *CondBank = getRegBank(Reg: CondRegs[0], MRI, TRI: *TRI);
2271 if (CondBank == &AMDGPU::SGPRRegBank) {
2272 const LLT S32 = LLT::scalar(SizeInBits: 32);
2273 Register NewCondReg = MRI.createGenericVirtualRegister(Ty: S32);
2274 MRI.setRegBank(Reg: NewCondReg, RegBank: AMDGPU::SGPRRegBank);
2275
2276 MI.getOperand(i: 1).setReg(NewCondReg);
2277 B.buildZExt(Res: NewCondReg, Op: CondRegs[0]);
2278 }
2279
2280 if (DstTy.getSizeInBits() != 64)
2281 break;
2282
2283 LLT HalfTy = getHalfSizedType(Ty: DstTy);
2284
2285 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2286 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2287 SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(OpIdx: 3));
2288
2289 // All inputs are SGPRs, nothing special to do.
2290 if (DefRegs.empty()) {
2291 assert(Src1Regs.empty() && Src2Regs.empty());
2292 break;
2293 }
2294
2295 if (Src1Regs.empty())
2296 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2297 else {
2298 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2299 }
2300
2301 if (Src2Regs.empty())
2302 split64BitValueForMapping(B, Regs&: Src2Regs, HalfTy, Reg: MI.getOperand(i: 3).getReg());
2303 else
2304 setRegsToType(MRI, Regs: Src2Regs, NewTy: HalfTy);
2305
2306 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2307
2308 auto Flags = MI.getFlags();
2309 B.buildSelect(Res: DefRegs[0], Tst: CondRegs[0], Op0: Src1Regs[0], Op1: Src2Regs[0], Flags);
2310 B.buildSelect(Res: DefRegs[1], Tst: CondRegs[0], Op0: Src1Regs[1], Op1: Src2Regs[1], Flags);
2311
2312 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2313 MI.eraseFromParent();
2314 return;
2315 }
2316 case AMDGPU::G_BRCOND: {
2317 Register CondReg = MI.getOperand(i: 0).getReg();
2318 // FIXME: Should use legalizer helper, but should change bool ext type.
2319 const RegisterBank *CondBank =
2320 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2321
2322 if (CondBank == &AMDGPU::SGPRRegBank) {
2323 const LLT S32 = LLT::scalar(SizeInBits: 32);
2324 Register NewCondReg = MRI.createGenericVirtualRegister(Ty: S32);
2325 MRI.setRegBank(Reg: NewCondReg, RegBank: AMDGPU::SGPRRegBank);
2326
2327 MI.getOperand(i: 0).setReg(NewCondReg);
2328 B.buildZExt(Res: NewCondReg, Op: CondReg);
2329 return;
2330 }
2331
2332 break;
2333 }
2334 case AMDGPU::G_AND:
2335 case AMDGPU::G_OR:
2336 case AMDGPU::G_XOR: {
2337 // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
2338 // there is a VGPR input.
2339 Register DstReg = MI.getOperand(i: 0).getReg();
2340 LLT DstTy = MRI.getType(Reg: DstReg);
2341
2342 const RegisterBank *DstBank =
2343 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2344
2345 if (DstTy.getSizeInBits() == 1) {
2346 if (DstBank == &AMDGPU::VCCRegBank)
2347 break;
2348
2349 MachineFunction *MF = MI.getMF();
2350 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
2351 LegalizerHelper Helper(*MF, ApplyBank, B);
2352
2353 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: LLT::scalar(SizeInBits: 32)) !=
2354 LegalizerHelper::Legalized)
2355 llvm_unreachable("widen scalar should have succeeded");
2356 return;
2357 }
2358
2359 if (DstTy.getSizeInBits() == 16 && DstBank == &AMDGPU::SGPRRegBank) {
2360 const LLT S32 = LLT::scalar(SizeInBits: 32);
2361 MachineBasicBlock *MBB = MI.getParent();
2362 MachineFunction *MF = MBB->getParent();
2363 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank);
2364 LegalizerHelper Helper(*MF, ApplySALU, B);
2365 // Widen to S32, but handle `G_XOR x, -1` differently. Legalizer widening
2366 // will use a G_ANYEXT to extend the -1 which prevents matching G_XOR -1
2367 // as "not".
2368 if (MI.getOpcode() == AMDGPU::G_XOR &&
2369 mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: -1))) {
2370 Helper.widenScalarSrc(MI, WideTy: S32, OpIdx: 1, ExtOpcode: AMDGPU::G_ANYEXT);
2371 Helper.widenScalarSrc(MI, WideTy: S32, OpIdx: 2, ExtOpcode: AMDGPU::G_SEXT);
2372 Helper.widenScalarDst(MI, WideTy: S32);
2373 } else {
2374 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2375 llvm_unreachable("widen scalar should have succeeded");
2376 }
2377 return;
2378 }
2379
2380 if (DstTy.getSizeInBits() != 64)
2381 break;
2382
2383 LLT HalfTy = getHalfSizedType(Ty: DstTy);
2384 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2385 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(OpIdx: 1));
2386 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2387
2388 // All inputs are SGPRs, nothing special to do.
2389 if (DefRegs.empty()) {
2390 assert(Src0Regs.empty() && Src1Regs.empty());
2391 break;
2392 }
2393
2394 assert(DefRegs.size() == 2);
2395 assert(Src0Regs.size() == Src1Regs.size() &&
2396 (Src0Regs.empty() || Src0Regs.size() == 2));
2397
2398 // Depending on where the source registers came from, the generic code may
2399 // have decided to split the inputs already or not. If not, we still need to
2400 // extract the values.
2401
2402 if (Src0Regs.empty())
2403 split64BitValueForMapping(B, Regs&: Src0Regs, HalfTy, Reg: MI.getOperand(i: 1).getReg());
2404 else
2405 setRegsToType(MRI, Regs: Src0Regs, NewTy: HalfTy);
2406
2407 if (Src1Regs.empty())
2408 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2409 else
2410 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2411
2412 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2413
2414 auto Flags = MI.getFlags();
2415 B.buildInstr(Opc, DstOps: {DefRegs[0]}, SrcOps: {Src0Regs[0], Src1Regs[0]}, Flags);
2416 B.buildInstr(Opc, DstOps: {DefRegs[1]}, SrcOps: {Src0Regs[1], Src1Regs[1]}, Flags);
2417
2418 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2419 MI.eraseFromParent();
2420 return;
2421 }
2422 case AMDGPU::G_ABS: {
2423 Register SrcReg = MI.getOperand(i: 1).getReg();
2424 const RegisterBank *SrcBank = MRI.getRegBankOrNull(Reg: SrcReg);
2425
2426 // There is no VALU abs instruction so we need to replace it with a sub and
2427 // max combination.
2428 if (SrcBank && SrcBank == &AMDGPU::VGPRRegBank) {
2429 MachineFunction *MF = MI.getMF();
2430 ApplyRegBankMapping Apply(B, *this, MRI, &AMDGPU::VGPRRegBank);
2431 LegalizerHelper Helper(*MF, Apply, B);
2432
2433 if (Helper.lowerAbsToMaxNeg(MI) != LegalizerHelper::Legalized)
2434 llvm_unreachable("lowerAbsToMaxNeg should have succeeded");
2435 return;
2436 }
2437 [[fallthrough]];
2438 }
2439 case AMDGPU::G_ADD:
2440 case AMDGPU::G_SUB:
2441 case AMDGPU::G_MUL:
2442 case AMDGPU::G_SHL:
2443 case AMDGPU::G_LSHR:
2444 case AMDGPU::G_ASHR:
2445 case AMDGPU::G_SMIN:
2446 case AMDGPU::G_SMAX:
2447 case AMDGPU::G_UMIN:
2448 case AMDGPU::G_UMAX: {
2449 Register DstReg = MI.getOperand(i: 0).getReg();
2450 LLT DstTy = MRI.getType(Reg: DstReg);
2451
2452 // Special case for s_mul_u64. There is not a vector equivalent of
2453 // s_mul_u64. Hence, we have to break down s_mul_u64 into 32-bit vector
2454 // multiplications.
2455 if (!Subtarget.useVMulU64Inst() && Opc == AMDGPU::G_MUL &&
2456 DstTy.getSizeInBits() == 64) {
2457 applyMappingSMULU64(B, OpdMapper);
2458 return;
2459 }
2460
2461 // 16-bit operations are VALU only, but can be promoted to 32-bit SALU.
2462 // Packed 16-bit operations need to be scalarized and promoted.
2463 if (DstTy != LLT::scalar(SizeInBits: 16) && DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16))
2464 break;
2465
2466 const RegisterBank *DstBank =
2467 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2468 if (DstBank == &AMDGPU::VGPRRegBank)
2469 break;
2470
2471 const LLT S32 = LLT::scalar(SizeInBits: 32);
2472 MachineBasicBlock *MBB = MI.getParent();
2473 MachineFunction *MF = MBB->getParent();
2474 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank);
2475
2476 if (DstTy.isVector() && Opc == AMDGPU::G_ABS) {
2477 Register WideSrcLo, WideSrcHi;
2478
2479 std::tie(args&: WideSrcLo, args&: WideSrcHi) =
2480 unpackV2S16ToS32(B, Src: MI.getOperand(i: 1).getReg(), ExtOpcode: TargetOpcode::G_SEXT);
2481 auto Lo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {S32}, SrcOps: {WideSrcLo});
2482 auto Hi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {S32}, SrcOps: {WideSrcHi});
2483 B.buildBuildVectorTrunc(Res: DstReg, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
2484 MI.eraseFromParent();
2485 return;
2486 }
2487
2488 if (DstTy.isVector()) {
2489 Register WideSrc0Lo, WideSrc0Hi;
2490 Register WideSrc1Lo, WideSrc1Hi;
2491
2492 unsigned ExtendOp = getExtendOp(Opc: MI.getOpcode());
2493 std::tie(args&: WideSrc0Lo, args&: WideSrc0Hi)
2494 = unpackV2S16ToS32(B, Src: MI.getOperand(i: 1).getReg(), ExtOpcode: ExtendOp);
2495 std::tie(args&: WideSrc1Lo, args&: WideSrc1Hi)
2496 = unpackV2S16ToS32(B, Src: MI.getOperand(i: 2).getReg(), ExtOpcode: ExtendOp);
2497 auto Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {S32}, SrcOps: {WideSrc0Lo, WideSrc1Lo});
2498 auto Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {S32}, SrcOps: {WideSrc0Hi, WideSrc1Hi});
2499 B.buildBuildVectorTrunc(Res: DstReg, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
2500 MI.eraseFromParent();
2501 } else {
2502 LegalizerHelper Helper(*MF, ApplySALU, B);
2503
2504 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2505 llvm_unreachable("widen scalar should have succeeded");
2506
2507 // FIXME: s16 shift amounts should be legal.
2508 if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR ||
2509 Opc == AMDGPU::G_ASHR) {
2510 B.setInsertPt(MBB&: *MBB, II: MI.getIterator());
2511 if (Helper.widenScalar(MI, TypeIdx: 1, WideTy: S32) != LegalizerHelper::Legalized)
2512 llvm_unreachable("widen scalar should have succeeded");
2513 }
2514 }
2515
2516 return;
2517 }
2518 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
2519 case AMDGPU::G_AMDGPU_S_MUL_U64_U32: {
2520 // This is a special case for s_mul_u64. We use
2521 // G_AMDGPU_S_MUL_I64_I32 opcode to represent an s_mul_u64 operation
2522 // where the 33 higher bits are sign-extended and
2523 // G_AMDGPU_S_MUL_U64_U32 opcode to represent an s_mul_u64 operation
2524 // where the 32 higher bits are zero-extended. In case scalar registers are
2525 // selected, both opcodes are lowered as s_mul_u64. If the vector registers
2526 // are selected, then G_AMDGPU_S_MUL_I64_I32 and
2527 // G_AMDGPU_S_MUL_U64_U32 are lowered with a vector mad instruction.
2528
2529 // Insert basic copies.
2530 applyDefaultMapping(OpdMapper);
2531
2532 Register DstReg = MI.getOperand(i: 0).getReg();
2533 Register SrcReg0 = MI.getOperand(i: 1).getReg();
2534 Register SrcReg1 = MI.getOperand(i: 2).getReg();
2535 const LLT S32 = LLT::scalar(SizeInBits: 32);
2536 const LLT S64 = LLT::scalar(SizeInBits: 64);
2537 assert(MRI.getType(DstReg) == S64 && "This is a special case for s_mul_u64 "
2538 "that handles only 64-bit operands.");
2539 const RegisterBank *DstBank =
2540 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2541
2542 // Replace G_AMDGPU_S_MUL_I64_I32 and G_AMDGPU_S_MUL_U64_U32
2543 // with s_mul_u64 operation.
2544 if (DstBank == &AMDGPU::SGPRRegBank) {
2545 MI.setDesc(TII->get(Opcode: AMDGPU::S_MUL_U64));
2546 MRI.setRegClass(Reg: DstReg, RC: &AMDGPU::SGPR_64RegClass);
2547 MRI.setRegClass(Reg: SrcReg0, RC: &AMDGPU::SGPR_64RegClass);
2548 MRI.setRegClass(Reg: SrcReg1, RC: &AMDGPU::SGPR_64RegClass);
2549 return;
2550 }
2551
2552 // Replace G_AMDGPU_S_MUL_I64_I32 and G_AMDGPU_S_MUL_U64_U32
2553 // with a vector mad.
2554 assert(MRI.getRegBankOrNull(DstReg) == &AMDGPU::VGPRRegBank &&
2555 "The destination operand should be in vector registers.");
2556
2557 // Extract the lower subregister from the first operand.
2558 Register Op0L = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2559 MRI.setRegClass(Reg: Op0L, RC: &AMDGPU::VGPR_32RegClass);
2560 MRI.setType(VReg: Op0L, Ty: S32);
2561 B.buildTrunc(Res: Op0L, Op: SrcReg0);
2562
2563 // Extract the lower subregister from the second operand.
2564 Register Op1L = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2565 MRI.setRegClass(Reg: Op1L, RC: &AMDGPU::VGPR_32RegClass);
2566 MRI.setType(VReg: Op1L, Ty: S32);
2567 B.buildTrunc(Res: Op1L, Op: SrcReg1);
2568
2569 unsigned NewOpc = Opc == AMDGPU::G_AMDGPU_S_MUL_U64_U32
2570 ? AMDGPU::G_AMDGPU_MAD_U64_U32
2571 : AMDGPU::G_AMDGPU_MAD_I64_I32;
2572
2573 MachineIRBuilder B(MI);
2574 Register Zero64 = B.buildConstant(Res: S64, Val: 0).getReg(Idx: 0);
2575 MRI.setRegClass(Reg: Zero64, RC: &AMDGPU::VReg_64RegClass);
2576 Register CarryOut = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
2577 MRI.setRegClass(Reg: CarryOut, RC: &AMDGPU::VReg_64RegClass);
2578 B.buildInstr(Opc: NewOpc, DstOps: {DstReg, CarryOut}, SrcOps: {Op0L, Op1L, Zero64});
2579 MI.eraseFromParent();
2580 return;
2581 }
2582 case AMDGPU::G_SEXT_INREG: {
2583 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
2584 if (SrcRegs.empty())
2585 break; // Nothing to repair
2586
2587 const LLT S32 = LLT::scalar(SizeInBits: 32);
2588 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank);
2589
2590 // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs
2591 // we would need to further expand, and doesn't let us directly set the
2592 // result registers.
2593 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
2594
2595 int Amt = MI.getOperand(i: 2).getImm();
2596 if (Amt <= 32) {
2597 // Downstream users have expectations for the high bit behavior, so freeze
2598 // incoming undefined bits.
2599 if (Amt == 32) {
2600 // The low bits are unchanged.
2601 B.buildFreeze(Dst: DstRegs[0], Src: SrcRegs[0]);
2602 } else {
2603 auto Freeze = B.buildFreeze(Dst: S32, Src: SrcRegs[0]);
2604 // Extend in the low bits and propagate the sign bit to the high half.
2605 B.buildSExtInReg(Res: DstRegs[0], Op: Freeze, ImmOp: Amt);
2606 }
2607
2608 B.buildAShr(Dst: DstRegs[1], Src0: DstRegs[0], Src1: B.buildConstant(Res: S32, Val: 31));
2609 } else {
2610 // The low bits are unchanged, and extend in the high bits.
2611 // No freeze required
2612 B.buildCopy(Res: DstRegs[0], Op: SrcRegs[0]);
2613 B.buildSExtInReg(Res: DstRegs[1], Op: DstRegs[0], ImmOp: Amt - 32);
2614 }
2615
2616 Register DstReg = MI.getOperand(i: 0).getReg();
2617 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2618 MI.eraseFromParent();
2619 return;
2620 }
2621 case AMDGPU::G_CTPOP:
2622 case AMDGPU::G_BITREVERSE: {
2623 const RegisterBank *DstBank =
2624 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2625 if (DstBank == &AMDGPU::SGPRRegBank)
2626 break;
2627
2628 Register SrcReg = MI.getOperand(i: 1).getReg();
2629 const LLT S32 = LLT::scalar(SizeInBits: 32);
2630 LLT Ty = MRI.getType(Reg: SrcReg);
2631 if (Ty == S32)
2632 break;
2633
2634 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank);
2635
2636 MachineFunction &MF = B.getMF();
2637 LegalizerHelper Helper(MF, ApplyVALU, B);
2638
2639 if (Helper.narrowScalar(MI, TypeIdx: 1, NarrowTy: S32) != LegalizerHelper::Legalized)
2640 llvm_unreachable("narrowScalar should have succeeded");
2641 return;
2642 }
2643 case AMDGPU::G_AMDGPU_FFBH_U32:
2644 case AMDGPU::G_AMDGPU_FFBL_B32:
2645 case AMDGPU::G_CTLZ_ZERO_POISON:
2646 case AMDGPU::G_CTTZ_ZERO_POISON: {
2647 const RegisterBank *DstBank =
2648 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2649 if (DstBank == &AMDGPU::SGPRRegBank)
2650 break;
2651
2652 Register SrcReg = MI.getOperand(i: 1).getReg();
2653 const LLT S32 = LLT::scalar(SizeInBits: 32);
2654 LLT Ty = MRI.getType(Reg: SrcReg);
2655 if (Ty == S32)
2656 break;
2657
2658 // We can narrow this more efficiently than Helper can by using ffbh/ffbl
2659 // which return -1 when the input is zero:
2660 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
2661 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
2662 // (ffbh hi:lo) -> (umin (ffbh hi), (uaddsat (ffbh lo), 32))
2663 // (ffbl hi:lo) -> (umin (uaddsat (ffbh hi), 32), (ffbh lo))
2664 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank);
2665 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
2666 unsigned NewOpc = Opc == AMDGPU::G_CTLZ_ZERO_POISON
2667 ? (unsigned)AMDGPU::G_AMDGPU_FFBH_U32
2668 : Opc == AMDGPU::G_CTTZ_ZERO_POISON
2669 ? (unsigned)AMDGPU::G_AMDGPU_FFBL_B32
2670 : Opc;
2671 unsigned Idx = NewOpc == AMDGPU::G_AMDGPU_FFBH_U32;
2672 auto X = B.buildInstr(Opc: NewOpc, DstOps: {S32}, SrcOps: {SrcRegs[Idx]});
2673 auto Y = B.buildInstr(Opc: NewOpc, DstOps: {S32}, SrcOps: {SrcRegs[Idx ^ 1]});
2674 unsigned AddOpc =
2675 Opc == AMDGPU::G_CTLZ_ZERO_POISON || Opc == AMDGPU::G_CTTZ_ZERO_POISON
2676 ? AMDGPU::G_ADD
2677 : AMDGPU::G_UADDSAT;
2678 Y = B.buildInstr(Opc: AddOpc, DstOps: {S32}, SrcOps: {Y, B.buildConstant(Res: S32, Val: 32)});
2679 Register DstReg = MI.getOperand(i: 0).getReg();
2680 B.buildUMin(Dst: DstReg, Src0: X, Src1: Y);
2681 MI.eraseFromParent();
2682 return;
2683 }
2684 case AMDGPU::G_SEXT:
2685 case AMDGPU::G_ZEXT:
2686 case AMDGPU::G_ANYEXT: {
2687 Register SrcReg = MI.getOperand(i: 1).getReg();
2688 LLT SrcTy = MRI.getType(Reg: SrcReg);
2689 const bool Signed = Opc == AMDGPU::G_SEXT;
2690
2691 assert(OpdMapper.getVRegs(1).empty());
2692
2693 const RegisterBank *SrcBank =
2694 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2695
2696 Register DstReg = MI.getOperand(i: 0).getReg();
2697 LLT DstTy = MRI.getType(Reg: DstReg);
2698 if (DstTy.isScalar() &&
2699 SrcBank != &AMDGPU::SGPRRegBank &&
2700 SrcBank != &AMDGPU::VCCRegBank &&
2701 // FIXME: Should handle any type that round to s64 when irregular
2702 // breakdowns supported.
2703 DstTy.getSizeInBits() == 64 &&
2704 SrcTy.getSizeInBits() <= 32) {
2705 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2706
2707 // Extend to 32-bit, and then extend the low half.
2708 if (Signed) {
2709 // TODO: Should really be buildSExtOrCopy
2710 B.buildSExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2711 } else if (Opc == AMDGPU::G_ZEXT) {
2712 B.buildZExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2713 } else {
2714 B.buildAnyExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2715 }
2716
2717 extendLow32IntoHigh32(B, Hi32Reg: DefRegs[1], Lo32Reg: DefRegs[0], ExtOpc: Opc, RegBank: *SrcBank);
2718 MRI.setRegBank(Reg: DstReg, RegBank: *SrcBank);
2719 MI.eraseFromParent();
2720 return;
2721 }
2722
2723 if (SrcTy != LLT::scalar(SizeInBits: 1))
2724 return;
2725
2726 // It is not legal to have a legalization artifact with a VCC source. Rather
2727 // than introducing a copy, insert the select we would have to select the
2728 // copy to.
2729 if (SrcBank == &AMDGPU::VCCRegBank) {
2730 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2731
2732 const RegisterBank *DstBank = &AMDGPU::VGPRRegBank;
2733
2734 unsigned DstSize = DstTy.getSizeInBits();
2735 // 64-bit select is SGPR only
2736 const bool UseSel64 = DstSize > 32 &&
2737 SrcBank->getID() == AMDGPU::SGPRRegBankID;
2738
2739 // TODO: Should s16 select be legal?
2740 LLT SelType = UseSel64 ? LLT::scalar(SizeInBits: 64) : LLT::scalar(SizeInBits: 32);
2741 auto True = B.buildConstant(Res: SelType, Val: Signed ? -1 : 1);
2742 auto False = B.buildConstant(Res: SelType, Val: 0);
2743
2744 MRI.setRegBank(Reg: True.getReg(Idx: 0), RegBank: *DstBank);
2745 MRI.setRegBank(Reg: False.getReg(Idx: 0), RegBank: *DstBank);
2746 MRI.setRegBank(Reg: DstReg, RegBank: *DstBank);
2747
2748 if (DstSize > 32) {
2749 B.buildSelect(Res: DefRegs[0], Tst: SrcReg, Op0: True, Op1: False);
2750 extendLow32IntoHigh32(B, Hi32Reg: DefRegs[1], Lo32Reg: DefRegs[0], ExtOpc: Opc, RegBank: *SrcBank, IsBooleanSrc: true);
2751 } else if (DstSize < 32) {
2752 auto Sel = B.buildSelect(Res: SelType, Tst: SrcReg, Op0: True, Op1: False);
2753 MRI.setRegBank(Reg: Sel.getReg(Idx: 0), RegBank: *DstBank);
2754 B.buildTrunc(Res: DstReg, Op: Sel);
2755 } else {
2756 B.buildSelect(Res: DstReg, Tst: SrcReg, Op0: True, Op1: False);
2757 }
2758
2759 MI.eraseFromParent();
2760 return;
2761 }
2762
2763 break;
2764 }
2765 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
2766 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
2767
2768 assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty());
2769
2770 Register DstReg = MI.getOperand(i: 0).getReg();
2771 Register SrcReg = MI.getOperand(i: 1).getReg();
2772
2773 const LLT S32 = LLT::scalar(SizeInBits: 32);
2774 LLT DstTy = MRI.getType(Reg: DstReg);
2775 LLT SrcTy = MRI.getType(Reg: SrcReg);
2776
2777 if (foldExtractEltToCmpSelect(B, MI, OpdMapper))
2778 return;
2779
2780 const ValueMapping &DstMapping
2781 = OpdMapper.getInstrMapping().getOperandMapping(i: 0);
2782 const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank;
2783 const RegisterBank *SrcBank =
2784 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2785 const RegisterBank *IdxBank =
2786 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2787
2788 Register BaseIdxReg;
2789 unsigned ConstOffset;
2790 std::tie(args&: BaseIdxReg, args&: ConstOffset) =
2791 AMDGPU::getBaseWithConstantOffset(MRI, Reg: MI.getOperand(i: 2).getReg());
2792
2793 // See if the index is an add of a constant which will be foldable by moving
2794 // the base register of the index later if this is going to be executed in a
2795 // waterfall loop. This is essentially to reassociate the add of a constant
2796 // with the readfirstlane.
2797 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2798 ConstOffset > 0 &&
2799 ConstOffset < SrcTy.getNumElements();
2800
2801 // Move the base register. We'll re-insert the add later.
2802 if (ShouldMoveIndexIntoLoop)
2803 MI.getOperand(i: 2).setReg(BaseIdxReg);
2804
2805 // If this is a VGPR result only because the index was a VGPR result, the
2806 // actual indexing will be done on the SGPR source vector, which will
2807 // produce a scalar result. We need to copy to the VGPR result inside the
2808 // waterfall loop.
2809 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank &&
2810 SrcBank == &AMDGPU::SGPRRegBank;
2811 if (DstRegs.empty()) {
2812 applyDefaultMapping(OpdMapper);
2813
2814 executeInWaterfallLoop(B, MI, OpIndices: {2});
2815
2816 if (NeedCopyToVGPR) {
2817 // We don't want a phi for this temporary reg.
2818 Register TmpReg = MRI.createGenericVirtualRegister(Ty: DstTy);
2819 MRI.setRegBank(Reg: TmpReg, RegBank: AMDGPU::SGPRRegBank);
2820 MI.getOperand(i: 0).setReg(TmpReg);
2821 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
2822
2823 // Use a v_mov_b32 here to make the exec dependency explicit.
2824 buildVCopy(B, DstReg, SrcReg: TmpReg);
2825 }
2826
2827 // Re-insert the constant offset add inside the waterfall loop.
2828 if (ShouldMoveIndexIntoLoop)
2829 reinsertVectorIndexAdd(B, IdxUseInstr&: MI, OpIdx: 2, ConstOffset);
2830
2831 return;
2832 }
2833
2834 assert(DstTy.getSizeInBits() == 64);
2835
2836 LLT Vec32 = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
2837
2838 auto CastSrc = B.buildBitcast(Dst: Vec32, Src: SrcReg);
2839 auto One = B.buildConstant(Res: S32, Val: 1);
2840
2841 MachineBasicBlock::iterator MII = MI.getIterator();
2842
2843 // Split the vector index into 32-bit pieces. Prepare to move all of the
2844 // new instructions into a waterfall loop if necessary.
2845 //
2846 // Don't put the bitcast or constant in the loop.
2847 MachineInstrSpan Span(MII, &B.getMBB());
2848
2849 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2850 auto IdxLo = B.buildShl(Dst: S32, Src0: BaseIdxReg, Src1: One);
2851 auto IdxHi = B.buildAdd(Dst: S32, Src0: IdxLo, Src1: One);
2852
2853 auto Extract0 = B.buildExtractVectorElement(Res: DstRegs[0], Val: CastSrc, Idx: IdxLo);
2854 auto Extract1 = B.buildExtractVectorElement(Res: DstRegs[1], Val: CastSrc, Idx: IdxHi);
2855
2856 MRI.setRegBank(Reg: DstReg, RegBank: *DstBank);
2857 MRI.setRegBank(Reg: CastSrc.getReg(Idx: 0), RegBank: *SrcBank);
2858 MRI.setRegBank(Reg: One.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2859 MRI.setRegBank(Reg: IdxLo.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2860 MRI.setRegBank(Reg: IdxHi.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2861
2862 SmallSet<Register, 4> OpsToWaterfall;
2863 if (!collectWaterfallOperands(SGPROperandRegs&: OpsToWaterfall, MI, MRI, OpIndices: { 2 })) {
2864 MI.eraseFromParent();
2865 return;
2866 }
2867
2868 // Remove the original instruction to avoid potentially confusing the
2869 // waterfall loop logic.
2870 B.setInstr(*Span.begin());
2871 MI.eraseFromParent();
2872 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
2873 SGPROperandRegs&: OpsToWaterfall);
2874
2875 if (NeedCopyToVGPR) {
2876 MachineBasicBlock *LoopBB = Extract1->getParent();
2877 Register TmpReg0 = MRI.createGenericVirtualRegister(Ty: S32);
2878 Register TmpReg1 = MRI.createGenericVirtualRegister(Ty: S32);
2879 MRI.setRegBank(Reg: TmpReg0, RegBank: AMDGPU::SGPRRegBank);
2880 MRI.setRegBank(Reg: TmpReg1, RegBank: AMDGPU::SGPRRegBank);
2881
2882 Extract0->getOperand(i: 0).setReg(TmpReg0);
2883 Extract1->getOperand(i: 0).setReg(TmpReg1);
2884
2885 B.setInsertPt(MBB&: *LoopBB, II: ++Extract1->getIterator());
2886
2887 buildVCopy(B, DstReg: DstRegs[0], SrcReg: TmpReg0);
2888 buildVCopy(B, DstReg: DstRegs[1], SrcReg: TmpReg1);
2889 }
2890
2891 if (ShouldMoveIndexIntoLoop)
2892 reinsertVectorIndexAdd(B, IdxUseInstr&: *IdxLo, OpIdx: 1, ConstOffset);
2893
2894 return;
2895 }
2896 case AMDGPU::G_INSERT_VECTOR_ELT: {
2897 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(OpIdx: 2));
2898
2899 Register DstReg = MI.getOperand(i: 0).getReg();
2900 LLT VecTy = MRI.getType(Reg: DstReg);
2901
2902 assert(OpdMapper.getVRegs(0).empty());
2903 assert(OpdMapper.getVRegs(3).empty());
2904
2905 if (substituteSimpleCopyRegs(OpdMapper, OpIdx: 1))
2906 MRI.setType(VReg: MI.getOperand(i: 1).getReg(), Ty: VecTy);
2907
2908 if (foldInsertEltToCmpSelect(B, MI, OpdMapper))
2909 return;
2910
2911 const RegisterBank *IdxBank =
2912 OpdMapper.getInstrMapping().getOperandMapping(i: 3).BreakDown[0].RegBank;
2913
2914 Register SrcReg = MI.getOperand(i: 1).getReg();
2915 Register InsReg = MI.getOperand(i: 2).getReg();
2916 LLT InsTy = MRI.getType(Reg: InsReg);
2917 (void)InsTy;
2918
2919 Register BaseIdxReg;
2920 unsigned ConstOffset;
2921 std::tie(args&: BaseIdxReg, args&: ConstOffset) =
2922 AMDGPU::getBaseWithConstantOffset(MRI, Reg: MI.getOperand(i: 3).getReg());
2923
2924 // See if the index is an add of a constant which will be foldable by moving
2925 // the base register of the index later if this is going to be executed in a
2926 // waterfall loop. This is essentially to reassociate the add of a constant
2927 // with the readfirstlane.
2928 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2929 ConstOffset > 0 &&
2930 ConstOffset < VecTy.getNumElements();
2931
2932 // Move the base register. We'll re-insert the add later.
2933 if (ShouldMoveIndexIntoLoop)
2934 MI.getOperand(i: 3).setReg(BaseIdxReg);
2935
2936
2937 if (InsRegs.empty()) {
2938 executeInWaterfallLoop(B, MI, OpIndices: {3});
2939
2940 // Re-insert the constant offset add inside the waterfall loop.
2941 if (ShouldMoveIndexIntoLoop) {
2942 reinsertVectorIndexAdd(B, IdxUseInstr&: MI, OpIdx: 3, ConstOffset);
2943 }
2944
2945 return;
2946 }
2947
2948 assert(InsTy.getSizeInBits() == 64);
2949
2950 const LLT S32 = LLT::scalar(SizeInBits: 32);
2951 LLT Vec32 = LLT::fixed_vector(NumElements: 2 * VecTy.getNumElements(), ScalarSizeInBits: 32);
2952
2953 auto CastSrc = B.buildBitcast(Dst: Vec32, Src: SrcReg);
2954 auto One = B.buildConstant(Res: S32, Val: 1);
2955
2956 // Split the vector index into 32-bit pieces. Prepare to move all of the
2957 // new instructions into a waterfall loop if necessary.
2958 //
2959 // Don't put the bitcast or constant in the loop.
2960 MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB());
2961
2962 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2963 auto IdxLo = B.buildShl(Dst: S32, Src0: BaseIdxReg, Src1: One);
2964 auto IdxHi = B.buildAdd(Dst: S32, Src0: IdxLo, Src1: One);
2965
2966 auto InsLo = B.buildInsertVectorElement(Res: Vec32, Val: CastSrc, Elt: InsRegs[0], Idx: IdxLo);
2967 auto InsHi = B.buildInsertVectorElement(Res: Vec32, Val: InsLo, Elt: InsRegs[1], Idx: IdxHi);
2968
2969 const RegisterBank *DstBank =
2970 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2971 const RegisterBank *SrcBank =
2972 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2973 const RegisterBank *InsSrcBank =
2974 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2975
2976 MRI.setRegBank(Reg: InsReg, RegBank: *InsSrcBank);
2977 MRI.setRegBank(Reg: CastSrc.getReg(Idx: 0), RegBank: *SrcBank);
2978 MRI.setRegBank(Reg: InsLo.getReg(Idx: 0), RegBank: *DstBank);
2979 MRI.setRegBank(Reg: InsHi.getReg(Idx: 0), RegBank: *DstBank);
2980 MRI.setRegBank(Reg: One.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2981 MRI.setRegBank(Reg: IdxLo.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2982 MRI.setRegBank(Reg: IdxHi.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2983
2984
2985 SmallSet<Register, 4> OpsToWaterfall;
2986 if (!collectWaterfallOperands(SGPROperandRegs&: OpsToWaterfall, MI, MRI, OpIndices: { 3 })) {
2987 B.setInsertPt(MBB&: B.getMBB(), II: MI);
2988 B.buildBitcast(Dst: DstReg, Src: InsHi);
2989 MI.eraseFromParent();
2990 return;
2991 }
2992
2993 B.setInstr(*Span.begin());
2994 MI.eraseFromParent();
2995
2996 // Figure out the point after the waterfall loop before mangling the control
2997 // flow.
2998 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
2999 SGPROperandRegs&: OpsToWaterfall);
3000
3001 // The insertion point is now right after the original instruction.
3002 //
3003 // Keep the bitcast to the original vector type out of the loop. Doing this
3004 // saved an extra phi we don't need inside the loop.
3005 B.buildBitcast(Dst: DstReg, Src: InsHi);
3006
3007 // Re-insert the constant offset add inside the waterfall loop.
3008 if (ShouldMoveIndexIntoLoop)
3009 reinsertVectorIndexAdd(B, IdxUseInstr&: *IdxLo, OpIdx: 1, ConstOffset);
3010
3011 return;
3012 }
3013 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
3014 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
3015 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
3016 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
3017 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
3018 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
3019 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
3020 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
3021 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
3022 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
3023 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
3024 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
3025 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
3026 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
3027 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
3028 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
3029 case AMDGPU::G_AMDGPU_BUFFER_STORE:
3030 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
3031 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
3032 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
3033 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16:
3034 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
3035 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: {
3036 applyDefaultMapping(OpdMapper);
3037 executeInWaterfallLoop(B, MI, OpIndices: {1, 4});
3038 return;
3039 }
3040 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
3041 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
3042 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
3043 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
3044 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
3045 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
3046 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
3047 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
3048 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
3049 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
3050 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
3051 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
3052 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
3053 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
3054 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
3055 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
3056 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
3057 applyDefaultMapping(OpdMapper);
3058 executeInWaterfallLoop(B, MI, OpIndices: {2, 5});
3059 return;
3060 }
3061 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
3062 applyDefaultMapping(OpdMapper);
3063 executeInWaterfallLoop(B, MI, OpIndices: {3, 6});
3064 return;
3065 }
3066 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
3067 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
3068 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
3069 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
3070 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
3071 applyMappingSBufferLoad(B, OpdMapper);
3072 return;
3073 }
3074 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
3075 constrainOpWithReadfirstlane(B, MI, OpIdx: 0);
3076 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3077 return;
3078 case AMDGPU::G_INTRINSIC:
3079 case AMDGPU::G_INTRINSIC_CONVERGENT: {
3080 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3081 case Intrinsic::amdgcn_readlane: {
3082 substituteSimpleCopyRegs(OpdMapper, OpIdx: 2);
3083
3084 assert(OpdMapper.getVRegs(0).empty());
3085 assert(OpdMapper.getVRegs(3).empty());
3086
3087 // Make sure the index is an SGPR. It doesn't make sense to run this in a
3088 // waterfall loop, so assume it's a uniform value.
3089 constrainOpWithReadfirstlane(B, MI, OpIdx: 3); // Index
3090 return;
3091 }
3092 case Intrinsic::amdgcn_writelane: {
3093 assert(OpdMapper.getVRegs(0).empty());
3094 assert(OpdMapper.getVRegs(2).empty());
3095 assert(OpdMapper.getVRegs(3).empty());
3096
3097 substituteSimpleCopyRegs(OpdMapper, OpIdx: 4); // VGPR input val
3098 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // Source value
3099 constrainOpWithReadfirstlane(B, MI, OpIdx: 3); // Index
3100 return;
3101 }
3102 case Intrinsic::amdgcn_interp_p1:
3103 case Intrinsic::amdgcn_interp_p2:
3104 case Intrinsic::amdgcn_interp_mov:
3105 case Intrinsic::amdgcn_interp_p1_f16:
3106 case Intrinsic::amdgcn_interp_p2_f16:
3107 case Intrinsic::amdgcn_lds_param_load: {
3108 applyDefaultMapping(OpdMapper);
3109
3110 // Readlane for m0 value, which is always the last operand.
3111 // FIXME: Should this be a waterfall loop instead?
3112 constrainOpWithReadfirstlane(B, MI, OpIdx: MI.getNumOperands() - 1); // Index
3113 return;
3114 }
3115 case Intrinsic::amdgcn_interp_inreg_p10:
3116 case Intrinsic::amdgcn_interp_inreg_p2:
3117 case Intrinsic::amdgcn_interp_inreg_p10_f16:
3118 case Intrinsic::amdgcn_interp_inreg_p2_f16:
3119 case Intrinsic::amdgcn_interp_p10_rtz_f16:
3120 case Intrinsic::amdgcn_interp_p2_rtz_f16:
3121 case Intrinsic::amdgcn_permlane16_swap:
3122 case Intrinsic::amdgcn_permlane32_swap:
3123 applyDefaultMapping(OpdMapper);
3124 return;
3125 case Intrinsic::amdgcn_permlane16:
3126 case Intrinsic::amdgcn_permlanex16: {
3127 // Doing a waterfall loop over these wouldn't make any sense.
3128 substituteSimpleCopyRegs(OpdMapper, OpIdx: 2);
3129 substituteSimpleCopyRegs(OpdMapper, OpIdx: 3);
3130 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3131 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3132 return;
3133 }
3134 case Intrinsic::amdgcn_permlane_bcast:
3135 case Intrinsic::amdgcn_permlane_up:
3136 case Intrinsic::amdgcn_permlane_down:
3137 case Intrinsic::amdgcn_permlane_xor:
3138 // Doing a waterfall loop over these wouldn't make any sense.
3139 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3140 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3141 return;
3142 case Intrinsic::amdgcn_permlane_idx_gen: {
3143 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3144 return;
3145 }
3146 case Intrinsic::amdgcn_sbfe:
3147 applyMappingBFE(B, OpdMapper, Signed: true);
3148 return;
3149 case Intrinsic::amdgcn_ubfe:
3150 applyMappingBFE(B, OpdMapper, Signed: false);
3151 return;
3152 case Intrinsic::amdgcn_inverse_ballot:
3153 case Intrinsic::amdgcn_s_bitreplicate:
3154 case Intrinsic::amdgcn_s_quadmask:
3155 case Intrinsic::amdgcn_s_wqm:
3156 applyDefaultMapping(OpdMapper);
3157 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // Mask
3158 return;
3159 case Intrinsic::amdgcn_ballot:
3160 // Use default handling and insert copy to vcc source.
3161 break;
3162 }
3163 break;
3164 }
3165 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3166 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
3167 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
3168 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
3169 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
3170 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3171 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
3172 assert(RSrcIntrin && RSrcIntrin->IsImage);
3173 // Non-images can have complications from operands that allow both SGPR
3174 // and VGPR. For now it's too complicated to figure out the final opcode
3175 // to derive the register bank from the MCInstrDesc.
3176 applyMappingImage(B, MI, OpdMapper, RsrcIdx: RSrcIntrin->RsrcArg);
3177 return;
3178 }
3179 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
3180 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
3181 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
3182 bool IsDualOrBVH8 =
3183 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
3184 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
3185 unsigned NumMods = IsDualOrBVH8 ? 0 : 1; // Has A16 modifier
3186 unsigned LastRegOpIdx = MI.getNumExplicitOperands() - 1 - NumMods;
3187 applyDefaultMapping(OpdMapper);
3188 executeInWaterfallLoop(B, MI, OpIndices: {LastRegOpIdx});
3189 return;
3190 }
3191 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
3192 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
3193 auto IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3194 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
3195 AMDGPU::lookupBufferIntrinsic(Intr: IntrID)) {
3196 // The legalizer turns all other buffer intrinsics into
3197 // G_AMDGPU_BUFFER_* and similar instructions.
3198 assert(BufInfo->Kind == AMDGPU::BufferKindLoadToLDS &&
3199 "buffer intrinsic should have been legalized");
3200 auto OpIdx = [&](unsigned Arg) {
3201 return AMDGPU::getMIROperandIdx(MI, Arg);
3202 };
3203 applyDefaultMapping(OpdMapper);
3204 constrainOpWithReadfirstlane(B, MI, OpIdx: OpIdx(BufInfo->RsrcArg));
3205 constrainOpWithReadfirstlane(B, MI,
3206 OpIdx: OpIdx(*BufInfo->getLDSPtrArg())); // M0
3207 constrainOpWithReadfirstlane(B, MI, OpIdx: OpIdx(*BufInfo->getSOffsetArg()));
3208 return;
3209 }
3210 switch (IntrID) {
3211 case Intrinsic::amdgcn_ds_ordered_add:
3212 case Intrinsic::amdgcn_ds_ordered_swap: {
3213 // This is only allowed to execute with 1 lane, so readfirstlane is safe.
3214 assert(OpdMapper.getVRegs(0).empty());
3215 substituteSimpleCopyRegs(OpdMapper, OpIdx: 3);
3216 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3217 return;
3218 }
3219 case Intrinsic::amdgcn_ds_gws_init:
3220 case Intrinsic::amdgcn_ds_gws_barrier:
3221 case Intrinsic::amdgcn_ds_gws_sema_br: {
3222 // Only the first lane is executes, so readfirstlane is safe.
3223 substituteSimpleCopyRegs(OpdMapper, OpIdx: 1);
3224 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3225 return;
3226 }
3227 case Intrinsic::amdgcn_ds_gws_sema_v:
3228 case Intrinsic::amdgcn_ds_gws_sema_p:
3229 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
3230 // Only the first lane is executes, so readfirstlane is safe.
3231 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // M0
3232 return;
3233 }
3234 case Intrinsic::amdgcn_ds_append:
3235 case Intrinsic::amdgcn_ds_consume: {
3236 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3237 return;
3238 }
3239 case Intrinsic::amdgcn_s_alloc_vgpr:
3240 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3241 return;
3242 case Intrinsic::amdgcn_s_sendmsg:
3243 case Intrinsic::amdgcn_s_sendmsghalt: {
3244 // FIXME: Should this use a waterfall loop?
3245 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3246 return;
3247 }
3248 case Intrinsic::amdgcn_s_setreg: {
3249 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3250 return;
3251 }
3252 case Intrinsic::amdgcn_s_ttracedata:
3253 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // M0
3254 return;
3255 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
3256 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
3257 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
3258 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
3259 applyDefaultMapping(OpdMapper);
3260 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3261 return;
3262 }
3263 case Intrinsic::amdgcn_load_to_lds:
3264 case Intrinsic::amdgcn_load_async_to_lds:
3265 case Intrinsic::amdgcn_global_load_lds:
3266 case Intrinsic::amdgcn_global_load_async_lds: {
3267 applyDefaultMapping(OpdMapper);
3268 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3269 return;
3270 }
3271 case Intrinsic::amdgcn_lds_direct_load: {
3272 applyDefaultMapping(OpdMapper);
3273 // Readlane for m0 value, which is always the last operand.
3274 constrainOpWithReadfirstlane(B, MI, OpIdx: MI.getNumOperands() - 1); // Index
3275 return;
3276 }
3277 case Intrinsic::amdgcn_exp_row:
3278 applyDefaultMapping(OpdMapper);
3279 constrainOpWithReadfirstlane(B, MI, OpIdx: 8); // M0
3280 return;
3281 case Intrinsic::amdgcn_cluster_load_b32:
3282 case Intrinsic::amdgcn_cluster_load_b64:
3283 case Intrinsic::amdgcn_cluster_load_b128: {
3284 applyDefaultMapping(OpdMapper);
3285 constrainOpWithReadfirstlane(B, MI, OpIdx: 4); // M0
3286 return;
3287 }
3288 case Intrinsic::amdgcn_s_sleep_var:
3289 assert(OpdMapper.getVRegs(1).empty());
3290 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3291 return;
3292 case Intrinsic::amdgcn_s_barrier_join:
3293 case Intrinsic::amdgcn_s_wakeup_barrier:
3294 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3295 return;
3296 case Intrinsic::amdgcn_s_barrier_init:
3297 case Intrinsic::amdgcn_s_barrier_signal_var:
3298 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3299 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3300 return;
3301 case Intrinsic::amdgcn_s_get_barrier_state:
3302 case Intrinsic::amdgcn_s_get_named_barrier_state: {
3303 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3304 return;
3305 }
3306 case Intrinsic::amdgcn_s_prefetch_data:
3307 case Intrinsic::amdgcn_s_prefetch_inst: {
3308 Register PtrReg = MI.getOperand(i: 1).getReg();
3309 unsigned AS = MRI.getType(Reg: PtrReg).getAddressSpace();
3310 if (AMDGPU::isFlatGlobalAddrSpace(AS)) {
3311 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3312 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3313 } else
3314 MI.eraseFromParent();
3315 return;
3316 }
3317 case Intrinsic::amdgcn_tensor_load_to_lds:
3318 case Intrinsic::amdgcn_tensor_store_from_lds: {
3319 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3320 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3321 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3322 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3323 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3324 return;
3325 }
3326 default: {
3327 if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3328 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
3329 // Non-images can have complications from operands that allow both SGPR
3330 // and VGPR. For now it's too complicated to figure out the final opcode
3331 // to derive the register bank from the MCInstrDesc.
3332 if (RSrcIntrin->IsImage) {
3333 applyMappingImage(B, MI, OpdMapper, RsrcIdx: RSrcIntrin->RsrcArg);
3334 return;
3335 }
3336 }
3337
3338 break;
3339 }
3340 }
3341 break;
3342 }
3343 case AMDGPU::G_SI_CALL: {
3344 // Use a set to avoid extra readfirstlanes in the case where multiple
3345 // operands are the same register.
3346 SmallSet<Register, 4> SGPROperandRegs;
3347
3348 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI, OpIndices: {1}))
3349 break;
3350
3351 // Move all copies to physical SGPRs that are used by the call instruction
3352 // into the loop block. Start searching for these copies until the
3353 // ADJCALLSTACKUP.
3354 unsigned FrameSetupOpcode = AMDGPU::ADJCALLSTACKUP;
3355 unsigned FrameDestroyOpcode = AMDGPU::ADJCALLSTACKDOWN;
3356
3357 // Move all non-copies before the copies, so that a complete range can be
3358 // moved into the waterfall loop.
3359 SmallVector<MachineInstr *, 4> NonCopyInstrs;
3360 // Count of NonCopyInstrs found until the current LastCopy.
3361 unsigned NonCopyInstrsLen = 0;
3362 MachineBasicBlock::iterator Start(&MI);
3363 MachineBasicBlock::iterator LastCopy = Start;
3364 MachineBasicBlock *MBB = MI.getParent();
3365 const SIMachineFunctionInfo *Info =
3366 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
3367 while (Start->getOpcode() != FrameSetupOpcode) {
3368 --Start;
3369 bool IsCopy = false;
3370 if (Start->getOpcode() == AMDGPU::COPY) {
3371 auto &Dst = Start->getOperand(i: 0);
3372 if (Dst.isReg()) {
3373 Register Reg = Dst.getReg();
3374 if (Reg.isPhysical() && MI.readsRegister(Reg, TRI)) {
3375 IsCopy = true;
3376 } else {
3377 // Also move the copy from the scratch rsrc descriptor into the loop
3378 // to allow it to be optimized away.
3379 auto &Src = Start->getOperand(i: 1);
3380 if (Src.isReg()) {
3381 Reg = Src.getReg();
3382 IsCopy = Info->getScratchRSrcReg() == Reg;
3383 }
3384 }
3385 }
3386 }
3387
3388 if (IsCopy) {
3389 LastCopy = Start;
3390 NonCopyInstrsLen = NonCopyInstrs.size();
3391 } else {
3392 NonCopyInstrs.push_back(Elt: &*Start);
3393 }
3394 }
3395 NonCopyInstrs.resize(N: NonCopyInstrsLen);
3396
3397 for (auto *NonCopy : reverse(C&: NonCopyInstrs)) {
3398 MBB->splice(Where: LastCopy, Other: MBB, From: NonCopy->getIterator());
3399 }
3400 Start = LastCopy;
3401
3402 // Do the same for copies after the loop
3403 NonCopyInstrs.clear();
3404 NonCopyInstrsLen = 0;
3405 MachineBasicBlock::iterator End(&MI);
3406 LastCopy = End;
3407 while (End->getOpcode() != FrameDestroyOpcode) {
3408 ++End;
3409 bool IsCopy = false;
3410 if (End->getOpcode() == AMDGPU::COPY) {
3411 auto &Src = End->getOperand(i: 1);
3412 if (Src.isReg()) {
3413 Register Reg = Src.getReg();
3414 IsCopy = Reg.isPhysical() && MI.modifiesRegister(Reg, TRI);
3415 }
3416 }
3417
3418 if (IsCopy) {
3419 LastCopy = End;
3420 NonCopyInstrsLen = NonCopyInstrs.size();
3421 } else {
3422 NonCopyInstrs.push_back(Elt: &*End);
3423 }
3424 }
3425 NonCopyInstrs.resize(N: NonCopyInstrsLen);
3426
3427 End = LastCopy;
3428 ++LastCopy;
3429 for (auto *NonCopy : reverse(C&: NonCopyInstrs)) {
3430 MBB->splice(Where: LastCopy, Other: MBB, From: NonCopy->getIterator());
3431 }
3432
3433 ++End;
3434 B.setInsertPt(MBB&: B.getMBB(), II: Start);
3435 executeInWaterfallLoop(B, Range: make_range(x: Start, y: End), SGPROperandRegs);
3436 break;
3437 }
3438 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
3439 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR:
3440 case AMDGPU::G_LOAD:
3441 case AMDGPU::G_ZEXTLOAD:
3442 case AMDGPU::G_SEXTLOAD: {
3443 if (applyMappingLoad(B, OpdMapper, MI))
3444 return;
3445 break;
3446 }
3447 case AMDGPU::G_DYN_STACKALLOC:
3448 applyMappingDynStackAlloc(B, OpdMapper, MI);
3449 return;
3450 case AMDGPU::G_STACKRESTORE: {
3451 applyDefaultMapping(OpdMapper);
3452 constrainOpWithReadfirstlane(B, MI, OpIdx: 0);
3453 return;
3454 }
3455 case AMDGPU::G_SBFX:
3456 applyMappingBFE(B, OpdMapper, /*Signed*/ true);
3457 return;
3458 case AMDGPU::G_UBFX:
3459 applyMappingBFE(B, OpdMapper, /*Signed*/ false);
3460 return;
3461 case AMDGPU::G_AMDGPU_MAD_U64_U32:
3462 case AMDGPU::G_AMDGPU_MAD_I64_I32:
3463 applyMappingMAD_64_32(B, OpdMapper);
3464 return;
3465 case AMDGPU::G_PREFETCH: {
3466 if (!Subtarget.hasSafeSmemPrefetch() && !Subtarget.hasVmemPrefInsts()) {
3467 MI.eraseFromParent();
3468 return;
3469 }
3470 Register PtrReg = MI.getOperand(i: 0).getReg();
3471 unsigned PtrBank = getRegBankID(Reg: PtrReg, MRI, Default: AMDGPU::SGPRRegBankID);
3472 if (PtrBank == AMDGPU::VGPRRegBankID &&
3473 (!Subtarget.hasVmemPrefInsts() || !MI.getOperand(i: 3).getImm())) {
3474 // Cannot do I$ prefetch with divergent pointer.
3475 MI.eraseFromParent();
3476 return;
3477 }
3478 unsigned AS = MRI.getType(Reg: PtrReg).getAddressSpace();
3479 if ((!AMDGPU::isFlatGlobalAddrSpace(AS) &&
3480 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
3481 (!Subtarget.hasSafeSmemPrefetch() &&
3482 (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
3483 !MI.getOperand(i: 3).getImm() /* I$ prefetch */))) {
3484 MI.eraseFromParent();
3485 return;
3486 }
3487 applyDefaultMapping(OpdMapper);
3488 return;
3489 }
3490 default:
3491 break;
3492 }
3493
3494 return applyDefaultMapping(OpdMapper);
3495}
3496
3497// vgpr, sgpr -> vgpr
3498// vgpr, agpr -> vgpr
3499// agpr, agpr -> agpr
3500// agpr, sgpr -> vgpr
3501static unsigned regBankUnion(unsigned RB0, unsigned RB1) {
3502 if (RB0 == AMDGPU::InvalidRegBankID)
3503 return RB1;
3504 if (RB1 == AMDGPU::InvalidRegBankID)
3505 return RB0;
3506
3507 if (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID)
3508 return AMDGPU::SGPRRegBankID;
3509
3510 if (RB0 == AMDGPU::AGPRRegBankID && RB1 == AMDGPU::AGPRRegBankID)
3511 return AMDGPU::AGPRRegBankID;
3512
3513 return AMDGPU::VGPRRegBankID;
3514}
3515
3516static unsigned regBankBoolUnion(unsigned RB0, unsigned RB1) {
3517 if (RB0 == AMDGPU::InvalidRegBankID)
3518 return RB1;
3519 if (RB1 == AMDGPU::InvalidRegBankID)
3520 return RB0;
3521
3522 // vcc, vcc -> vcc
3523 // vcc, sgpr -> vcc
3524 // vcc, vgpr -> vcc
3525 if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID)
3526 return AMDGPU::VCCRegBankID;
3527
3528 // vcc, vgpr -> vgpr
3529 return regBankUnion(RB0, RB1);
3530}
3531
3532unsigned AMDGPURegisterBankInfo::getMappingType(const MachineRegisterInfo &MRI,
3533 const MachineInstr &MI) const {
3534 unsigned RegBank = AMDGPU::InvalidRegBankID;
3535
3536 for (const MachineOperand &MO : MI.operands()) {
3537 if (!MO.isReg())
3538 continue;
3539 Register Reg = MO.getReg();
3540 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
3541 RegBank = regBankUnion(RB0: RegBank, RB1: Bank->getID());
3542 if (RegBank == AMDGPU::VGPRRegBankID)
3543 break;
3544 }
3545 }
3546
3547 return RegBank;
3548}
3549
3550bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const {
3551 const MachineFunction &MF = *MI.getMF();
3552 const MachineRegisterInfo &MRI = MF.getRegInfo();
3553 for (const MachineOperand &MO : MI.operands()) {
3554 if (!MO.isReg())
3555 continue;
3556 Register Reg = MO.getReg();
3557 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
3558 if (Bank->getID() != AMDGPU::SGPRRegBankID)
3559 return false;
3560 }
3561 }
3562 return true;
3563}
3564
3565const RegisterBankInfo::InstructionMapping &
3566AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const {
3567 const MachineFunction &MF = *MI.getMF();
3568 const MachineRegisterInfo &MRI = MF.getRegInfo();
3569 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3570
3571 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3572 const MachineOperand &SrcOp = MI.getOperand(i);
3573 if (!SrcOp.isReg())
3574 continue;
3575
3576 unsigned Size = TRI->getRegSizeInBits(Reg: SrcOp.getReg(), MRI);
3577 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
3578 }
3579 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3580 NumOperands: MI.getNumOperands());
3581}
3582
3583const RegisterBankInfo::InstructionMapping &
3584AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
3585 const MachineFunction &MF = *MI.getMF();
3586 const MachineRegisterInfo &MRI = MF.getRegInfo();
3587 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3588
3589 // Even though we technically could use SGPRs, this would require knowledge of
3590 // the constant bus restriction. Force all sources to VGPR (except for VCC).
3591 //
3592 // TODO: Unary ops are trivially OK, so accept SGPRs?
3593 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3594 const MachineOperand &Src = MI.getOperand(i);
3595 if (!Src.isReg())
3596 continue;
3597
3598 unsigned Size = TRI->getRegSizeInBits(Reg: Src.getReg(), MRI);
3599 unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
3600 OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
3601 }
3602
3603 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3604 NumOperands: MI.getNumOperands());
3605}
3606
3607const RegisterBankInfo::InstructionMapping &
3608AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
3609 const MachineFunction &MF = *MI.getMF();
3610 const MachineRegisterInfo &MRI = MF.getRegInfo();
3611 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3612
3613 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
3614 const MachineOperand &Op = MI.getOperand(i: I);
3615 if (!Op.isReg())
3616 continue;
3617
3618 unsigned Size = TRI->getRegSizeInBits(Reg: Op.getReg(), MRI);
3619 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3620 }
3621
3622 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3623 NumOperands: MI.getNumOperands());
3624}
3625
3626const RegisterBankInfo::InstructionMapping &
3627AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
3628 const MachineInstr &MI,
3629 int RsrcIdx) const {
3630 // The reported argument index is relative to the IR intrinsic call arguments,
3631 // so we need to shift by the number of defs and the intrinsic ID.
3632 RsrcIdx += MI.getNumExplicitDefs() + 1;
3633
3634 const int NumOps = MI.getNumOperands();
3635 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps);
3636
3637 // TODO: Should packed/unpacked D16 difference be reported here as part of
3638 // the value mapping?
3639 for (int I = 0; I != NumOps; ++I) {
3640 if (!MI.getOperand(i: I).isReg())
3641 continue;
3642
3643 Register OpReg = MI.getOperand(i: I).getReg();
3644 // We replace some dead address operands with $noreg
3645 if (!OpReg)
3646 continue;
3647
3648 unsigned Size = TRI->getRegSizeInBits(Reg: OpReg, MRI);
3649
3650 // FIXME: Probably need a new intrinsic register bank searchable table to
3651 // handle arbitrary intrinsics easily.
3652 //
3653 // If this has a sampler, it immediately follows rsrc.
3654 const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1;
3655
3656 if (MustBeSGPR) {
3657 // If this must be an SGPR, so we must report whatever it is as legal.
3658 unsigned NewBank = getRegBankID(Reg: OpReg, MRI, Default: AMDGPU::SGPRRegBankID);
3659 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: NewBank, Size);
3660 } else {
3661 // Some operands must be VGPR, and these are easy to copy to.
3662 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3663 }
3664 }
3665
3666 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping), NumOperands: NumOps);
3667}
3668
3669/// Return the mapping for a pointer argument.
3670const RegisterBankInfo::ValueMapping *
3671AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI,
3672 Register PtrReg) const {
3673 LLT PtrTy = MRI.getType(Reg: PtrReg);
3674 unsigned Size = PtrTy.getSizeInBits();
3675 if (Subtarget.useFlatForGlobal() ||
3676 !AMDGPU::isFlatGlobalAddrSpace(AS: PtrTy.getAddressSpace()))
3677 return AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3678
3679 // If we're using MUBUF instructions for global memory, an SGPR base register
3680 // is possible. Otherwise this needs to be a VGPR.
3681 const RegisterBank *PtrBank = getRegBank(Reg: PtrReg, MRI, TRI: *TRI);
3682 return AMDGPU::getValueMapping(BankID: PtrBank->getID(), Size);
3683}
3684
3685const RegisterBankInfo::InstructionMapping &
3686AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
3687
3688 const MachineFunction &MF = *MI.getMF();
3689 const MachineRegisterInfo &MRI = MF.getRegInfo();
3690 SmallVector<const ValueMapping*, 2> OpdsMapping(2);
3691 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
3692 Register PtrReg = MI.getOperand(i: 1).getReg();
3693 LLT PtrTy = MRI.getType(Reg: PtrReg);
3694 unsigned AS = PtrTy.getAddressSpace();
3695 unsigned PtrSize = PtrTy.getSizeInBits();
3696
3697 const ValueMapping *ValMapping;
3698 const ValueMapping *PtrMapping;
3699
3700 const RegisterBank *PtrBank = getRegBank(Reg: PtrReg, MRI, TRI: *TRI);
3701
3702 if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) {
3703 if (isScalarLoadLegal(MI)) {
3704 // We have a uniform instruction so we want to use an SMRD load
3705 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
3706 PtrMapping = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: PtrSize);
3707 } else {
3708 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3709
3710 // If we're using MUBUF instructions for global memory, an SGPR base
3711 // register is possible. Otherwise this needs to be a VGPR.
3712 unsigned PtrBankID = Subtarget.useFlatForGlobal() ?
3713 AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID;
3714
3715 PtrMapping = AMDGPU::getValueMapping(BankID: PtrBankID, Size: PtrSize);
3716 }
3717 } else {
3718 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3719 PtrMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize);
3720 }
3721
3722 OpdsMapping[0] = ValMapping;
3723 OpdsMapping[1] = PtrMapping;
3724 const RegisterBankInfo::InstructionMapping &Mapping = getInstructionMapping(
3725 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping), NumOperands: MI.getNumOperands());
3726 return Mapping;
3727
3728 // FIXME: Do we want to add a mapping for FLAT load, or should we just
3729 // handle that during instruction selection?
3730}
3731
3732unsigned
3733AMDGPURegisterBankInfo::getRegBankID(Register Reg,
3734 const MachineRegisterInfo &MRI,
3735 unsigned Default) const {
3736 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
3737 return Bank ? Bank->getID() : Default;
3738}
3739
3740const RegisterBankInfo::ValueMapping *
3741AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
3742 const MachineRegisterInfo &MRI,
3743 const TargetRegisterInfo &TRI) const {
3744 // Lie and claim anything is legal, even though this needs to be an SGPR
3745 // applyMapping will have to deal with it as a waterfall loop.
3746 unsigned Bank = getRegBankID(Reg, MRI, Default: AMDGPU::SGPRRegBankID);
3747 unsigned Size = TRI.getRegSizeInBits(Reg, MRI);
3748 return AMDGPU::getValueMapping(BankID: Bank, Size);
3749}
3750
3751const RegisterBankInfo::ValueMapping *
3752AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
3753 const MachineRegisterInfo &MRI,
3754 const TargetRegisterInfo &TRI) const {
3755 unsigned Size = TRI.getRegSizeInBits(Reg, MRI);
3756 return AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3757}
3758
3759const RegisterBankInfo::ValueMapping *
3760AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg,
3761 const MachineRegisterInfo &MRI,
3762 const TargetRegisterInfo &TRI) const {
3763 unsigned Size = TRI.getRegSizeInBits(Reg, MRI);
3764 return AMDGPU::getValueMapping(BankID: AMDGPU::AGPRRegBankID, Size);
3765}
3766
3767///
3768/// This function must return a legal mapping, because
3769/// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called
3770/// in RegBankSelect::Mode::Fast. Any mapping that would cause a
3771/// VGPR to SGPR generated is illegal.
3772///
3773// Operands that must be SGPRs must accept potentially divergent VGPRs as
3774// legal. These will be dealt with in applyMappingImpl.
3775//
3776const RegisterBankInfo::InstructionMapping &
3777AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
3778 const MachineFunction &MF = *MI.getMF();
3779 const MachineRegisterInfo &MRI = MF.getRegInfo();
3780
3781 if (MI.isCopy() || MI.getOpcode() == AMDGPU::G_FREEZE) {
3782 Register DstReg = MI.getOperand(i: 0).getReg();
3783 Register SrcReg = MI.getOperand(i: 1).getReg();
3784
3785 // The default logic bothers to analyze impossible alternative mappings. We
3786 // want the most straightforward mapping, so just directly handle this.
3787 const RegisterBank *DstBank = getRegBank(Reg: DstReg, MRI, TRI: *TRI);
3788 const RegisterBank *SrcBank = getRegBank(Reg: SrcReg, MRI, TRI: *TRI);
3789
3790 // For COPY between a physical reg and an s1, there is no type associated so
3791 // we need to take the virtual register's type as a hint on how to interpret
3792 // s1 values.
3793 unsigned Size;
3794 if (!SrcReg.isVirtual() && !DstBank &&
3795 MRI.getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1)) {
3796 DstBank = &AMDGPU::VCCRegBank;
3797 Size = 1;
3798 } else if (!DstReg.isVirtual() && MRI.getType(Reg: SrcReg) == LLT::scalar(SizeInBits: 1)) {
3799 DstBank = &AMDGPU::VCCRegBank;
3800 Size = 1;
3801 } else {
3802 Size = TRI->getRegSizeInBits(Reg: DstReg, MRI);
3803 }
3804
3805 if (!DstBank)
3806 DstBank = SrcBank;
3807 else if (!SrcBank)
3808 SrcBank = DstBank;
3809
3810 if (MI.getOpcode() != AMDGPU::G_FREEZE &&
3811 cannotCopy(Dst: *DstBank, Src: *SrcBank, Size: TypeSize::getFixed(ExactSize: Size)))
3812 return getInvalidInstructionMapping();
3813
3814 const ValueMapping &ValMap = getValueMapping(StartIdx: 0, Length: Size, RegBank: *DstBank);
3815 unsigned OpdsMappingSize = MI.isCopy() ? 1 : 2;
3816 SmallVector<const ValueMapping *, 1> OpdsMapping(OpdsMappingSize);
3817 OpdsMapping[0] = &ValMap;
3818 if (MI.getOpcode() == AMDGPU::G_FREEZE)
3819 OpdsMapping[1] = &ValMap;
3820
3821 return getInstructionMapping(
3822 ID: 1, /*Cost*/ 1,
3823 /*OperandsMapping*/ getOperandsMapping(OpdsMapping), NumOperands: OpdsMappingSize);
3824 }
3825
3826 if (MI.isRegSequence()) {
3827 // If any input is a VGPR, the result must be a VGPR. The default handling
3828 // assumes any copy between banks is legal.
3829 unsigned BankID = AMDGPU::SGPRRegBankID;
3830
3831 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3832 auto OpBank = getRegBankID(Reg: MI.getOperand(i: I).getReg(), MRI);
3833 // It doesn't make sense to use vcc or scc banks here, so just ignore
3834 // them.
3835 if (OpBank != AMDGPU::SGPRRegBankID) {
3836 BankID = AMDGPU::VGPRRegBankID;
3837 break;
3838 }
3839 }
3840 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
3841
3842 const ValueMapping &ValMap = getValueMapping(StartIdx: 0, Length: Size, RegBank: getRegBank(ID: BankID));
3843 return getInstructionMapping(
3844 ID: 1, /*Cost*/ 1,
3845 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: {&ValMap}), NumOperands: 1);
3846 }
3847
3848 // The default handling is broken and doesn't handle illegal SGPR->VGPR copies
3849 // properly.
3850 //
3851 // TODO: There are additional exec masking dependencies to analyze.
3852 if (auto *PHI = dyn_cast<GPhi>(Val: &MI)) {
3853 unsigned ResultBank = AMDGPU::InvalidRegBankID;
3854 Register DstReg = PHI->getReg(Idx: 0);
3855
3856 // Sometimes the result may have already been assigned a bank.
3857 if (const RegisterBank *DstBank = getRegBank(Reg: DstReg, MRI, TRI: *TRI))
3858 ResultBank = DstBank->getID();
3859
3860 for (unsigned I = 0; I < PHI->getNumIncomingValues(); ++I) {
3861 Register Reg = PHI->getIncomingValue(I);
3862 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
3863
3864 // FIXME: Assuming VGPR for any undetermined inputs.
3865 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) {
3866 ResultBank = AMDGPU::VGPRRegBankID;
3867 break;
3868 }
3869
3870 // FIXME: Need to promote SGPR case to s32
3871 unsigned OpBank = Bank->getID();
3872 ResultBank = regBankBoolUnion(RB0: ResultBank, RB1: OpBank);
3873 }
3874
3875 assert(ResultBank != AMDGPU::InvalidRegBankID);
3876
3877 unsigned Size = MRI.getType(Reg: DstReg).getSizeInBits();
3878
3879 const ValueMapping &ValMap =
3880 getValueMapping(StartIdx: 0, Length: Size, RegBank: getRegBank(ID: ResultBank));
3881 return getInstructionMapping(
3882 ID: 1, /*Cost*/ 1,
3883 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: {&ValMap}), NumOperands: 1);
3884 }
3885
3886 const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
3887 if (Mapping.isValid())
3888 return Mapping;
3889
3890 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3891
3892 switch (MI.getOpcode()) {
3893 default:
3894 return getInvalidInstructionMapping();
3895
3896 case AMDGPU::G_AND:
3897 case AMDGPU::G_OR:
3898 case AMDGPU::G_XOR:
3899 case AMDGPU::G_MUL: {
3900 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
3901 if (Size == 1) {
3902 const RegisterBank *DstBank
3903 = getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3904
3905 unsigned TargetBankID = AMDGPU::InvalidRegBankID;
3906 unsigned BankLHS = AMDGPU::InvalidRegBankID;
3907 unsigned BankRHS = AMDGPU::InvalidRegBankID;
3908 if (DstBank) {
3909 TargetBankID = DstBank->getID();
3910 if (DstBank == &AMDGPU::VCCRegBank) {
3911 TargetBankID = AMDGPU::VCCRegBankID;
3912 BankLHS = AMDGPU::VCCRegBankID;
3913 BankRHS = AMDGPU::VCCRegBankID;
3914 } else {
3915 BankLHS = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
3916 Default: AMDGPU::SGPRRegBankID);
3917 BankRHS = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
3918 Default: AMDGPU::SGPRRegBankID);
3919 }
3920 } else {
3921 BankLHS = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
3922 Default: AMDGPU::VCCRegBankID);
3923 BankRHS = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
3924 Default: AMDGPU::VCCRegBankID);
3925
3926 // Both inputs should be true booleans to produce a boolean result.
3927 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) {
3928 TargetBankID = AMDGPU::VGPRRegBankID;
3929 } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) {
3930 TargetBankID = AMDGPU::VCCRegBankID;
3931 BankLHS = AMDGPU::VCCRegBankID;
3932 BankRHS = AMDGPU::VCCRegBankID;
3933 } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) {
3934 TargetBankID = AMDGPU::SGPRRegBankID;
3935 }
3936 }
3937
3938 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: TargetBankID, Size);
3939 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: BankLHS, Size);
3940 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: BankRHS, Size);
3941 break;
3942 }
3943
3944 if (Size == 64) {
3945
3946 if (isSALUMapping(MI)) {
3947 OpdsMapping[0] = getValueMappingSGPR64Only(BankID: AMDGPU::SGPRRegBankID, Size);
3948 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
3949 } else {
3950 if (MI.getOpcode() == AMDGPU::G_MUL && Subtarget.useVMulU64Inst())
3951 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3952 else
3953 OpdsMapping[0] =
3954 getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size);
3955 unsigned Bank1 = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI /*, DefaultBankID*/);
3956 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank1, Size);
3957
3958 unsigned Bank2 = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI /*, DefaultBankID*/);
3959 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank2, Size);
3960 }
3961
3962 break;
3963 }
3964
3965 [[fallthrough]];
3966 }
3967 case AMDGPU::G_PTR_ADD:
3968 case AMDGPU::G_PTRMASK:
3969 case AMDGPU::G_ADD:
3970 case AMDGPU::G_SUB:
3971 case AMDGPU::G_SHL:
3972 case AMDGPU::G_LSHR:
3973 case AMDGPU::G_ASHR:
3974 case AMDGPU::G_UADDO:
3975 case AMDGPU::G_USUBO:
3976 case AMDGPU::G_UADDE:
3977 case AMDGPU::G_SADDE:
3978 case AMDGPU::G_USUBE:
3979 case AMDGPU::G_SSUBE:
3980 case AMDGPU::G_ABS:
3981 case AMDGPU::G_SHUFFLE_VECTOR:
3982 case AMDGPU::G_SBFX:
3983 case AMDGPU::G_UBFX:
3984 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
3985 case AMDGPU::G_AMDGPU_S_MUL_U64_U32:
3986 if (isSALUMapping(MI)) {
3987 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
3988 unsigned Size = Ty.getSizeInBits();
3989 // Packed add and sub are VALU only.
3990 if (Subtarget.hasAnyPackedU64Ops() && Ty.isVector() && Size == 128)
3991 return getDefaultMappingVOP(MI);
3992 return getDefaultMappingSOP(MI);
3993 }
3994 return getDefaultMappingVOP(MI);
3995 case AMDGPU::G_SMIN:
3996 case AMDGPU::G_SMAX:
3997 case AMDGPU::G_UMIN:
3998 case AMDGPU::G_UMAX:
3999 if (isSALUMapping(MI)) {
4000 // There are no scalar 64-bit min and max, use vector instruction instead.
4001 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() == 64 &&
4002 Subtarget.useMinMaxI64Insts())
4003 return getDefaultMappingVOP(MI);
4004 return getDefaultMappingSOP(MI);
4005 }
4006 return getDefaultMappingVOP(MI);
4007 case AMDGPU::G_FADD:
4008 case AMDGPU::G_FSUB:
4009 case AMDGPU::G_FMUL:
4010 case AMDGPU::G_FMA:
4011 case AMDGPU::G_FFLOOR:
4012 case AMDGPU::G_FCEIL:
4013 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
4014 case AMDGPU::G_INTRINSIC_TRUNC:
4015 case AMDGPU::G_STRICT_FADD:
4016 case AMDGPU::G_STRICT_FSUB:
4017 case AMDGPU::G_STRICT_FMUL:
4018 case AMDGPU::G_STRICT_FMA: {
4019 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4020 unsigned Size = Ty.getSizeInBits();
4021 if (Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
4022 (Size == 32 || Size == 16) && isSALUMapping(MI))
4023 return getDefaultMappingSOP(MI);
4024 return getDefaultMappingVOP(MI);
4025 }
4026 case AMDGPU::G_FMINNUM:
4027 case AMDGPU::G_FMAXNUM:
4028 case AMDGPU::G_FMINIMUMNUM:
4029 case AMDGPU::G_FMAXIMUMNUM: {
4030 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4031 unsigned Size = Ty.getSizeInBits();
4032 // This case is for GFX11.7, which has the following features:
4033 // 1. `hasIEEEMinimumMaximumInsts()` returns true
4034 // 2. `hasSALUMinimumMaximumInsts()` returns false
4035 // This indicates that GFX11.7 is using:
4036 // 1. `s_max/min_f32/f16` for scalar
4037 // 2. `v_max/min_num_f32/f16` for vector
4038 // By testing on real card, `s_max/min_f32/f16` has the same behavior
4039 // as `v_max/min_num_f32/f16`. So for GFX11.7, even if
4040 // `hasSALUMinimumMaximumInsts()` returns false, we can still choose SGPR
4041 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
4042 bool ScalarNumIsSafe =
4043 !MFI->getMode().IEEE || Subtarget.hasIEEEMinimumMaximumInsts();
4044 if (Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
4045 (Size == 32 || Size == 16) && isSALUMapping(MI) && ScalarNumIsSafe)
4046 return getDefaultMappingSOP(MI);
4047 return getDefaultMappingVOP(MI);
4048 }
4049 case AMDGPU::G_FMINIMUM:
4050 case AMDGPU::G_FMAXIMUM: {
4051 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4052 unsigned Size = Ty.getSizeInBits();
4053 if (Subtarget.hasSALUMinimumMaximumInsts() && Ty.isScalar() &&
4054 (Size == 32 || Size == 16) && isSALUMapping(MI))
4055 return getDefaultMappingSOP(MI);
4056 return getDefaultMappingVOP(MI);
4057 }
4058 case AMDGPU::G_FPTOSI:
4059 case AMDGPU::G_FPTOUI:
4060 case AMDGPU::G_FPTOSI_SAT:
4061 case AMDGPU::G_FPTOUI_SAT:
4062 case AMDGPU::G_SITOFP:
4063 case AMDGPU::G_UITOFP: {
4064 unsigned SizeDst = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4065 unsigned SizeSrc = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4066 if (Subtarget.hasSALUFloatInsts() && SizeDst == 32 && SizeSrc == 32 &&
4067 isSALUMapping(MI))
4068 return getDefaultMappingSOP(MI);
4069 return getDefaultMappingVOP(MI);
4070 }
4071 case AMDGPU::G_FPTRUNC:
4072 case AMDGPU::G_FPEXT: {
4073 unsigned SizeDst = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4074 unsigned SizeSrc = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4075 if (Subtarget.hasSALUFloatInsts() && SizeDst != 64 && SizeSrc != 64 &&
4076 isSALUMapping(MI))
4077 return getDefaultMappingSOP(MI);
4078 return getDefaultMappingVOP(MI);
4079 }
4080 case AMDGPU::G_FSQRT:
4081 case AMDGPU::G_FEXP2:
4082 case AMDGPU::G_FLOG2: {
4083 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4084 if (Subtarget.hasPseudoScalarTrans() && (Size == 16 || Size == 32) &&
4085 isSALUMapping(MI))
4086 return getDefaultMappingSOP(MI);
4087 return getDefaultMappingVOP(MI);
4088 }
4089 case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU
4090 case AMDGPU::G_SSUBSAT:
4091 case AMDGPU::G_UADDSAT:
4092 case AMDGPU::G_USUBSAT:
4093 case AMDGPU::G_FMAD:
4094 case AMDGPU::G_FLDEXP:
4095 case AMDGPU::G_FMINNUM_IEEE:
4096 case AMDGPU::G_FMAXNUM_IEEE:
4097 case AMDGPU::G_FCANONICALIZE:
4098 case AMDGPU::G_STRICT_FLDEXP:
4099 case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar?
4100 case AMDGPU::G_FSHR: // TODO: Expand for scalar
4101 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
4102 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
4103 case AMDGPU::G_AMDGPU_RCP_IFLAG:
4104 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
4105 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
4106 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
4107 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
4108 case AMDGPU::G_AMDGPU_CVT_PK_I16_I32:
4109 case AMDGPU::G_AMDGPU_SMED3:
4110 case AMDGPU::G_AMDGPU_FMED3:
4111 return getDefaultMappingVOP(MI);
4112 case AMDGPU::G_UMULH:
4113 case AMDGPU::G_SMULH: {
4114 if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI))
4115 return getDefaultMappingSOP(MI);
4116 return getDefaultMappingVOP(MI);
4117 }
4118 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4119 case AMDGPU::G_AMDGPU_MAD_I64_I32: {
4120 // Three possible mappings:
4121 //
4122 // - Default SOP
4123 // - Default VOP
4124 // - Scalar multiply: src0 and src1 are SGPRs, the rest is VOP.
4125 //
4126 // This allows instruction selection to keep the multiplication part of the
4127 // instruction on the SALU.
4128 bool AllSalu = true;
4129 bool MulSalu = true;
4130 for (unsigned i = 0; i < 5; ++i) {
4131 Register Reg = MI.getOperand(i).getReg();
4132 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
4133 if (Bank->getID() != AMDGPU::SGPRRegBankID) {
4134 AllSalu = false;
4135 if (i == 2 || i == 3) {
4136 MulSalu = false;
4137 break;
4138 }
4139 }
4140 }
4141 }
4142
4143 if (AllSalu)
4144 return getDefaultMappingSOP(MI);
4145
4146 // If the multiply-add is full-rate in VALU, use that even if the
4147 // multiplication part is scalar. Accumulating separately on the VALU would
4148 // take two instructions.
4149 if (!MulSalu || Subtarget.hasFullRate64Ops())
4150 return getDefaultMappingVOP(MI);
4151
4152 // Keep the multiplication on the SALU, then accumulate on the VALU.
4153 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 64);
4154 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4155 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4156 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4157 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 64);
4158 break;
4159 }
4160 case AMDGPU::G_IMPLICIT_DEF: {
4161 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4162 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4163 break;
4164 }
4165 case AMDGPU::G_FCONSTANT:
4166 case AMDGPU::G_CONSTANT:
4167 case AMDGPU::G_GLOBAL_VALUE:
4168 case AMDGPU::G_FRAME_INDEX:
4169 case AMDGPU::G_BLOCK_ADDR:
4170 case AMDGPU::G_READSTEADYCOUNTER:
4171 case AMDGPU::G_READCYCLECOUNTER: {
4172 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4173 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4174 break;
4175 }
4176 case AMDGPU::G_DYN_STACKALLOC: {
4177 // Result is always uniform, and a wave reduction is needed for the source.
4178 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4179 unsigned SrcBankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4180 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: SrcBankID, Size: 32);
4181 break;
4182 }
4183 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: {
4184 // This case is weird because we expect a physical register in the source,
4185 // but need to set a bank anyway.
4186 //
4187 // TODO: We could select the result to SGPR or VGPR
4188 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4189 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4190 break;
4191 }
4192 case AMDGPU::G_INSERT: {
4193 unsigned BankID = getMappingType(MRI, MI);
4194 unsigned DstSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
4195 unsigned SrcSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
4196 unsigned EltSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 2).getReg(), MRI);
4197 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: DstSize);
4198 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size: SrcSize);
4199 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, Size: EltSize);
4200 OpdsMapping[3] = nullptr;
4201 break;
4202 }
4203 case AMDGPU::G_EXTRACT: {
4204 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4205 unsigned DstSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
4206 unsigned SrcSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
4207 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: DstSize);
4208 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size: SrcSize);
4209 OpdsMapping[2] = nullptr;
4210 break;
4211 }
4212 case AMDGPU::G_BUILD_VECTOR:
4213 case AMDGPU::G_BUILD_VECTOR_TRUNC: {
4214 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4215 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16)) {
4216 unsigned DstSize = DstTy.getSizeInBits();
4217 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4218 unsigned Src0BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4219 unsigned Src1BankID = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4220 unsigned DstBankID = regBankUnion(RB0: Src0BankID, RB1: Src1BankID);
4221
4222 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: DstBankID, Size: DstSize);
4223 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Src0BankID, Size: SrcSize);
4224 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Src1BankID, Size: SrcSize);
4225 break;
4226 }
4227
4228 [[fallthrough]];
4229 }
4230 case AMDGPU::G_MERGE_VALUES:
4231 case AMDGPU::G_CONCAT_VECTORS: {
4232 unsigned Bank = getMappingType(MRI, MI);
4233 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4234 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4235
4236 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: DstSize);
4237 // Op1 and Dst should use the same register bank.
4238 for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i)
4239 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: Bank, Size: SrcSize);
4240 break;
4241 }
4242 case AMDGPU::G_BITREVERSE:
4243 case AMDGPU::G_BITCAST:
4244 case AMDGPU::G_INTTOPTR:
4245 case AMDGPU::G_PTRTOINT:
4246 case AMDGPU::G_FABS:
4247 case AMDGPU::G_FNEG: {
4248 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4249 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4250 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
4251 break;
4252 }
4253 case AMDGPU::G_AMDGPU_FFBH_U32:
4254 case AMDGPU::G_AMDGPU_FFBL_B32:
4255 case AMDGPU::G_CTLZ_ZERO_POISON:
4256 case AMDGPU::G_CTTZ_ZERO_POISON: {
4257 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4258 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4259 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: 32);
4260 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID, Size);
4261 break;
4262 }
4263 case AMDGPU::G_CTPOP: {
4264 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4265 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4266 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: 32);
4267
4268 // This should really be getValueMappingSGPR64Only, but allowing the generic
4269 // code to handle the register split just makes using LegalizerHelper more
4270 // difficult.
4271 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
4272 break;
4273 }
4274 case AMDGPU::G_TRUNC: {
4275 Register Dst = MI.getOperand(i: 0).getReg();
4276 Register Src = MI.getOperand(i: 1).getReg();
4277 unsigned Bank = getRegBankID(Reg: Src, MRI);
4278 unsigned DstSize = TRI->getRegSizeInBits(Reg: Dst, MRI);
4279 unsigned SrcSize = TRI->getRegSizeInBits(Reg: Src, MRI);
4280 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: DstSize);
4281 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: SrcSize);
4282 break;
4283 }
4284 case AMDGPU::G_ZEXT:
4285 case AMDGPU::G_SEXT:
4286 case AMDGPU::G_ANYEXT:
4287 case AMDGPU::G_SEXT_INREG: {
4288 Register Dst = MI.getOperand(i: 0).getReg();
4289 Register Src = MI.getOperand(i: 1).getReg();
4290 unsigned DstSize = TRI->getRegSizeInBits(Reg: Dst, MRI);
4291 unsigned SrcSize = TRI->getRegSizeInBits(Reg: Src, MRI);
4292
4293 unsigned DstBank;
4294 const RegisterBank *SrcBank = getRegBank(Reg: Src, MRI, TRI: *TRI);
4295 assert(SrcBank);
4296 switch (SrcBank->getID()) {
4297 case AMDGPU::SGPRRegBankID:
4298 DstBank = AMDGPU::SGPRRegBankID;
4299 break;
4300 default:
4301 DstBank = AMDGPU::VGPRRegBankID;
4302 break;
4303 }
4304
4305 // Scalar extend can use 64-bit BFE, but VGPRs require extending to
4306 // 32-bits, and then to 64.
4307 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: DstBank, Size: DstSize);
4308 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID: SrcBank->getID(),
4309 Size: SrcSize);
4310 break;
4311 }
4312 case AMDGPU::G_IS_FPCLASS: {
4313 Register SrcReg = MI.getOperand(i: 1).getReg();
4314 unsigned SrcSize = MRI.getType(Reg: SrcReg).getSizeInBits();
4315 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4316 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: DstSize);
4317 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4318 break;
4319 }
4320 case AMDGPU::G_STORE: {
4321 assert(MI.getOperand(0).isReg());
4322 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4323
4324 // FIXME: We need to specify a different reg bank once scalar stores are
4325 // supported.
4326 const ValueMapping *ValMapping =
4327 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4328 OpdsMapping[0] = ValMapping;
4329 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
4330 break;
4331 }
4332 case AMDGPU::G_ICMP:
4333 case AMDGPU::G_FCMP: {
4334 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4335
4336 // See if the result register has already been constrained to vcc, which may
4337 // happen due to control flow intrinsic lowering.
4338 unsigned DstBank = getRegBankID(Reg: MI.getOperand(i: 0).getReg(), MRI,
4339 Default: AMDGPU::SGPRRegBankID);
4340 unsigned Op2Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4341 unsigned Op3Bank = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI);
4342
4343 auto canUseSCCICMP = [&]() {
4344 auto Pred =
4345 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4346 return Size == 32 ||
4347 (Size == 64 &&
4348 (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
4349 Subtarget.hasScalarCompareEq64());
4350 };
4351 auto canUseSCCFCMP = [&]() {
4352 return Subtarget.hasSALUFloatInsts() && (Size == 32 || Size == 16);
4353 };
4354
4355 bool isICMP = MI.getOpcode() == AMDGPU::G_ICMP;
4356 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
4357 Op2Bank == AMDGPU::SGPRRegBankID &&
4358 Op3Bank == AMDGPU::SGPRRegBankID &&
4359 (isICMP ? canUseSCCICMP() : canUseSCCFCMP());
4360
4361 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4362 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4363
4364 // TODO: Use 32-bit for scalar output size.
4365 // SCC results will need to be copied to a 32-bit SGPR virtual register.
4366 const unsigned ResultSize = 1;
4367
4368 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: DstBank, Size: ResultSize);
4369 OpdsMapping[1] = nullptr; // Predicate Operand.
4370 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: SrcBank, Size);
4371 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: SrcBank, Size);
4372 break;
4373 }
4374 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
4375 // VGPR index can be used for waterfall when indexing a SGPR vector.
4376 unsigned SrcBankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4377 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4378 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4379 unsigned IdxSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4380 unsigned IdxBank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4381 unsigned OutputBankID = regBankUnion(RB0: SrcBankID, RB1: IdxBank);
4382
4383 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: OutputBankID, Size: DstSize);
4384 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: SrcBankID, Size: SrcSize);
4385
4386 // The index can be either if the source vector is VGPR.
4387 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4388 break;
4389 }
4390 case AMDGPU::G_INSERT_VECTOR_ELT: {
4391 unsigned OutputBankID = isSALUMapping(MI) ?
4392 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4393
4394 unsigned VecSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4395 unsigned InsertSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4396 unsigned IdxSize = MRI.getType(Reg: MI.getOperand(i: 3).getReg()).getSizeInBits();
4397 unsigned InsertEltBankID = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4398 unsigned IdxBankID = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI);
4399
4400 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: OutputBankID, Size: VecSize);
4401 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: OutputBankID, Size: VecSize);
4402
4403 // This is a weird case, because we need to break down the mapping based on
4404 // the register bank of a different operand.
4405 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) {
4406 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(BankID: InsertEltBankID,
4407 Size: InsertSize);
4408 } else {
4409 assert(InsertSize == 32 || InsertSize == 64);
4410 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: InsertEltBankID, Size: InsertSize);
4411 }
4412
4413 // The index can be either if the source vector is VGPR.
4414 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBankID, Size: IdxSize);
4415 break;
4416 }
4417 case AMDGPU::G_UNMERGE_VALUES: {
4418 unsigned Bank = getMappingType(MRI, MI);
4419
4420 // Op1 and Dst should use the same register bank.
4421 // FIXME: Shouldn't this be the default? Why do we need to handle this?
4422 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
4423 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i).getReg(), MRI);
4424 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: Bank, Size);
4425 }
4426 break;
4427 }
4428 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
4429 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
4430 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
4431 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
4432 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
4433 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
4434 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
4435 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
4436 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
4437 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
4438 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
4439 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
4440 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
4441 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
4442 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
4443 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
4444 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
4445 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
4446 case AMDGPU::G_AMDGPU_BUFFER_STORE:
4447 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
4448 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
4449 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
4450 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: {
4451 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4452
4453 // rsrc
4454 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4455
4456 // vindex
4457 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4458
4459 // voffset
4460 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4461
4462 // soffset
4463 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4464
4465 // Any remaining operands are immediates and were correctly null
4466 // initialized.
4467 break;
4468 }
4469 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
4470 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
4471 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
4472 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
4473 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
4474 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
4475 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
4476 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
4477 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
4478 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
4479 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
4480 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
4481 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
4482 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
4483 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
4484 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
4485 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
4486 // vdata_out
4487 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4488
4489 // vdata_in
4490 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4491
4492 // rsrc
4493 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4494
4495 // vindex
4496 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4497
4498 // voffset
4499 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4500
4501 // soffset
4502 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
4503
4504 // Any remaining operands are immediates and were correctly null
4505 // initialized.
4506 break;
4507 }
4508 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
4509 // vdata_out
4510 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4511
4512 // vdata_in
4513 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4514
4515 // cmp
4516 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4517
4518 // rsrc
4519 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4520
4521 // vindex
4522 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4523
4524 // voffset
4525 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
4526
4527 // soffset
4528 OpdsMapping[6] = getSGPROpMapping(Reg: MI.getOperand(i: 6).getReg(), MRI, TRI: *TRI);
4529
4530 // Any remaining operands are immediates and were correctly null
4531 // initialized.
4532 break;
4533 }
4534 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
4535 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
4536 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
4537 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
4538 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
4539 // Lie and claim everything is legal, even though some need to be
4540 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
4541 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4542 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4543
4544 // We need to convert this to a MUBUF if either the resource of offset is
4545 // VGPR.
4546 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
4547 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
4548 unsigned ResultBank = regBankUnion(RB0: RSrcBank, RB1: OffsetBank);
4549
4550 unsigned Size0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4551 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: ResultBank, Size: Size0);
4552 break;
4553 }
4554 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
4555 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4556 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4557 break;
4558 case AMDGPU::G_AMDGPU_SPONENTRY: {
4559 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4560 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4561 break;
4562 }
4563 case AMDGPU::G_INTRINSIC:
4564 case AMDGPU::G_INTRINSIC_CONVERGENT: {
4565 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
4566 default:
4567 return getInvalidInstructionMapping();
4568 case Intrinsic::amdgcn_div_fmas: {
4569 // The unscaled IEEE-input case can select S_FMAC_F32.
4570 if (Subtarget.hasSALUFloatInsts() &&
4571 MRI.getType(Reg: MI.getOperand(i: 0).getReg()) == LLT::scalar(SizeInBits: 32) &&
4572 isSALUMapping(MI) && AMDGPU::isUnscaledIEEEDivFmas(MI, MRI))
4573 return getDefaultMappingSOP(MI);
4574 return getDefaultMappingVOP(MI);
4575 }
4576 case Intrinsic::amdgcn_div_fixup:
4577 case Intrinsic::amdgcn_trig_preop:
4578 case Intrinsic::amdgcn_sin:
4579 case Intrinsic::amdgcn_cos:
4580 case Intrinsic::amdgcn_log_clamp:
4581 case Intrinsic::amdgcn_rcp_legacy:
4582 case Intrinsic::amdgcn_rsq_legacy:
4583 case Intrinsic::amdgcn_rsq_clamp:
4584 case Intrinsic::amdgcn_tanh:
4585 case Intrinsic::amdgcn_fmul_legacy:
4586 case Intrinsic::amdgcn_fma_legacy:
4587 case Intrinsic::amdgcn_frexp_mant:
4588 case Intrinsic::amdgcn_frexp_exp:
4589 case Intrinsic::amdgcn_fract:
4590 case Intrinsic::amdgcn_cvt_pknorm_i16:
4591 case Intrinsic::amdgcn_cvt_pknorm_u16:
4592 case Intrinsic::amdgcn_cvt_pk_i16:
4593 case Intrinsic::amdgcn_cvt_pk_u16:
4594 case Intrinsic::amdgcn_cvt_sr_pk_f16_f32:
4595 case Intrinsic::amdgcn_cvt_sr_pk_bf16_f32:
4596 case Intrinsic::amdgcn_cvt_pk_f16_fp8:
4597 case Intrinsic::amdgcn_cvt_pk_f16_bf8:
4598 case Intrinsic::amdgcn_cvt_pk_fp8_f16:
4599 case Intrinsic::amdgcn_cvt_pk_bf8_f16:
4600 case Intrinsic::amdgcn_cvt_sr_fp8_f16:
4601 case Intrinsic::amdgcn_cvt_sr_bf8_f16:
4602 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
4603 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
4604 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
4605 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
4606 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
4607 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
4608 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
4609 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
4610 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
4611 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
4612 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
4613 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
4614 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
4615 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
4616 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
4617 case Intrinsic::amdgcn_cvt_scale_pk32_f16_fp6:
4618 case Intrinsic::amdgcn_cvt_scale_pk32_bf16_fp6:
4619 case Intrinsic::amdgcn_cvt_scale_pk32_f16_bf6:
4620 case Intrinsic::amdgcn_cvt_scale_pk32_bf16_bf6:
4621 case Intrinsic::amdgcn_cvt_scale_pk32_f32_fp6:
4622 case Intrinsic::amdgcn_cvt_scale_pk32_f32_bf6:
4623 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_bf16:
4624 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_bf16:
4625 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f16:
4626 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f16:
4627 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f32:
4628 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f32:
4629 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f32:
4630 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f16:
4631 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_bf16:
4632 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f32:
4633 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f32:
4634 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f16:
4635 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f16:
4636 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_bf16:
4637 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_bf16:
4638 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_bf16:
4639 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_bf16:
4640 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f16:
4641 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f16:
4642 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f32:
4643 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f32:
4644 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f32:
4645 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f16:
4646 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_bf16:
4647 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f32:
4648 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f32:
4649 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f16:
4650 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f16:
4651 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_bf16:
4652 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_bf16:
4653 case Intrinsic::amdgcn_sat_pk4_i4_i8:
4654 case Intrinsic::amdgcn_sat_pk4_u4_u8:
4655 case Intrinsic::amdgcn_fmed3:
4656 case Intrinsic::amdgcn_cubeid:
4657 case Intrinsic::amdgcn_cubema:
4658 case Intrinsic::amdgcn_cubesc:
4659 case Intrinsic::amdgcn_cubetc:
4660 case Intrinsic::amdgcn_sffbh:
4661 case Intrinsic::amdgcn_fmad_ftz:
4662 case Intrinsic::amdgcn_mbcnt_lo:
4663 case Intrinsic::amdgcn_mbcnt_hi:
4664 case Intrinsic::amdgcn_mul_u24:
4665 case Intrinsic::amdgcn_mul_i24:
4666 case Intrinsic::amdgcn_mulhi_u24:
4667 case Intrinsic::amdgcn_mulhi_i24:
4668 case Intrinsic::amdgcn_lerp:
4669 case Intrinsic::amdgcn_sad_u8:
4670 case Intrinsic::amdgcn_msad_u8:
4671 case Intrinsic::amdgcn_sad_hi_u8:
4672 case Intrinsic::amdgcn_sad_u16:
4673 case Intrinsic::amdgcn_qsad_pk_u16_u8:
4674 case Intrinsic::amdgcn_mqsad_pk_u16_u8:
4675 case Intrinsic::amdgcn_mqsad_u32_u8:
4676 case Intrinsic::amdgcn_cvt_pk_u8_f32:
4677 case Intrinsic::amdgcn_alignbyte:
4678 case Intrinsic::amdgcn_perm:
4679 case Intrinsic::amdgcn_prng_b32:
4680 case Intrinsic::amdgcn_exclusive_scan_sum_i32:
4681 case Intrinsic::amdgcn_exclusive_scan_sum_u32:
4682 case Intrinsic::amdgcn_exclusive_scan_xor_b32:
4683 case Intrinsic::amdgcn_exclusive_scan_or_b32:
4684 case Intrinsic::amdgcn_exclusive_scan_and_b32:
4685 case Intrinsic::amdgcn_exclusive_scan_min_i16:
4686 case Intrinsic::amdgcn_exclusive_scan_min_u16:
4687 case Intrinsic::amdgcn_exclusive_scan_min_i32:
4688 case Intrinsic::amdgcn_exclusive_scan_min_u32:
4689 case Intrinsic::amdgcn_exclusive_scan_max_i16:
4690 case Intrinsic::amdgcn_exclusive_scan_max_u16:
4691 case Intrinsic::amdgcn_exclusive_scan_max_i32:
4692 case Intrinsic::amdgcn_exclusive_scan_max_u32:
4693 case Intrinsic::amdgcn_wave_match_b32:
4694 case Intrinsic::amdgcn_fdot2:
4695 case Intrinsic::amdgcn_sdot2:
4696 case Intrinsic::amdgcn_udot2:
4697 case Intrinsic::amdgcn_sdot4:
4698 case Intrinsic::amdgcn_udot4:
4699 case Intrinsic::amdgcn_sdot8:
4700 case Intrinsic::amdgcn_udot8:
4701 case Intrinsic::amdgcn_fdot2_bf16_bf16:
4702 case Intrinsic::amdgcn_fdot2_f16_f16:
4703 case Intrinsic::amdgcn_fdot2_f32_bf16:
4704 case Intrinsic::amdgcn_fdot2c_f32_bf16:
4705 case Intrinsic::amdgcn_sudot4:
4706 case Intrinsic::amdgcn_sudot8:
4707 case Intrinsic::amdgcn_dot4_f32_fp8_bf8:
4708 case Intrinsic::amdgcn_dot4_f32_bf8_fp8:
4709 case Intrinsic::amdgcn_dot4_f32_fp8_fp8:
4710 case Intrinsic::amdgcn_dot4_f32_bf8_bf8:
4711 case Intrinsic::amdgcn_cvt_f32_fp8:
4712 case Intrinsic::amdgcn_cvt_f32_fp8_e5m3:
4713 case Intrinsic::amdgcn_cvt_f32_bf8:
4714 case Intrinsic::amdgcn_cvt_off_f32_i4:
4715 case Intrinsic::amdgcn_cvt_pk_f32_fp8:
4716 case Intrinsic::amdgcn_cvt_pk_f32_bf8:
4717 case Intrinsic::amdgcn_cvt_pk_fp8_f32:
4718 case Intrinsic::amdgcn_cvt_pk_fp8_f32_e5m3:
4719 case Intrinsic::amdgcn_cvt_pk_bf8_f32:
4720 case Intrinsic::amdgcn_cvt_sr_fp8_f32:
4721 case Intrinsic::amdgcn_cvt_sr_fp8_f32_e5m3:
4722 case Intrinsic::amdgcn_cvt_sr_bf8_f32:
4723 case Intrinsic::amdgcn_cvt_sr_bf16_f32:
4724 case Intrinsic::amdgcn_cvt_sr_f16_f32:
4725 case Intrinsic::amdgcn_cvt_f16_fp8:
4726 case Intrinsic::amdgcn_cvt_f16_bf8:
4727 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f16:
4728 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f16:
4729 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_bf16:
4730 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_bf16:
4731 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f32:
4732 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f32:
4733 case Intrinsic::amdgcn_cvt_scalef32_f16_fp8:
4734 case Intrinsic::amdgcn_cvt_scalef32_f16_bf8:
4735 case Intrinsic::amdgcn_cvt_scalef32_f32_fp8:
4736 case Intrinsic::amdgcn_cvt_scalef32_f32_bf8:
4737 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f32:
4738 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f32:
4739 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp8:
4740 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_bf8:
4741 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f16:
4742 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_bf16:
4743 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f16:
4744 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_bf16:
4745 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp4:
4746 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f32:
4747 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp4:
4748 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp4:
4749 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_fp6:
4750 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_bf6:
4751 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_bf6:
4752 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_bf6:
4753 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_fp6:
4754 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_fp6:
4755 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_bf8:
4756 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_bf8:
4757 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp8:
4758 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp8:
4759 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f16:
4760 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_bf16:
4761 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f16:
4762 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_bf16:
4763 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f32:
4764 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_bf16:
4765 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f16:
4766 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f32:
4767 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_bf16:
4768 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f16:
4769 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f32:
4770 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_bf16:
4771 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f16:
4772 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f32:
4773 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_bf16:
4774 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f16:
4775 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f32:
4776 case Intrinsic::amdgcn_ashr_pk_i8_i32:
4777 case Intrinsic::amdgcn_ashr_pk_u8_i32:
4778 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_fp6_f32:
4779 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_bf6_f32:
4780 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16:
4781 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16:
4782 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied:
4783 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied:
4784 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf16:
4785 case Intrinsic::amdgcn_wmma_f32_16x16x16_f16:
4786 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu4:
4787 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu8:
4788 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8:
4789 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8:
4790 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8:
4791 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8:
4792 case Intrinsic::amdgcn_wmma_i32_16x16x32_iu4:
4793 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
4794 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
4795 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
4796 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
4797 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
4798 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
4799 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4:
4800 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
4801 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
4802 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
4803 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8:
4804 case Intrinsic::amdgcn_wmma_f64_16x16x4_f64:
4805 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4806 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4807 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4808 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4809 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4810 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
4811 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_fp8:
4812 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_bf8:
4813 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_fp8:
4814 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_bf8:
4815 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_fp8:
4816 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_bf8:
4817 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_fp8:
4818 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_bf8:
4819 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_fp8:
4820 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_bf8:
4821 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_fp8:
4822 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_bf8:
4823 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_fp8:
4824 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_bf8:
4825 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_fp8:
4826 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_bf8:
4827 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
4828 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
4829 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
4830 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
4831 case Intrinsic::amdgcn_wmma_f32_32x16x128_f4:
4832 case Intrinsic::amdgcn_wmma_scale_f32_32x16x128_f4:
4833 case Intrinsic::amdgcn_wmma_scale16_f32_32x16x128_f4:
4834 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
4835 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
4836 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
4837 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
4838 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
4839 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
4840 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
4841 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
4842 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
4843 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
4844 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
4845 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
4846 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8:
4847 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
4848 case Intrinsic::amdgcn_perm_pk16_b4_u4:
4849 case Intrinsic::amdgcn_perm_pk16_b6_u4:
4850 case Intrinsic::amdgcn_perm_pk16_b8_u4:
4851 case Intrinsic::amdgcn_add_max_i32:
4852 case Intrinsic::amdgcn_add_max_u32:
4853 case Intrinsic::amdgcn_add_min_i32:
4854 case Intrinsic::amdgcn_add_min_u32:
4855 case Intrinsic::amdgcn_pk_add_max_i16:
4856 case Intrinsic::amdgcn_pk_add_max_u16:
4857 case Intrinsic::amdgcn_pk_add_min_i16:
4858 case Intrinsic::amdgcn_pk_add_min_u16:
4859 return getDefaultMappingVOP(MI);
4860 case Intrinsic::amdgcn_log:
4861 case Intrinsic::amdgcn_exp2:
4862 case Intrinsic::amdgcn_rcp:
4863 case Intrinsic::amdgcn_rsq:
4864 case Intrinsic::amdgcn_sqrt: {
4865 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4866 unsigned Size = Ty.getSizeInBits();
4867 // There is no pseudo scalar transcendental instruction for bf16.
4868 if (Subtarget.hasPseudoScalarTrans() && !Ty.isBFloat16() &&
4869 (Size == 16 || Size == 32) && isSALUMapping(MI))
4870 return getDefaultMappingSOP(MI);
4871 return getDefaultMappingVOP(MI);
4872 }
4873 case Intrinsic::amdgcn_sbfe:
4874 case Intrinsic::amdgcn_ubfe:
4875 if (isSALUMapping(MI))
4876 return getDefaultMappingSOP(MI);
4877 return getDefaultMappingVOP(MI);
4878 case Intrinsic::amdgcn_ds_swizzle:
4879 case Intrinsic::amdgcn_ds_permute:
4880 case Intrinsic::amdgcn_ds_bpermute:
4881 case Intrinsic::amdgcn_update_dpp:
4882 case Intrinsic::amdgcn_mov_dpp8:
4883 case Intrinsic::amdgcn_mov_dpp:
4884 case Intrinsic::amdgcn_strict_wwm:
4885 case Intrinsic::amdgcn_wwm:
4886 case Intrinsic::amdgcn_strict_wqm:
4887 case Intrinsic::amdgcn_wqm:
4888 case Intrinsic::amdgcn_softwqm:
4889 case Intrinsic::amdgcn_set_inactive:
4890 case Intrinsic::amdgcn_set_inactive_chain_arg:
4891 case Intrinsic::amdgcn_permlane64:
4892 case Intrinsic::amdgcn_ds_bpermute_fi_b32:
4893 return getDefaultMappingAllVGPR(MI);
4894 case Intrinsic::amdgcn_cvt_pkrtz:
4895 if (Subtarget.hasSALUFloatInsts() && isSALUMapping(MI))
4896 return getDefaultMappingSOP(MI);
4897 return getDefaultMappingVOP(MI);
4898 case Intrinsic::amdgcn_kernarg_segment_ptr:
4899 case Intrinsic::amdgcn_s_getpc:
4900 case Intrinsic::amdgcn_groupstaticsize:
4901 case Intrinsic::amdgcn_reloc_constant:
4902 case Intrinsic::returnaddress: {
4903 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4904 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4905 break;
4906 }
4907 case Intrinsic::amdgcn_wqm_vote: {
4908 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4909 OpdsMapping[0] = OpdsMapping[2]
4910 = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size);
4911 break;
4912 }
4913 case Intrinsic::amdgcn_ps_live: {
4914 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4915 break;
4916 }
4917 case Intrinsic::amdgcn_div_scale: {
4918 unsigned Dst0Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4919 unsigned Dst1Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4920 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Dst0Size);
4921 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: Dst1Size);
4922
4923 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 3).getReg()).getSizeInBits();
4924 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4925 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4926 break;
4927 }
4928 case Intrinsic::amdgcn_class: {
4929 Register Src0Reg = MI.getOperand(i: 2).getReg();
4930 Register Src1Reg = MI.getOperand(i: 3).getReg();
4931 unsigned Src0Size = MRI.getType(Reg: Src0Reg).getSizeInBits();
4932 unsigned Src1Size = MRI.getType(Reg: Src1Reg).getSizeInBits();
4933 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4934 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: DstSize);
4935 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Src0Size);
4936 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Src1Size);
4937 break;
4938 }
4939 case Intrinsic::amdgcn_readlane: {
4940 // This must be an SGPR, but accept a VGPR.
4941 Register IdxReg = MI.getOperand(i: 3).getReg();
4942 unsigned IdxSize = MRI.getType(Reg: IdxReg).getSizeInBits();
4943 unsigned IdxBank = getRegBankID(Reg: IdxReg, MRI, Default: AMDGPU::SGPRRegBankID);
4944 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4945 [[fallthrough]];
4946 }
4947 case Intrinsic::amdgcn_readfirstlane: {
4948 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4949 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4950 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
4951 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4952 break;
4953 }
4954 case Intrinsic::amdgcn_writelane: {
4955 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4956 Register SrcReg = MI.getOperand(i: 2).getReg();
4957 unsigned SrcSize = MRI.getType(Reg: SrcReg).getSizeInBits();
4958 unsigned SrcBank = getRegBankID(Reg: SrcReg, MRI, Default: AMDGPU::SGPRRegBankID);
4959 Register IdxReg = MI.getOperand(i: 3).getReg();
4960 unsigned IdxSize = MRI.getType(Reg: IdxReg).getSizeInBits();
4961 unsigned IdxBank = getRegBankID(Reg: IdxReg, MRI, Default: AMDGPU::SGPRRegBankID);
4962 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
4963
4964 // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
4965 // to legalize.
4966 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: SrcBank, Size: SrcSize);
4967 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4968 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4969 break;
4970 }
4971 case Intrinsic::amdgcn_if_break: {
4972 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
4973 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4974 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4975 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4976 break;
4977 }
4978 case Intrinsic::amdgcn_permlane16:
4979 case Intrinsic::amdgcn_permlanex16: {
4980 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
4981 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4982 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4983 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4984 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4985 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4986 break;
4987 }
4988 case Intrinsic::amdgcn_permlane_bcast:
4989 case Intrinsic::amdgcn_permlane_up:
4990 case Intrinsic::amdgcn_permlane_down:
4991 case Intrinsic::amdgcn_permlane_xor: {
4992 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
4993 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4994 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4995 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4996 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4997 break;
4998 }
4999 case Intrinsic::amdgcn_permlane_idx_gen: {
5000 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
5001 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5002 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5003 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5004 break;
5005 }
5006 case Intrinsic::amdgcn_permlane16_var:
5007 case Intrinsic::amdgcn_permlanex16_var: {
5008 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
5009 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5010 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5011 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5012 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5013 break;
5014 }
5015 case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
5016 case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
5017 case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
5018 case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
5019 case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
5020 case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
5021 case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
5022 case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
5023 case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
5024 case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
5025 case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
5026 case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
5027 case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
5028 case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
5029 case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
5030 case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
5031 case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
5032 case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
5033 case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
5034 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
5035 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
5036 case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
5037 case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
5038 case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
5039 case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
5040 case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
5041 case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
5042 case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
5043 case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
5044 case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
5045 case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
5046 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
5047 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
5048 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
5049 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
5050 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
5051 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
5052 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
5053 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
5054 case Intrinsic::amdgcn_mfma_f32_16x16x32_f16:
5055 case Intrinsic::amdgcn_mfma_f32_32x32x16_f16:
5056 case Intrinsic::amdgcn_mfma_i32_16x16x64_i8:
5057 case Intrinsic::amdgcn_mfma_i32_32x32x32_i8:
5058 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf16: {
5059 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5060 unsigned MinNumRegsRequired = DstSize / 32;
5061
5062 // Default for MAI intrinsics.
5063 // srcC can also be an immediate which can be folded later.
5064 // FIXME: Should we eventually add an alternative mapping with AGPR src
5065 // for srcA/srcB?
5066 //
5067 // vdst, srcA, srcB, srcC
5068 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5069
5070 bool UseAGPRForm = !Subtarget.hasGFX90AInsts() ||
5071 Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5072
5073 OpdsMapping[0] =
5074 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI)
5075 : getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5076 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5077 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5078 OpdsMapping[4] =
5079 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5080 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5081 break;
5082 }
5083 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
5084 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
5085 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5086 unsigned MinNumRegsRequired = DstSize / 32;
5087
5088 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5089 bool UseAGPRForm = Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5090
5091 OpdsMapping[0] =
5092 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI)
5093 : getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5094
5095 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5096 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5097 OpdsMapping[4] =
5098 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5099 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5100
5101 OpdsMapping[8] = getVGPROpMapping(Reg: MI.getOperand(i: 8).getReg(), MRI, TRI: *TRI);
5102 OpdsMapping[10] = getVGPROpMapping(Reg: MI.getOperand(i: 10).getReg(), MRI, TRI: *TRI);
5103 break;
5104 }
5105 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
5106 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
5107 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
5108 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
5109 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
5110 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
5111 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
5112 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
5113 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
5114 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
5115 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
5116 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
5117 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
5118 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
5119 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
5120 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
5121 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
5122 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
5123 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
5124 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
5125 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
5126 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
5127 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
5128 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
5129 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
5130 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
5131 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
5132 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8: {
5133 Register DstReg = MI.getOperand(i: 0).getReg();
5134 unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
5135 unsigned MinNumRegsRequired = DstSize / 32;
5136 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5137 bool UseAGPRForm = Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5138
5139 // vdst, srcA, srcB, srcC, idx
5140 OpdsMapping[0] = UseAGPRForm ? getAGPROpMapping(Reg: DstReg, MRI, TRI: *TRI)
5141 : getVGPROpMapping(Reg: DstReg, MRI, TRI: *TRI);
5142
5143 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5144 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5145 OpdsMapping[4] =
5146 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5147 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5148 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5149 break;
5150 }
5151 case Intrinsic::amdgcn_interp_p1:
5152 case Intrinsic::amdgcn_interp_p2:
5153 case Intrinsic::amdgcn_interp_mov:
5154 case Intrinsic::amdgcn_interp_p1_f16:
5155 case Intrinsic::amdgcn_interp_p2_f16:
5156 case Intrinsic::amdgcn_lds_param_load: {
5157 const int M0Idx = MI.getNumOperands() - 1;
5158 Register M0Reg = MI.getOperand(i: M0Idx).getReg();
5159 unsigned M0Bank = getRegBankID(Reg: M0Reg, MRI, Default: AMDGPU::SGPRRegBankID);
5160 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5161
5162 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5163 for (int I = 2; I != M0Idx && MI.getOperand(i: I).isReg(); ++I)
5164 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5165
5166 // Must be SGPR, but we must take whatever the original bank is and fix it
5167 // later.
5168 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5169 break;
5170 }
5171 case Intrinsic::amdgcn_interp_inreg_p10:
5172 case Intrinsic::amdgcn_interp_inreg_p2:
5173 case Intrinsic::amdgcn_interp_inreg_p10_f16:
5174 case Intrinsic::amdgcn_interp_inreg_p2_f16:
5175 case Intrinsic::amdgcn_interp_p10_rtz_f16:
5176 case Intrinsic::amdgcn_interp_p2_rtz_f16: {
5177 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5178 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5179 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5180 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5181 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5182 break;
5183 }
5184 case Intrinsic::amdgcn_permlane16_swap:
5185 case Intrinsic::amdgcn_permlane32_swap: {
5186 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5187 OpdsMapping[0] = OpdsMapping[1] = OpdsMapping[3] = OpdsMapping[4] =
5188 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5189 break;
5190 }
5191 case Intrinsic::amdgcn_ballot: {
5192 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5193 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5194 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
5195 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: SrcSize);
5196 break;
5197 }
5198 case Intrinsic::amdgcn_inverse_ballot: {
5199 // This must be an SGPR, but accept a VGPR.
5200 Register MaskReg = MI.getOperand(i: 2).getReg();
5201 unsigned MaskSize = MRI.getType(Reg: MaskReg).getSizeInBits();
5202 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5203 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5204 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: MaskSize);
5205 break;
5206 }
5207 case Intrinsic::amdgcn_bitop3: {
5208 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
5209 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5210 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5211 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5212 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5213 break;
5214 }
5215 case Intrinsic::amdgcn_s_quadmask:
5216 case Intrinsic::amdgcn_s_wqm: {
5217 Register MaskReg = MI.getOperand(i: 2).getReg();
5218 unsigned MaskSize = MRI.getType(Reg: MaskReg).getSizeInBits();
5219 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5220 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: MaskSize);
5221 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: MaskSize);
5222 break;
5223 }
5224 case Intrinsic::amdgcn_wave_reduce_add:
5225 case Intrinsic::amdgcn_wave_reduce_fadd:
5226 case Intrinsic::amdgcn_wave_reduce_sub:
5227 case Intrinsic::amdgcn_wave_reduce_fsub:
5228 case Intrinsic::amdgcn_wave_reduce_min:
5229 case Intrinsic::amdgcn_wave_reduce_umin:
5230 case Intrinsic::amdgcn_wave_reduce_fmin:
5231 case Intrinsic::amdgcn_wave_reduce_max:
5232 case Intrinsic::amdgcn_wave_reduce_umax:
5233 case Intrinsic::amdgcn_wave_reduce_fmax:
5234 case Intrinsic::amdgcn_wave_reduce_and:
5235 case Intrinsic::amdgcn_wave_reduce_or:
5236 case Intrinsic::amdgcn_wave_reduce_xor: {
5237 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5238 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
5239 unsigned OpSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5240 auto regBankID =
5241 isSALUMapping(MI) ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5242 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: regBankID, Size: OpSize);
5243 break;
5244 }
5245 case Intrinsic::amdgcn_s_bitreplicate: {
5246 Register MaskReg = MI.getOperand(i: 2).getReg();
5247 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5248 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 64);
5249 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: 32);
5250 break;
5251 }
5252 case Intrinsic::amdgcn_wave_shuffle: {
5253 unsigned OpSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5254 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5255 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5256 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5257 break;
5258 }
5259 }
5260 break;
5261 }
5262 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
5263 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
5264 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
5265 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
5266 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
5267 auto IntrID = AMDGPU::getIntrinsicID(I: MI);
5268 const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(Intr: IntrID);
5269 assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic");
5270 // Non-images can have complications from operands that allow both SGPR
5271 // and VGPR. For now it's too complicated to figure out the final opcode
5272 // to derive the register bank from the MCInstrDesc.
5273 assert(RSrcIntrin->IsImage);
5274 return getImageMapping(MRI, MI, RsrcIdx: RSrcIntrin->RsrcArg);
5275 }
5276 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
5277 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
5278 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
5279 bool IsDualOrBVH8 =
5280 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
5281 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
5282 unsigned NumMods = IsDualOrBVH8 ? 0 : 1; // Has A16 modifier
5283 unsigned LastRegOpIdx = MI.getNumExplicitOperands() - 1 - NumMods;
5284 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5285 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5286 if (IsDualOrBVH8) {
5287 OpdsMapping[1] = AMDGPU::getValueMapping(
5288 BankID: AMDGPU::VGPRRegBankID,
5289 Size: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits());
5290 OpdsMapping[2] = AMDGPU::getValueMapping(
5291 BankID: AMDGPU::VGPRRegBankID,
5292 Size: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
5293 }
5294 OpdsMapping[LastRegOpIdx] =
5295 getSGPROpMapping(Reg: MI.getOperand(i: LastRegOpIdx).getReg(), MRI, TRI: *TRI);
5296 if (LastRegOpIdx == 3) {
5297 // Sequential form: all operands combined into VGPR256/VGPR512
5298 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5299 if (Size > 256)
5300 Size = 512;
5301 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5302 } else {
5303 // NSA form
5304 unsigned FirstSrcOpIdx = IsDualOrBVH8 ? 4 : 2;
5305 for (unsigned I = FirstSrcOpIdx; I < LastRegOpIdx; ++I) {
5306 unsigned Size = MRI.getType(Reg: MI.getOperand(i: I).getReg()).getSizeInBits();
5307 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5308 }
5309 }
5310 break;
5311 }
5312 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
5313 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
5314 auto IntrID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
5315 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
5316 AMDGPU::lookupBufferIntrinsic(Intr: IntrID)) {
5317 // The legalizer turns all other buffer intrinsics into
5318 // G_AMDGPU_BUFFER_* and similar instructions.
5319 if (BufInfo->Kind != AMDGPU::BufferKindLoadToLDS)
5320 return getInvalidInstructionMapping();
5321 auto MapArg = [&](unsigned Arg, bool IsSGPR) {
5322 unsigned Idx = AMDGPU::getMIROperandIdx(MI, Arg);
5323 Register Reg = MI.getOperand(i: Idx).getReg();
5324 OpdsMapping[Idx] = IsSGPR ? getSGPROpMapping(Reg, MRI, TRI: *TRI)
5325 : getVGPROpMapping(Reg, MRI, TRI: *TRI);
5326 };
5327 MapArg(BufInfo->RsrcArg, /*IsSGPR=*/true);
5328 MapArg(*BufInfo->getLDSPtrArg(), /*IsSGPR=*/true); // M0
5329 if (std::optional<unsigned> VIndexArg = BufInfo->getVIndexArg())
5330 MapArg(*VIndexArg, /*IsSGPR=*/false);
5331 MapArg(*BufInfo->getVOffsetArg(), /*IsSGPR=*/false);
5332 MapArg(*BufInfo->getSOffsetArg(), /*IsSGPR=*/true);
5333 break;
5334 }
5335 switch (IntrID) {
5336 case Intrinsic::amdgcn_s_getreg:
5337 case Intrinsic::amdgcn_s_memtime:
5338 case Intrinsic::amdgcn_s_memrealtime:
5339 case Intrinsic::amdgcn_s_get_waveid_in_workgroup:
5340 case Intrinsic::amdgcn_s_sendmsg_rtn: {
5341 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5342 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5343 break;
5344 }
5345 case Intrinsic::amdgcn_global_atomic_fmin_num:
5346 case Intrinsic::amdgcn_global_atomic_fmax_num:
5347 case Intrinsic::amdgcn_flat_atomic_fmin_num:
5348 case Intrinsic::amdgcn_flat_atomic_fmax_num:
5349 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
5350 case Intrinsic::amdgcn_global_load_tr_b64:
5351 case Intrinsic::amdgcn_global_load_tr_b128:
5352 case Intrinsic::amdgcn_global_load_tr4_b64:
5353 case Intrinsic::amdgcn_global_load_tr6_b96:
5354 case Intrinsic::amdgcn_ds_load_tr8_b64:
5355 case Intrinsic::amdgcn_ds_load_tr16_b128:
5356 case Intrinsic::amdgcn_ds_load_tr4_b64:
5357 case Intrinsic::amdgcn_ds_load_tr6_b96:
5358 case Intrinsic::amdgcn_ds_read_tr4_b64:
5359 case Intrinsic::amdgcn_ds_read_tr6_b96:
5360 case Intrinsic::amdgcn_ds_read_tr8_b64:
5361 case Intrinsic::amdgcn_ds_read_tr16_b64:
5362 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
5363 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
5364 return getDefaultMappingAllVGPR(MI);
5365 case Intrinsic::amdgcn_ds_ordered_add:
5366 case Intrinsic::amdgcn_ds_ordered_swap: {
5367 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5368 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5369 unsigned M0Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5370 Default: AMDGPU::SGPRRegBankID);
5371 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5372 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5373 break;
5374 }
5375 case Intrinsic::amdgcn_ds_append:
5376 case Intrinsic::amdgcn_ds_consume: {
5377 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5378 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5379 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5380 break;
5381 }
5382 case Intrinsic::amdgcn_exp_compr:
5383 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5384 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5385 break;
5386 case Intrinsic::amdgcn_exp:
5387 // FIXME: Could we support packed types here?
5388 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5389 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5390 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5391 OpdsMapping[6] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5392 break;
5393 case Intrinsic::amdgcn_exp_row:
5394 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5395 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5396 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5397 OpdsMapping[6] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5398 OpdsMapping[8] = getSGPROpMapping(Reg: MI.getOperand(i: 8).getReg(), MRI, TRI: *TRI);
5399 break;
5400 case Intrinsic::amdgcn_s_alloc_vgpr:
5401 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1);
5402 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
5403 break;
5404 case Intrinsic::amdgcn_s_sendmsg:
5405 case Intrinsic::amdgcn_s_sendmsghalt: {
5406 // This must be an SGPR, but accept a VGPR.
5407 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5408 Default: AMDGPU::SGPRRegBankID);
5409 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5410 break;
5411 }
5412 case Intrinsic::amdgcn_s_setreg: {
5413 // This must be an SGPR, but accept a VGPR.
5414 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5415 Default: AMDGPU::SGPRRegBankID);
5416 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5417 break;
5418 }
5419 case Intrinsic::amdgcn_s_ttracedata: {
5420 // This must be an SGPR, but accept a VGPR.
5421 unsigned Bank =
5422 getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5423 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5424 break;
5425 }
5426 case Intrinsic::amdgcn_end_cf: {
5427 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
5428 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5429 break;
5430 }
5431 case Intrinsic::amdgcn_else: {
5432 unsigned WaveSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
5433 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5434 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: WaveSize);
5435 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: WaveSize);
5436 break;
5437 }
5438 case Intrinsic::amdgcn_init_whole_wave:
5439 case Intrinsic::amdgcn_live_mask: {
5440 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5441 break;
5442 }
5443 case Intrinsic::amdgcn_wqm_demote:
5444 case Intrinsic::amdgcn_kill: {
5445 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5446 break;
5447 }
5448 case Intrinsic::amdgcn_init_exec_from_input: {
5449 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
5450 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5451 break;
5452 }
5453 case Intrinsic::amdgcn_ds_gws_init:
5454 case Intrinsic::amdgcn_ds_gws_barrier:
5455 case Intrinsic::amdgcn_ds_gws_sema_br: {
5456 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5457
5458 // This must be an SGPR, but accept a VGPR.
5459 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5460 Default: AMDGPU::SGPRRegBankID);
5461 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5462 break;
5463 }
5464 case Intrinsic::amdgcn_ds_gws_sema_v:
5465 case Intrinsic::amdgcn_ds_gws_sema_p:
5466 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
5467 // This must be an SGPR, but accept a VGPR.
5468 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
5469 Default: AMDGPU::SGPRRegBankID);
5470 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5471 break;
5472 }
5473 case Intrinsic::amdgcn_cluster_load_b32:
5474 case Intrinsic::amdgcn_cluster_load_b64:
5475 case Intrinsic::amdgcn_cluster_load_b128: {
5476 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5477 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5478 unsigned M0Bank =
5479 getRegBankID(Reg: MI.getOperand(i: 4).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5480 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5481 break;
5482 }
5483 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
5484 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
5485 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
5486 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
5487 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5488 // LDS address goes into $vdst (VGPR).
5489 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5490 unsigned M0Bank =
5491 getRegBankID(Reg: MI.getOperand(i: 5).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5492 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5493 break;
5494 }
5495 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
5496 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
5497 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
5498 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
5499 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
5500 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
5501 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
5502 case Intrinsic::amdgcn_global_load_async_to_lds_b128: {
5503 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5504 // LDS address goes into $vdst/$vdata (VGPR).
5505 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5506 break;
5507 }
5508 case Intrinsic::amdgcn_load_to_lds:
5509 case Intrinsic::amdgcn_load_async_to_lds:
5510 case Intrinsic::amdgcn_global_load_lds:
5511 case Intrinsic::amdgcn_global_load_async_lds: {
5512 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5513 // LDS address goes into M0 (SGPR).
5514 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5515 break;
5516 }
5517 case Intrinsic::amdgcn_lds_direct_load: {
5518 const int M0Idx = MI.getNumOperands() - 1;
5519 Register M0Reg = MI.getOperand(i: M0Idx).getReg();
5520 unsigned M0Bank = getRegBankID(Reg: M0Reg, MRI, Default: AMDGPU::SGPRRegBankID);
5521 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5522
5523 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5524 for (int I = 2; I != M0Idx && MI.getOperand(i: I).isReg(); ++I)
5525 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5526
5527 // Must be SGPR, but we must take whatever the original bank is and fix it
5528 // later.
5529 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5530 break;
5531 }
5532 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
5533 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn:
5534 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5535 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5536 break;
5537 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
5538 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
5539 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
5540 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
5541 OpdsMapping[0] =
5542 getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI); // %vdst
5543 OpdsMapping[1] =
5544 getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI); // %addr
5545 OpdsMapping[3] =
5546 getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI); // %addr
5547 OpdsMapping[4] =
5548 getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI); // %data0
5549 OpdsMapping[5] =
5550 getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI); // %data1
5551 break;
5552 }
5553 case Intrinsic::amdgcn_s_sleep_var:
5554 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5555 break;
5556 case Intrinsic::amdgcn_s_barrier_join:
5557 case Intrinsic::amdgcn_s_wakeup_barrier:
5558 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5559 break;
5560 case Intrinsic::amdgcn_s_barrier_init:
5561 case Intrinsic::amdgcn_s_barrier_signal_var:
5562 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5563 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5564 break;
5565 case Intrinsic::amdgcn_s_barrier_signal_isfirst: {
5566 const unsigned ResultSize = 1;
5567 OpdsMapping[0] =
5568 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: ResultSize);
5569 break;
5570 }
5571 case Intrinsic::amdgcn_s_get_barrier_state:
5572 case Intrinsic::amdgcn_s_get_named_barrier_state: {
5573 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5574 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5575 break;
5576 }
5577 case Intrinsic::amdgcn_pops_exiting_wave_id:
5578 return getDefaultMappingSOP(MI);
5579 case Intrinsic::amdgcn_tensor_load_to_lds:
5580 case Intrinsic::amdgcn_tensor_store_from_lds: {
5581 // Lie and claim everything is legal, even all operands need to be
5582 // SGPRs. applyMapping will have to deal with it with readfirstlane.
5583 for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
5584 if (MI.getOperand(i: I).isReg()) {
5585 Register Reg = MI.getOperand(i: I).getReg();
5586 auto OpBank = getRegBankID(Reg, MRI);
5587 unsigned Size = TRI->getRegSizeInBits(Reg, MRI);
5588 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: OpBank, Size);
5589 }
5590 }
5591 break;
5592 }
5593 case Intrinsic::amdgcn_s_prefetch_data:
5594 case Intrinsic::amdgcn_s_prefetch_inst: {
5595 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5596 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5597 break;
5598 }
5599 case Intrinsic::amdgcn_flat_prefetch:
5600 case Intrinsic::amdgcn_global_prefetch:
5601 return getDefaultMappingVOP(MI);
5602 default:
5603 return getInvalidInstructionMapping();
5604 }
5605 break;
5606 }
5607 case AMDGPU::G_SELECT: {
5608 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5609 unsigned Op2Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5610 Default: AMDGPU::SGPRRegBankID);
5611 unsigned Op3Bank = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI,
5612 Default: AMDGPU::SGPRRegBankID);
5613 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
5614 Op3Bank == AMDGPU::SGPRRegBankID;
5615
5616 unsigned CondBankDefault = SGPRSrcs ?
5617 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5618 unsigned CondBank = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
5619 Default: CondBankDefault);
5620 if (CondBank == AMDGPU::SGPRRegBankID)
5621 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5622 else if (CondBank == AMDGPU::VGPRRegBankID)
5623 CondBank = AMDGPU::VCCRegBankID;
5624
5625 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ?
5626 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5627
5628 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID);
5629
5630 // TODO: Should report 32-bit for scalar condition type.
5631 if (Size == 64) {
5632 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5633 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: CondBank, Size: 1);
5634 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5635 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5636 } else {
5637 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size);
5638 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: CondBank, Size: 1);
5639 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size);
5640 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: Bank, Size);
5641 }
5642
5643 break;
5644 }
5645
5646 case AMDGPU::G_SI_CALL: {
5647 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 64);
5648 // Lie and claim everything is legal, even though some need to be
5649 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
5650 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5651
5652 // Allow anything for implicit arguments
5653 for (unsigned I = 4; I < MI.getNumOperands(); ++I) {
5654 if (MI.getOperand(i: I).isReg()) {
5655 Register Reg = MI.getOperand(i: I).getReg();
5656 auto OpBank = getRegBankID(Reg, MRI);
5657 unsigned Size = TRI->getRegSizeInBits(Reg, MRI);
5658 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: OpBank, Size);
5659 }
5660 }
5661 break;
5662 }
5663 case AMDGPU::G_LOAD:
5664 case AMDGPU::G_ZEXTLOAD:
5665 case AMDGPU::G_SEXTLOAD:
5666 return getInstrMappingForLoad(MI);
5667
5668 case AMDGPU::G_ATOMICRMW_XCHG:
5669 case AMDGPU::G_ATOMICRMW_ADD:
5670 case AMDGPU::G_ATOMICRMW_SUB:
5671 case AMDGPU::G_ATOMICRMW_AND:
5672 case AMDGPU::G_ATOMICRMW_OR:
5673 case AMDGPU::G_ATOMICRMW_XOR:
5674 case AMDGPU::G_ATOMICRMW_MAX:
5675 case AMDGPU::G_ATOMICRMW_MIN:
5676 case AMDGPU::G_ATOMICRMW_UMAX:
5677 case AMDGPU::G_ATOMICRMW_UMIN:
5678 case AMDGPU::G_ATOMICRMW_FADD:
5679 case AMDGPU::G_ATOMICRMW_FMIN:
5680 case AMDGPU::G_ATOMICRMW_FMAX:
5681 case AMDGPU::G_ATOMICRMW_UINC_WRAP:
5682 case AMDGPU::G_ATOMICRMW_UDEC_WRAP:
5683 case AMDGPU::G_ATOMICRMW_USUB_COND:
5684 case AMDGPU::G_ATOMICRMW_USUB_SAT:
5685 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: {
5686 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5687 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
5688 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5689 break;
5690 }
5691 case AMDGPU::G_ATOMIC_CMPXCHG: {
5692 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5693 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
5694 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5695 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5696 break;
5697 }
5698 case AMDGPU::G_BRCOND: {
5699 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 0).getReg(), MRI,
5700 Default: AMDGPU::SGPRRegBankID);
5701 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
5702 if (Bank != AMDGPU::SGPRRegBankID)
5703 Bank = AMDGPU::VCCRegBankID;
5704
5705 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: 1);
5706 break;
5707 }
5708 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
5709 return getDefaultMappingVOP(MI);
5710 case AMDGPU::G_PREFETCH:
5711 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5712 break;
5713 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP:
5714 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN:
5715 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5716 break;
5717 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
5718 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR: {
5719 unsigned Size = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI);
5720 unsigned PtrSize = TRI->getRegSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI);
5721 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5722 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize);
5723 break;
5724 }
5725 }
5726
5727 return getInstructionMapping(/*ID*/1, /*Cost*/1,
5728 OperandsMapping: getOperandsMapping(OpdsMapping),
5729 NumOperands: MI.getNumOperands());
5730}
5731