1//===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the RegisterBankInfo class for
10/// AMDGPU.
11///
12/// \par
13///
14/// AMDGPU has unique register bank constraints that require special high level
15/// strategies to deal with. There are two main true physical register banks
16/// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a
17/// sort of pseudo-register bank needed to represent SGPRs used in a vector
18/// boolean context. There is also the AGPR bank, which is a special purpose
19/// physical register bank present on some subtargets.
20///
21/// Copying from VGPR to SGPR is generally illegal, unless the value is known to
22/// be uniform. It is generally not valid to legalize operands by inserting
23/// copies as on other targets. Operations which require uniform, SGPR operands
24/// generally require scalarization by repeatedly executing the instruction,
25/// activating each set of lanes using a unique set of input values. This is
26/// referred to as a waterfall loop.
27///
28/// \par Booleans
29///
30/// Booleans (s1 values) requires special consideration. A vector compare result
31/// is naturally a bitmask with one bit per lane, in a 32 or 64-bit
32/// register. These are represented with the VCC bank. During selection, we need
33/// to be able to unambiguously go back from a register class to a register
34/// bank. To distinguish whether an SGPR should use the SGPR or VCC register
35/// bank, we need to know the use context type. An SGPR s1 value always means a
36/// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets
37/// SCC, which is a 1-bit unaddressable register. This will need to be copied to
38/// a 32-bit virtual register. Taken together, this means we need to adjust the
39/// type of boolean operations to be regbank legal. All SALU booleans need to be
40/// widened to 32-bits, and all VALU booleans need to be s1 values.
41///
42/// A noteworthy exception to the s1-means-vcc rule is for legalization artifact
43/// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc
44/// bank. A non-boolean source (such as a truncate from a 1-bit load from
45/// memory) will require a copy to the VCC bank which will require clearing the
46/// high bits and inserting a compare.
47///
48/// \par Constant bus restriction
49///
50/// VALU instructions have a limitation known as the constant bus
51/// restriction. Most VALU instructions can use SGPR operands, but may read at
52/// most 1 SGPR or constant literal value (this to 2 in gfx10 for most
53/// instructions). This is one unique SGPR, so the same SGPR may be used for
54/// multiple operands. From a register bank perspective, any combination of
55/// operands should be legal as an SGPR, but this is contextually dependent on
56/// the SGPR operands all being the same register. There is therefore optimal to
57/// choose the SGPR with the most uses to minimize the number of copies.
58///
59/// We avoid trying to solve this problem in RegBankSelect. Any VALU G_*
60/// operation should have its source operands all mapped to VGPRs (except for
61/// VCC), inserting copies from any SGPR operands. This the most trivial legal
62/// mapping. Anything beyond the simplest 1:1 instruction selection would be too
63/// complicated to solve here. Every optimization pattern or instruction
64/// selected to multiple outputs would have to enforce this rule, and there
65/// would be additional complexity in tracking this rule for every G_*
66/// operation. By forcing all inputs to VGPRs, it also simplifies the task of
67/// picking the optimal operand combination from a post-isel optimization pass.
68///
69//===----------------------------------------------------------------------===//
70
71#include "AMDGPURegisterBankInfo.h"
72
73#include "AMDGPUGlobalISelUtils.h"
74#include "AMDGPUInstrInfo.h"
75#include "AMDGPULaneMaskUtils.h"
76#include "GCNSubtarget.h"
77#include "SIMachineFunctionInfo.h"
78#include "SIRegisterInfo.h"
79#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
80#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
81#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
82#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
83#include "llvm/CodeGen/RegisterBank.h"
84#include "llvm/IR/IntrinsicsAMDGPU.h"
85
86#define GET_TARGET_REGBANK_IMPL
87#include "AMDGPUGenRegisterBank.inc"
88
89// This file will be TableGen'ed at some point.
90#include "AMDGPUGenRegisterBankInfo.def"
91
92using namespace llvm;
93using namespace MIPatternMatch;
94
95namespace {
96
97// Observer to apply a register bank to new registers created by LegalizerHelper.
98class ApplyRegBankMapping final : public GISelChangeObserver {
99private:
100 MachineIRBuilder &B;
101 const AMDGPURegisterBankInfo &RBI;
102 MachineRegisterInfo &MRI;
103 const RegisterBank *NewBank;
104 SmallVector<MachineInstr *, 4> NewInsts;
105
106public:
107 ApplyRegBankMapping(MachineIRBuilder &B, const AMDGPURegisterBankInfo &RBI_,
108 MachineRegisterInfo &MRI_, const RegisterBank *RB)
109 : B(B), RBI(RBI_), MRI(MRI_), NewBank(RB) {
110 assert(!B.isObservingChanges());
111 B.setChangeObserver(*this);
112 }
113
114 ~ApplyRegBankMapping() override {
115 for (MachineInstr *MI : NewInsts)
116 applyBank(MI&: *MI);
117
118 B.stopObservingChanges();
119 }
120
121 /// Set any registers that don't have a set register class or bank to SALU.
122 void applyBank(MachineInstr &MI) {
123 const unsigned Opc = MI.getOpcode();
124 if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT ||
125 Opc == AMDGPU::G_SEXT) {
126 // LegalizerHelper wants to use the basic legalization artifacts when
127 // widening etc. We don't handle selection with vcc in artifact sources,
128 // so we need to use a select instead to handle these properly.
129 Register DstReg = MI.getOperand(i: 0).getReg();
130 Register SrcReg = MI.getOperand(i: 1).getReg();
131 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI, TRI: *RBI.TRI);
132 if (SrcBank == &AMDGPU::VCCRegBank) {
133 const LLT S32 = LLT::scalar(SizeInBits: 32);
134 assert(MRI.getType(SrcReg) == LLT::scalar(1));
135 assert(MRI.getType(DstReg) == S32);
136 assert(NewBank == &AMDGPU::VGPRRegBank);
137
138 // Replace the extension with a select, which really uses the boolean
139 // source.
140 B.setInsertPt(MBB&: *MI.getParent(), II: MI);
141
142 auto True = B.buildConstant(Res: S32, Val: Opc == AMDGPU::G_SEXT ? -1 : 1);
143 auto False = B.buildConstant(Res: S32, Val: 0);
144 B.buildSelect(Res: DstReg, Tst: SrcReg, Op0: True, Op1: False);
145 MRI.setRegBank(Reg: True.getReg(Idx: 0), RegBank: *NewBank);
146 MRI.setRegBank(Reg: False.getReg(Idx: 0), RegBank: *NewBank);
147 MI.eraseFromParent();
148 }
149
150 assert(!MRI.getRegClassOrRegBank(DstReg));
151 MRI.setRegBank(Reg: DstReg, RegBank: *NewBank);
152 return;
153 }
154
155#ifndef NDEBUG
156 if (Opc == AMDGPU::G_TRUNC) {
157 Register DstReg = MI.getOperand(0).getReg();
158 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI);
159 assert(DstBank != &AMDGPU::VCCRegBank);
160 }
161#endif
162
163 for (MachineOperand &Op : MI.operands()) {
164 if (!Op.isReg())
165 continue;
166
167 // We may see physical registers if building a real MI
168 Register Reg = Op.getReg();
169 if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg))
170 continue;
171
172 const RegisterBank *RB = NewBank;
173 if (MRI.getType(Reg) == LLT::scalar(SizeInBits: 1)) {
174 assert(NewBank == &AMDGPU::VGPRRegBank &&
175 "s1 operands should only be used for vector bools");
176 assert((MI.getOpcode() != AMDGPU::G_TRUNC &&
177 MI.getOpcode() != AMDGPU::G_ANYEXT) &&
178 "not expecting legalization artifacts here");
179 RB = &AMDGPU::VCCRegBank;
180 }
181
182 MRI.setRegBank(Reg, RegBank: *RB);
183 }
184 }
185
186 void erasingInstr(MachineInstr &MI) override {}
187
188 void createdInstr(MachineInstr &MI) override {
189 // At this point, the instruction was just inserted and has no operands.
190 NewInsts.push_back(Elt: &MI);
191 }
192
193 void changingInstr(MachineInstr &MI) override {}
194 void changedInstr(MachineInstr &MI) override {
195 // FIXME: In principle we should probably add the instruction to NewInsts,
196 // but the way the LegalizerHelper uses the observer, we will always see the
197 // registers we need to set the regbank on also referenced in a new
198 // instruction.
199 }
200};
201
202} // anonymous namespace
203
204AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
205 : Subtarget(ST), TRI(Subtarget.getRegisterInfo()),
206 TII(Subtarget.getInstrInfo()) {
207
208 // HACK: Until this is fully tablegen'd.
209 static llvm::once_flag InitializeRegisterBankFlag;
210
211 static auto InitializeRegisterBankOnce = [this]() {
212 assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank &&
213 &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank &&
214 &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank);
215 (void)this;
216 };
217
218 llvm::call_once(flag&: InitializeRegisterBankFlag, F&: InitializeRegisterBankOnce);
219}
220
221static bool isVectorRegisterBank(const RegisterBank &Bank) {
222 unsigned BankID = Bank.getID();
223 return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID;
224}
225
226bool AMDGPURegisterBankInfo::isDivergentRegBank(const RegisterBank *RB) const {
227 return RB != &AMDGPU::SGPRRegBank;
228}
229
230unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
231 const RegisterBank &Src,
232 TypeSize Size) const {
233 // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane?
234 if (Dst.getID() == AMDGPU::SGPRRegBankID &&
235 (isVectorRegisterBank(Bank: Src) || Src.getID() == AMDGPU::VCCRegBankID)) {
236 return std::numeric_limits<unsigned>::max();
237 }
238
239 // Bool values are tricky, because the meaning is based on context. The SCC
240 // and VCC banks are for the natural scalar and vector conditions produced by
241 // a compare.
242 //
243 // Legalization doesn't know about the necessary context, so an s1 use may
244 // have been a truncate from an arbitrary value, in which case a copy (lowered
245 // as a compare with 0) needs to be inserted.
246 if (Size == 1 &&
247 (Dst.getID() == AMDGPU::SGPRRegBankID) &&
248 (isVectorRegisterBank(Bank: Src) ||
249 Src.getID() == AMDGPU::SGPRRegBankID ||
250 Src.getID() == AMDGPU::VCCRegBankID))
251 return std::numeric_limits<unsigned>::max();
252
253 // There is no direct copy between AGPRs.
254 if (Dst.getID() == AMDGPU::AGPRRegBankID &&
255 Src.getID() == AMDGPU::AGPRRegBankID)
256 return 4;
257
258 return RegisterBankInfo::copyCost(A: Dst, B: Src, Size);
259}
260
261unsigned AMDGPURegisterBankInfo::getBreakDownCost(
262 const ValueMapping &ValMapping,
263 const RegisterBank *CurBank) const {
264 // Check if this is a breakdown for G_LOAD to move the pointer from SGPR to
265 // VGPR.
266 // FIXME: Is there a better way to do this?
267 if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64)
268 return 10; // This is expensive.
269
270 assert(ValMapping.NumBreakDowns == 2 &&
271 ValMapping.BreakDown[0].Length == 32 &&
272 ValMapping.BreakDown[0].StartIdx == 0 &&
273 ValMapping.BreakDown[1].Length == 32 &&
274 ValMapping.BreakDown[1].StartIdx == 32 &&
275 ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank);
276
277 // 32-bit extract of a 64-bit value is just access of a subregister, so free.
278 // TODO: Cost of 0 hits assert, though it's not clear it's what we really
279 // want.
280
281 // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
282 // alignment restrictions, but this probably isn't important.
283 return 1;
284}
285
286const RegisterBank &
287AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
288 LLT Ty) const {
289 // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a
290 // VCC-like use.
291 if (TRI->isSGPRClass(RC: &RC)) {
292 // FIXME: This probably came from a copy from a physical register, which
293 // should be inferable from the copied to-type. We don't have many boolean
294 // physical register constraints so just assume a normal SGPR for now.
295 if (!Ty.isValid())
296 return AMDGPU::SGPRRegBank;
297
298 return Ty == LLT::scalar(SizeInBits: 1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
299 }
300
301 return TRI->isAGPRClass(RC: &RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank;
302}
303
304template <unsigned NumOps>
305RegisterBankInfo::InstructionMappings
306AMDGPURegisterBankInfo::addMappingFromTable(
307 const MachineInstr &MI, const MachineRegisterInfo &MRI,
308 const std::array<unsigned, NumOps> RegSrcOpIdx,
309 ArrayRef<OpRegBankEntry<NumOps>> Table) const {
310
311 InstructionMappings AltMappings;
312
313 SmallVector<const ValueMapping *, 10> Operands(MI.getNumOperands());
314
315 unsigned Sizes[NumOps];
316 for (unsigned I = 0; I < NumOps; ++I) {
317 Register Reg = MI.getOperand(RegSrcOpIdx[I]).getReg();
318 Sizes[I] = getSizeInBits(Reg, MRI, TRI: *TRI);
319 }
320
321 for (unsigned I = 0, E = MI.getNumExplicitDefs(); I != E; ++I) {
322 unsigned SizeI = getSizeInBits(Reg: MI.getOperand(i: I).getReg(), MRI, TRI: *TRI);
323 Operands[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SizeI);
324 }
325
326 // getInstrMapping's default mapping uses ID 1, so start at 2.
327 unsigned MappingID = 2;
328 for (const auto &Entry : Table) {
329 for (unsigned I = 0; I < NumOps; ++I) {
330 int OpIdx = RegSrcOpIdx[I];
331 Operands[OpIdx] = AMDGPU::getValueMapping(BankID: Entry.RegBanks[I], Size: Sizes[I]);
332 }
333
334 AltMappings.push_back(Elt: &getInstructionMapping(ID: MappingID++, Cost: Entry.Cost,
335 OperandsMapping: getOperandsMapping(OpdsMapping: Operands),
336 NumOperands: Operands.size()));
337 }
338
339 return AltMappings;
340}
341
342RegisterBankInfo::InstructionMappings
343AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(
344 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
345 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
346 case Intrinsic::amdgcn_readlane: {
347 static const OpRegBankEntry<3> Table[2] = {
348 // Perfectly legal.
349 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1 },
350
351 // Need a readfirstlane for the index.
352 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 }
353 };
354
355 const std::array<unsigned, 3> RegSrcOpIdx = { ._M_elems: { 0, 2, 3 } };
356 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, Table);
357 }
358 case Intrinsic::amdgcn_writelane: {
359 static const OpRegBankEntry<4> Table[4] = {
360 // Perfectly legal.
361 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1 },
362
363 // Need readfirstlane of first op
364 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 },
365
366 // Need readfirstlane of second op
367 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 },
368
369 // Need readfirstlane of both ops
370 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 3 }
371 };
372
373 // rsrc, voffset, offset
374 const std::array<unsigned, 4> RegSrcOpIdx = { ._M_elems: { 0, 2, 3, 4 } };
375 return addMappingFromTable<4>(MI, MRI, RegSrcOpIdx, Table);
376 }
377 default:
378 return RegisterBankInfo::getInstrAlternativeMappings(MI);
379 }
380}
381
382RegisterBankInfo::InstructionMappings
383AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(
384 const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
385
386 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
387 case Intrinsic::amdgcn_s_buffer_load:
388 case Intrinsic::amdgcn_ptr_s_buffer_load: {
389 static const OpRegBankEntry<2> Table[4] = {
390 // Perfectly legal.
391 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1 },
392
393 // Only need 1 register in loop
394 { .RegBanks: { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 300 },
395
396 // Have to waterfall the resource.
397 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, .Cost: 1000 },
398
399 // Have to waterfall the resource, and the offset.
400 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1500 }
401 };
402
403 // rsrc, offset
404 const std::array<unsigned, 2> RegSrcOpIdx = { ._M_elems: { 2, 3 } };
405 return addMappingFromTable<2>(MI, MRI, RegSrcOpIdx, Table);
406 }
407 case Intrinsic::amdgcn_ds_ordered_add:
408 case Intrinsic::amdgcn_ds_ordered_swap: {
409 // VGPR = M0, VGPR
410 static const OpRegBankEntry<3> Table[2] = {
411 // Perfectly legal.
412 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 1 },
413
414 // Need a readfirstlane for m0
415 { .RegBanks: { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, .Cost: 2 }
416 };
417
418 const std::array<unsigned, 3> RegSrcOpIdx = { ._M_elems: { 0, 2, 3 } };
419 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, Table);
420 }
421 case Intrinsic::amdgcn_s_sendmsg:
422 case Intrinsic::amdgcn_s_sendmsghalt: {
423 // FIXME: Should have no register for immediate
424 static const OpRegBankEntry<1> Table[2] = {
425 // Perfectly legal.
426 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 },
427
428 // Need readlane
429 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 3 }
430 };
431
432 const std::array<unsigned, 1> RegSrcOpIdx = { ._M_elems: { 2 } };
433 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx, Table);
434 }
435 default:
436 return RegisterBankInfo::getInstrAlternativeMappings(MI);
437 }
438}
439
440// FIXME: Returns uniform if there's no source value information. This is
441// probably wrong.
442bool AMDGPURegisterBankInfo::isScalarLoadLegal(const MachineInstr &MI) const {
443 if (!MI.hasOneMemOperand())
444 return false;
445
446 const MachineMemOperand *MMO = *MI.memoperands_begin();
447 const unsigned AS = MMO->getAddrSpace();
448 const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS ||
449 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
450 const unsigned MemSize = 8 * MMO->getSize().getValue();
451
452 // Require 4-byte alignment.
453 return (MMO->getAlign() >= Align(4) ||
454 (Subtarget.hasScalarSubwordLoads() &&
455 ((MemSize == 16 && MMO->getAlign() >= Align(2)) ||
456 (MemSize == 8 && MMO->getAlign() >= Align(1))))) &&
457 // Can't do a scalar atomic load.
458 !MMO->isAtomic() &&
459 // Don't use scalar loads for volatile accesses to non-constant address
460 // spaces.
461 (IsConst || !MMO->isVolatile()) &&
462 // Memory must be known constant, or not written before this load.
463 (IsConst || MMO->isInvariant() || (MMO->getFlags() & MONoClobber)) &&
464 AMDGPU::isUniformMMO(MMO);
465}
466
467RegisterBankInfo::InstructionMappings
468AMDGPURegisterBankInfo::getInstrAlternativeMappings(
469 const MachineInstr &MI) const {
470
471 const MachineFunction &MF = *MI.getMF();
472 const MachineRegisterInfo &MRI = MF.getRegInfo();
473
474
475 InstructionMappings AltMappings;
476 switch (MI.getOpcode()) {
477 case TargetOpcode::G_CONSTANT:
478 case TargetOpcode::G_IMPLICIT_DEF: {
479 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
480 if (Size == 1) {
481 static const OpRegBankEntry<1> Table[3] = {
482 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 1 },
483 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 },
484 { .RegBanks: { AMDGPU::VCCRegBankID }, .Cost: 1 }
485 };
486
487 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx: {._M_elems: { 0 }}, Table);
488 }
489
490 [[fallthrough]];
491 }
492 case TargetOpcode::G_FCONSTANT:
493 case TargetOpcode::G_FRAME_INDEX:
494 case TargetOpcode::G_GLOBAL_VALUE: {
495 static const OpRegBankEntry<1> Table[2] = {
496 { .RegBanks: { AMDGPU::VGPRRegBankID }, .Cost: 1 },
497 { .RegBanks: { AMDGPU::SGPRRegBankID }, .Cost: 1 }
498 };
499
500 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx: {._M_elems: { 0 }}, Table);
501 }
502 case TargetOpcode::G_AND:
503 case TargetOpcode::G_OR:
504 case TargetOpcode::G_XOR: {
505 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
506
507 if (Size == 1) {
508 // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0.
509 const InstructionMapping &SCCMapping = getInstructionMapping(
510 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
511 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32),
512 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32),
513 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32)}),
514 NumOperands: 3); // Num Operands
515 AltMappings.push_back(Elt: &SCCMapping);
516
517 const InstructionMapping &VCCMapping0 = getInstructionMapping(
518 ID: 2, Cost: 1, OperandsMapping: getOperandsMapping(
519 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size),
520 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size),
521 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size)}),
522 NumOperands: 3); // Num Operands
523 AltMappings.push_back(Elt: &VCCMapping0);
524 return AltMappings;
525 }
526
527 if (Size != 64)
528 break;
529
530 const InstructionMapping &SSMapping = getInstructionMapping(
531 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
532 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
533 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
534 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size)}),
535 NumOperands: 3); // Num Operands
536 AltMappings.push_back(Elt: &SSMapping);
537
538 const InstructionMapping &VVMapping = getInstructionMapping(
539 ID: 2, Cost: 2, OperandsMapping: getOperandsMapping(
540 OpdsMapping: {AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
541 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
542 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size)}),
543 NumOperands: 3); // Num Operands
544 AltMappings.push_back(Elt: &VVMapping);
545 break;
546 }
547 case TargetOpcode::G_LOAD:
548 case TargetOpcode::G_ZEXTLOAD:
549 case TargetOpcode::G_SEXTLOAD: {
550 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
551 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
552 unsigned PtrSize = PtrTy.getSizeInBits();
553 unsigned AS = PtrTy.getAddressSpace();
554
555 if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
556 AS != AMDGPUAS::PRIVATE_ADDRESS) &&
557 isScalarLoadLegal(MI)) {
558 const InstructionMapping &SSMapping = getInstructionMapping(
559 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
560 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
561 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: PtrSize)}),
562 NumOperands: 2); // Num Operands
563 AltMappings.push_back(Elt: &SSMapping);
564 }
565
566 const InstructionMapping &VVMapping = getInstructionMapping(
567 ID: 2, Cost: 1,
568 OperandsMapping: getOperandsMapping(
569 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
570 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize)}),
571 NumOperands: 2); // Num Operands
572 AltMappings.push_back(Elt: &VVMapping);
573
574 // It may be possible to have a vgpr = load sgpr mapping here, because
575 // the mubuf instructions support this kind of load, but probably for only
576 // gfx7 and older. However, the addressing mode matching in the instruction
577 // selector should be able to do a better job of detecting and selecting
578 // these kinds of loads from the vgpr = load vgpr mapping.
579
580 return AltMappings;
581
582 }
583 case TargetOpcode::G_SELECT: {
584 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
585 const InstructionMapping &SSMapping = getInstructionMapping(ID: 1, Cost: 1,
586 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
587 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1),
588 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
589 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size)}),
590 NumOperands: 4); // Num Operands
591 AltMappings.push_back(Elt: &SSMapping);
592
593 const InstructionMapping &VVMapping = getInstructionMapping(ID: 2, Cost: 1,
594 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
595 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1),
596 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size),
597 AMDGPU::getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size)}),
598 NumOperands: 4); // Num Operands
599 AltMappings.push_back(Elt: &VVMapping);
600
601 return AltMappings;
602 }
603 case TargetOpcode::G_UADDE:
604 case TargetOpcode::G_USUBE:
605 case TargetOpcode::G_SADDE:
606 case TargetOpcode::G_SSUBE: {
607 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
608 const InstructionMapping &SSMapping = getInstructionMapping(ID: 1, Cost: 1,
609 OperandsMapping: getOperandsMapping(
610 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
611 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1),
612 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
613 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size),
614 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1)}),
615 NumOperands: 5); // Num Operands
616 AltMappings.push_back(Elt: &SSMapping);
617
618 const InstructionMapping &VVMapping = getInstructionMapping(ID: 2, Cost: 1,
619 OperandsMapping: getOperandsMapping(OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
620 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1),
621 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
622 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size),
623 AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1)}),
624 NumOperands: 5); // Num Operands
625 AltMappings.push_back(Elt: &VVMapping);
626 return AltMappings;
627 }
628 case AMDGPU::G_BRCOND: {
629 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
630
631 // TODO: Change type to 32 for scalar
632 const InstructionMapping &SMapping = getInstructionMapping(
633 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
634 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1), nullptr}),
635 NumOperands: 2); // Num Operands
636 AltMappings.push_back(Elt: &SMapping);
637
638 const InstructionMapping &VMapping = getInstructionMapping(
639 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(
640 OpdsMapping: {AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1), nullptr }),
641 NumOperands: 2); // Num Operands
642 AltMappings.push_back(Elt: &VMapping);
643 return AltMappings;
644 }
645 case AMDGPU::G_INTRINSIC:
646 case AMDGPU::G_INTRINSIC_CONVERGENT:
647 return getInstrAlternativeMappingsIntrinsic(MI, MRI);
648 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
649 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
650 return getInstrAlternativeMappingsIntrinsicWSideEffects(MI, MRI);
651 default:
652 break;
653 }
654 return RegisterBankInfo::getInstrAlternativeMappings(MI);
655}
656
657void AMDGPURegisterBankInfo::split64BitValueForMapping(
658 MachineIRBuilder &B,
659 SmallVector<Register, 2> &Regs,
660 LLT HalfTy,
661 Register Reg) const {
662 assert(HalfTy.getSizeInBits() == 32);
663 MachineRegisterInfo *MRI = B.getMRI();
664 Register LoLHS = MRI->createGenericVirtualRegister(Ty: HalfTy);
665 Register HiLHS = MRI->createGenericVirtualRegister(Ty: HalfTy);
666 const RegisterBank *Bank = getRegBank(Reg, MRI: *MRI, TRI: *TRI);
667 MRI->setRegBank(Reg: LoLHS, RegBank: *Bank);
668 MRI->setRegBank(Reg: HiLHS, RegBank: *Bank);
669
670 Regs.push_back(Elt: LoLHS);
671 Regs.push_back(Elt: HiLHS);
672
673 B.buildInstr(Opcode: AMDGPU::G_UNMERGE_VALUES)
674 .addDef(RegNo: LoLHS)
675 .addDef(RegNo: HiLHS)
676 .addUse(RegNo: Reg);
677}
678
679/// Replace the current type each register in \p Regs has with \p NewTy
680static void setRegsToType(MachineRegisterInfo &MRI, ArrayRef<Register> Regs,
681 LLT NewTy) {
682 for (Register Reg : Regs) {
683 assert(MRI.getType(Reg).getSizeInBits() == NewTy.getSizeInBits());
684 MRI.setType(VReg: Reg, Ty: NewTy);
685 }
686}
687
688static LLT getHalfSizedType(LLT Ty) {
689 if (Ty.isVector()) {
690 assert(Ty.getElementCount().isKnownMultipleOf(2));
691 return LLT::scalarOrVector(EC: Ty.getElementCount().divideCoefficientBy(RHS: 2),
692 ScalarTy: Ty.getElementType());
693 }
694
695 assert(Ty.getScalarSizeInBits() % 2 == 0);
696 return LLT::scalar(SizeInBits: Ty.getScalarSizeInBits() / 2);
697}
698
699// Build one or more V_READFIRSTLANE_B32 instructions to move the given vector
700// source value into a scalar register.
701Register AMDGPURegisterBankInfo::buildReadFirstLane(MachineIRBuilder &B,
702 MachineRegisterInfo &MRI,
703 Register Src) const {
704 LLT Ty = MRI.getType(Reg: Src);
705 const RegisterBank *Bank = getRegBank(Reg: Src, MRI, TRI: *TRI);
706
707 if (Bank == &AMDGPU::SGPRRegBank)
708 return Src;
709
710 unsigned Bits = Ty.getSizeInBits();
711 assert(Bits % 32 == 0);
712
713 if (Bank != &AMDGPU::VGPRRegBank) {
714 // We need to copy from AGPR to VGPR
715 Src = B.buildCopy(Res: Ty, Op: Src).getReg(Idx: 0);
716 MRI.setRegBank(Reg: Src, RegBank: AMDGPU::VGPRRegBank);
717 }
718
719 LLT S32 = LLT::scalar(SizeInBits: 32);
720 unsigned NumParts = Bits / 32;
721 SmallVector<Register, 8> SrcParts;
722 SmallVector<Register, 8> DstParts;
723
724 if (Bits == 32) {
725 SrcParts.push_back(Elt: Src);
726 } else {
727 auto Unmerge = B.buildUnmerge(Res: S32, Op: Src);
728 for (unsigned i = 0; i < NumParts; ++i)
729 SrcParts.push_back(Elt: Unmerge.getReg(Idx: i));
730 }
731
732 for (unsigned i = 0; i < NumParts; ++i) {
733 Register SrcPart = SrcParts[i];
734 Register DstPart = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
735 MRI.setType(VReg: DstPart, Ty: NumParts == 1 ? Ty : S32);
736
737 const TargetRegisterClass *Constrained =
738 constrainGenericRegister(Reg: SrcPart, RC: AMDGPU::VGPR_32RegClass, MRI);
739 (void)Constrained;
740 assert(Constrained && "Failed to constrain readfirstlane src reg");
741
742 B.buildInstr(Opc: AMDGPU::V_READFIRSTLANE_B32, DstOps: {DstPart}, SrcOps: {SrcPart});
743
744 DstParts.push_back(Elt: DstPart);
745 }
746
747 if (Bits == 32)
748 return DstParts[0];
749
750 Register Dst = B.buildMergeLikeInstr(Res: Ty, Ops: DstParts).getReg(Idx: 0);
751 MRI.setRegBank(Reg: Dst, RegBank: AMDGPU::SGPRRegBank);
752 return Dst;
753}
754
755/// Legalize instruction \p MI where operands in \p OpIndices must be SGPRs. If
756/// any of the required SGPR operands are VGPRs, perform a waterfall loop to
757/// execute the instruction for each unique combination of values in all lanes
758/// in the wave. The block will be split such that rest of the instructions are
759/// moved to a new block.
760///
761/// Essentially performs this loop:
762//
763/// Save Execution Mask
764/// For (Lane : Wavefront) {
765/// Enable Lane, Disable all other lanes
766/// SGPR = read SGPR value for current lane from VGPR
767/// VGPRResult[Lane] = use_op SGPR
768/// }
769/// Restore Execution Mask
770///
771/// There is additional complexity to try for compare values to identify the
772/// unique values used.
773bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
774 MachineIRBuilder &B, iterator_range<MachineBasicBlock::iterator> Range,
775 SmallSet<Register, 4> &SGPROperandRegs) const {
776 // Track use registers which have already been expanded with a readfirstlane
777 // sequence. This may have multiple uses if moving a sequence.
778 DenseMap<Register, Register> WaterfalledRegMap;
779
780 MachineBasicBlock &MBB = B.getMBB();
781 MachineFunction *MF = &B.getMF();
782
783 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
784 const AMDGPU::LaneMaskConstants &LMC =
785 AMDGPU::LaneMaskConstants::get(ST: Subtarget);
786
787#ifndef NDEBUG
788 const int OrigRangeSize = std::distance(Range.begin(), Range.end());
789#endif
790
791 MachineRegisterInfo &MRI = *B.getMRI();
792 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
793 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
794
795 // Don't bother using generic instructions/registers for the exec mask.
796 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF)
797 .addDef(RegNo: InitSaveExecReg);
798
799 Register PhiExec = MRI.createVirtualRegister(RegClass: WaveRC);
800 Register NewExec = MRI.createVirtualRegister(RegClass: WaveRC);
801
802 // To insert the loop we need to split the block. Move everything before this
803 // point to a new block, and insert a new empty block before this instruction.
804 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
805 MachineBasicBlock *BodyBB = MF->CreateMachineBasicBlock();
806 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
807 MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock();
808 MachineFunction::iterator MBBI(MBB);
809 ++MBBI;
810 MF->insert(MBBI, MBB: LoopBB);
811 MF->insert(MBBI, MBB: BodyBB);
812 MF->insert(MBBI, MBB: RestoreExecBB);
813 MF->insert(MBBI, MBB: RemainderBB);
814
815 LoopBB->addSuccessor(Succ: BodyBB);
816 BodyBB->addSuccessor(Succ: RestoreExecBB);
817 BodyBB->addSuccessor(Succ: LoopBB);
818
819 // Move the rest of the block into a new block.
820 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
821 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Range.end(), To: MBB.end());
822
823 MBB.addSuccessor(Succ: LoopBB);
824 RestoreExecBB->addSuccessor(Succ: RemainderBB);
825
826 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
827
828 B.buildInstr(Opcode: TargetOpcode::PHI)
829 .addDef(RegNo: PhiExec)
830 .addReg(RegNo: InitSaveExecReg)
831 .addMBB(MBB: &MBB)
832 .addReg(RegNo: NewExec)
833 .addMBB(MBB: BodyBB);
834
835 const DebugLoc &DL = B.getDL();
836
837 MachineInstr &FirstInst = *Range.begin();
838
839 // Move the instruction into the loop body. Note we moved everything after
840 // Range.end() already into a new block, so Range.end() is no longer valid.
841 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: Range.begin(), To: MBB.end());
842
843 // Figure out the iterator range after splicing the instructions.
844 MachineBasicBlock::iterator NewBegin = FirstInst.getIterator();
845 auto NewEnd = BodyBB->end();
846
847 B.setMBB(*LoopBB);
848
849 LLT S1 = LLT::scalar(SizeInBits: 1);
850 Register CondReg;
851
852 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
853
854 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
855 for (MachineOperand &Op : MI.all_uses()) {
856 Register OldReg = Op.getReg();
857 if (!SGPROperandRegs.count(V: OldReg))
858 continue;
859
860 // See if we already processed this register in another instruction in the
861 // sequence.
862 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
863 if (OldVal != WaterfalledRegMap.end()) {
864 Op.setReg(OldVal->second);
865 continue;
866 }
867
868 Register OpReg = Op.getReg();
869 LLT OpTy = MRI.getType(Reg: OpReg);
870
871 const RegisterBank *OpBank = getRegBank(Reg: OpReg, MRI, TRI: *TRI);
872 if (OpBank != &AMDGPU::VGPRRegBank) {
873 // Insert copy from AGPR to VGPR before the loop.
874 B.setMBB(MBB);
875 OpReg = B.buildCopy(Res: OpTy, Op: OpReg).getReg(Idx: 0);
876 MRI.setRegBank(Reg: OpReg, RegBank: AMDGPU::VGPRRegBank);
877 B.setMBB(*LoopBB);
878 }
879
880 Register CurrentLaneReg = buildReadFirstLane(B, MRI, Src: OpReg);
881
882 // Build the comparison(s).
883 unsigned OpSize = OpTy.getSizeInBits();
884 bool Is64 = OpSize % 64 == 0;
885 unsigned PartSize = Is64 ? 64 : 32;
886 LLT PartTy = LLT::scalar(SizeInBits: PartSize);
887 unsigned NumParts = OpSize / PartSize;
888 SmallVector<Register, 8> OpParts;
889 SmallVector<Register, 8> CurrentLaneParts;
890
891 if (NumParts == 1) {
892 OpParts.push_back(Elt: OpReg);
893 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
894 } else {
895 auto UnmergeOp = B.buildUnmerge(Res: PartTy, Op: OpReg);
896 auto UnmergeCurrentLane = B.buildUnmerge(Res: PartTy, Op: CurrentLaneReg);
897 for (unsigned i = 0; i < NumParts; ++i) {
898 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
899 CurrentLaneParts.push_back(Elt: UnmergeCurrentLane.getReg(Idx: i));
900 MRI.setRegBank(Reg: OpParts[i], RegBank: AMDGPU::VGPRRegBank);
901 MRI.setRegBank(Reg: CurrentLaneParts[i], RegBank: AMDGPU::SGPRRegBank);
902 }
903 }
904
905 for (unsigned i = 0; i < NumParts; ++i) {
906 auto CmpReg = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: S1, Op0: CurrentLaneParts[i],
907 Op1: OpParts[i]).getReg(Idx: 0);
908 MRI.setRegBank(Reg: CmpReg, RegBank: AMDGPU::VCCRegBank);
909
910 if (!CondReg) {
911 CondReg = CmpReg;
912 } else {
913 CondReg = B.buildAnd(Dst: S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
914 MRI.setRegBank(Reg: CondReg, RegBank: AMDGPU::VCCRegBank);
915 }
916 }
917
918 Op.setReg(CurrentLaneReg);
919
920 // Make sure we don't re-process this register again.
921 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
922 }
923 }
924
925 // The ballot becomes a no-op during instruction selection.
926 CondReg = B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot,
927 Res: {LLT::scalar(SizeInBits: Subtarget.isWave32() ? 32 : 64)})
928 .addReg(RegNo: CondReg)
929 .getReg(Idx: 0);
930 MRI.setRegClass(Reg: CondReg, RC: WaveRC);
931
932 // Update EXEC, save the original EXEC value to VCC.
933 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
934 .addDef(RegNo: NewExec)
935 .addReg(RegNo: CondReg, Flags: RegState::Kill);
936
937 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
938
939 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
940
941 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
942 B.buildInstr(Opcode: LMC.XorTermOpc)
943 .addDef(RegNo: LMC.ExecReg)
944 .addReg(RegNo: LMC.ExecReg)
945 .addReg(RegNo: NewExec);
946
947 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
948 // s_cbranch_scc0?
949
950 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
951 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
952
953 // Save the EXEC mask before the loop.
954 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExecReg)
955 .addReg(RegNo: LMC.ExecReg);
956
957 // Restore the EXEC mask after the loop.
958 B.setMBB(*RestoreExecBB);
959 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
960
961 // Set the insert point after the original instruction, so any new
962 // instructions will be in the remainder.
963 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
964
965 return true;
966}
967
968// Return any unique registers used by \p MI at \p OpIndices that need to be
969// handled in a waterfall loop. Returns these registers in \p
970// SGPROperandRegs. Returns true if there are any operands to handle and a
971// waterfall loop is necessary.
972bool AMDGPURegisterBankInfo::collectWaterfallOperands(
973 SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI,
974 MachineRegisterInfo &MRI, ArrayRef<unsigned> OpIndices) const {
975 for (unsigned Op : OpIndices) {
976 assert(MI.getOperand(Op).isUse());
977 Register Reg = MI.getOperand(i: Op).getReg();
978 const RegisterBank *OpBank = getRegBank(Reg, MRI, TRI: *TRI);
979 if (OpBank->getID() != AMDGPU::SGPRRegBankID)
980 SGPROperandRegs.insert(V: Reg);
981 }
982
983 // No operands need to be replaced, so no need to loop.
984 return !SGPROperandRegs.empty();
985}
986
987bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
988 MachineIRBuilder &B, MachineInstr &MI, ArrayRef<unsigned> OpIndices) const {
989 // Use a set to avoid extra readfirstlanes in the case where multiple operands
990 // are the same register.
991 SmallSet<Register, 4> SGPROperandRegs;
992
993 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI&: *B.getMRI(), OpIndices))
994 return false;
995
996 MachineBasicBlock::iterator I = MI.getIterator();
997 return executeInWaterfallLoop(B, Range: make_range(x: I, y: std::next(x: I)),
998 SGPROperandRegs);
999}
1000
1001// Legalize an operand that must be an SGPR by inserting a readfirstlane.
1002void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane(
1003 MachineIRBuilder &B, MachineInstr &MI, unsigned OpIdx) const {
1004 Register Reg = MI.getOperand(i: OpIdx).getReg();
1005 MachineRegisterInfo &MRI = *B.getMRI();
1006 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
1007 if (Bank == &AMDGPU::SGPRRegBank)
1008 return;
1009
1010 Reg = buildReadFirstLane(B, MRI, Src: Reg);
1011 MI.getOperand(i: OpIdx).setReg(Reg);
1012}
1013
1014/// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the
1015/// rest will be in the remainder.
1016static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) {
1017 unsigned TotalSize = Ty.getSizeInBits();
1018 if (!Ty.isVector())
1019 return {LLT::scalar(SizeInBits: FirstSize), LLT::scalar(SizeInBits: TotalSize - FirstSize)};
1020
1021 LLT EltTy = Ty.getElementType();
1022 unsigned EltSize = EltTy.getSizeInBits();
1023 assert(FirstSize % EltSize == 0);
1024
1025 unsigned FirstPartNumElts = FirstSize / EltSize;
1026 unsigned RemainderElts = (TotalSize - FirstSize) / EltSize;
1027
1028 return {LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: FirstPartNumElts), ScalarTy: EltTy),
1029 LLT::scalarOrVector(EC: ElementCount::getFixed(MinVal: RemainderElts), ScalarTy: EltTy)};
1030}
1031
1032static LLT widen96To128(LLT Ty) {
1033 if (!Ty.isVector())
1034 return LLT::scalar(SizeInBits: 128);
1035
1036 LLT EltTy = Ty.getElementType();
1037 assert(128 % EltTy.getSizeInBits() == 0);
1038 return LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1039}
1040
1041bool AMDGPURegisterBankInfo::applyMappingLoad(
1042 MachineIRBuilder &B,
1043 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1044 MachineInstr &MI) const {
1045 MachineRegisterInfo &MRI = *B.getMRI();
1046 Register DstReg = MI.getOperand(i: 0).getReg();
1047 const LLT LoadTy = MRI.getType(Reg: DstReg);
1048 unsigned LoadSize = LoadTy.getSizeInBits();
1049 MachineMemOperand *MMO = *MI.memoperands_begin();
1050 const unsigned MaxNonSmrdLoadSize = 128;
1051
1052 const RegisterBank *DstBank =
1053 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1054 if (DstBank == &AMDGPU::SGPRRegBank) {
1055 // There are some special cases that we need to look at for 32 bit and 96
1056 // bit SGPR loads otherwise we have nothing to do.
1057 if (LoadSize != 32 && (LoadSize != 96 || Subtarget.hasScalarDwordx3Loads()))
1058 return false;
1059
1060 const unsigned MemSize = 8 * MMO->getSize().getValue();
1061 // Scalar loads of size 8 or 16 bit with proper alignment may be widened to
1062 // 32 bit. Check to see if we need to widen the memory access, 8 or 16 bit
1063 // scalar loads should have a load size of 32 but memory access size of less
1064 // than 32.
1065 if (LoadSize == 32 &&
1066 (MemSize == 32 || LoadTy.isVector() || !isScalarLoadLegal(MI)))
1067 return false;
1068
1069 if (LoadSize == 32 &&
1070 ((MemSize == 8 && MMO->getAlign() >= Align(1)) ||
1071 (MemSize == 16 && MMO->getAlign() >= Align(2))) &&
1072 isScalarLoadLegal(MI) &&
1073 Subtarget.getGeneration() >= AMDGPUSubtarget::GFX12)
1074 return false;
1075
1076 Register PtrReg = MI.getOperand(i: 1).getReg();
1077
1078 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
1079
1080 if (LoadSize == 32) {
1081 // This is an extending load from a sub-dword size. Widen the memory
1082 // access size to 4 bytes and clear the extra high bits appropriately
1083 const LLT S32 = LLT::scalar(SizeInBits: 32);
1084 if (MI.getOpcode() == AMDGPU::G_SEXTLOAD) {
1085 // Must extend the sign bit into higher bits for a G_SEXTLOAD
1086 auto WideLoad = B.buildLoadFromOffset(Dst: S32, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1087 B.buildSExtInReg(Res: MI.getOperand(i: 0), Op: WideLoad, ImmOp: MemSize);
1088 } else if (MI.getOpcode() == AMDGPU::G_ZEXTLOAD) {
1089 // Must extend zero into higher bits with an AND for a G_ZEXTLOAD
1090 auto WideLoad = B.buildLoadFromOffset(Dst: S32, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1091 B.buildZExtInReg(Res: MI.getOperand(i: 0), Op: WideLoad, ImmOp: MemSize);
1092 } else
1093 // We do not need to touch the higher bits for regular loads.
1094 B.buildLoadFromOffset(Dst: MI.getOperand(i: 0), BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1095 } else {
1096 // 96-bit loads are only available for vector loads. We need to split this
1097 // into a 64-bit part, and 32 (unless we can widen to a 128-bit load).
1098 if (MMO->getAlign() < Align(16)) {
1099 LegalizerHelper Helper(B.getMF(), ApplyBank, B);
1100 LLT Part64, Part32;
1101 std::tie(args&: Part64, args&: Part32) = splitUnequalType(Ty: LoadTy, FirstSize: 64);
1102 if (Helper.reduceLoadStoreWidth(MI&: cast<GAnyLoad>(Val&: MI), TypeIdx: 0, NarrowTy: Part64) !=
1103 LegalizerHelper::Legalized)
1104 return false;
1105 return true;
1106 }
1107 LLT WiderTy = widen96To128(Ty: LoadTy);
1108 auto WideLoad = B.buildLoadFromOffset(Dst: WiderTy, BasePtr: PtrReg, BaseMMO&: *MMO, Offset: 0);
1109 if (WiderTy.isScalar()) {
1110 B.buildTrunc(Res: MI.getOperand(i: 0), Op: WideLoad);
1111 } else {
1112 B.buildDeleteTrailingVectorElements(Res: MI.getOperand(i: 0).getReg(),
1113 Op0: WideLoad);
1114 }
1115 }
1116
1117 MI.eraseFromParent();
1118 return true;
1119 }
1120
1121 // 128-bit loads are supported for all instruction types.
1122 if (LoadSize <= MaxNonSmrdLoadSize)
1123 return false;
1124
1125 SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
1126
1127 if (SrcRegs.empty())
1128 SrcRegs.push_back(Elt: MI.getOperand(i: 1).getReg());
1129
1130 // RegBankSelect only emits scalar types, so we need to reset the pointer
1131 // operand to a pointer type.
1132 Register BasePtrReg = SrcRegs[0];
1133 LLT PtrTy = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
1134 MRI.setType(VReg: BasePtrReg, Ty: PtrTy);
1135
1136 // The following are the loads not splitted enough during legalization
1137 // because it was not clear they are smem-load or vmem-load
1138 if (AMDGPU::isExtendedGlobalAddrSpace(AS: MMO->getAddrSpace()) ||
1139 MMO->getAddrSpace() == AMDGPUAS::BUFFER_RESOURCE) {
1140 assert(LoadSize % MaxNonSmrdLoadSize == 0);
1141 unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize;
1142 const LLT LoadSplitTy = LoadTy.divide(Factor: NumSplitParts);
1143 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank);
1144 LegalizerHelper Helper(B.getMF(), O, B);
1145 if (LoadTy.isVector()) {
1146 if (Helper.fewerElementsVector(MI, TypeIdx: 0, NarrowTy: LoadSplitTy) !=
1147 LegalizerHelper::Legalized)
1148 return false;
1149 } else {
1150 if (Helper.narrowScalar(MI, TypeIdx: 0, NarrowTy: LoadSplitTy) != LegalizerHelper::Legalized)
1151 return false;
1152 }
1153 }
1154
1155 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
1156 return true;
1157}
1158
1159bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc(
1160 MachineIRBuilder &B,
1161 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1162 MachineInstr &MI) const {
1163 MachineRegisterInfo &MRI = *B.getMRI();
1164 const MachineFunction &MF = B.getMF();
1165 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1166 const auto &TFI = *ST.getFrameLowering();
1167
1168 // Guard in case the stack growth direction ever changes with scratch
1169 // instructions.
1170 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1171 "Stack grows upwards for AMDGPU");
1172
1173 Register Dst = MI.getOperand(i: 0).getReg();
1174 Register AllocSize = MI.getOperand(i: 1).getReg();
1175 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1176
1177 // When using flat-scratch, the stack offset is unscaled.
1178 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1179 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1180
1181 const RegisterBank *SizeBank = getRegBank(Reg: AllocSize, MRI, TRI: *TRI);
1182
1183 if (SizeBank != &AMDGPU::SGPRRegBank) {
1184 auto WaveReduction =
1185 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {LLT::scalar(SizeInBits: 32)})
1186 .addUse(RegNo: AllocSize)
1187 .addImm(Val: 0);
1188 AllocSize = WaveReduction.getReg(Idx: 0);
1189 }
1190
1191 LLT PtrTy = MRI.getType(Reg: Dst);
1192 LLT IntPtrTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
1193
1194 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1195 Register SPReg = Info->getStackPtrOffsetReg();
1196 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
1197
1198 Register ScaledSize = AllocSize;
1199 if (!HasFlatScratch) {
1200 auto WaveSize = B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: WavefrontSizeLog2);
1201 ScaledSize = B.buildShl(Dst: IntPtrTy, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1202 }
1203
1204 auto OldSP = B.buildCopy(Res: PtrTy, Op: SPReg);
1205 if (Alignment > TFI.getStackAlign()) {
1206 const uint64_t ScaledAlignment =
1207 HasFlatScratch ? Alignment.value()
1208 : (Alignment.value() << WavefrontSizeLog2);
1209 const uint64_t StackAlignMask = ScaledAlignment - 1;
1210 auto Tmp1 = B.buildPtrAdd(Res: PtrTy, Op0: OldSP,
1211 Op1: B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: StackAlignMask));
1212 B.buildMaskLowPtrBits(Res: Dst, Op0: Tmp1,
1213 NumBits: (HasFlatScratch
1214 ? Log2(A: Alignment)
1215 : Log2(A: Alignment) + WavefrontSizeLog2));
1216 } else {
1217 B.buildCopy(Res: Dst, Op: OldSP);
1218 }
1219 auto PtrAdd = B.buildPtrAdd(Res: PtrTy, Op0: Dst, Op1: ScaledSize);
1220 B.buildCopy(Res: SPReg, Op: PtrAdd);
1221 MI.eraseFromParent();
1222 return true;
1223}
1224
1225bool AMDGPURegisterBankInfo::applyMappingImage(
1226 MachineIRBuilder &B, MachineInstr &MI,
1227 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
1228 int RsrcIdx) const {
1229 const int NumDefs = MI.getNumExplicitDefs();
1230
1231 // The reported argument index is relative to the IR intrinsic call arguments,
1232 // so we need to shift by the number of defs and the intrinsic ID.
1233 RsrcIdx += NumDefs + 1;
1234
1235 // Insert copies to VGPR arguments.
1236 applyDefaultMapping(OpdMapper);
1237
1238 // Fixup any SGPR arguments.
1239 SmallVector<unsigned, 4> SGPRIndexes;
1240 for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) {
1241 if (!MI.getOperand(i: I).isReg())
1242 continue;
1243
1244 // If this intrinsic has a sampler, it immediately follows rsrc.
1245 if (I == RsrcIdx || I == RsrcIdx + 1)
1246 SGPRIndexes.push_back(Elt: I);
1247 }
1248
1249 executeInWaterfallLoop(B, MI, OpIndices: SGPRIndexes);
1250 return true;
1251}
1252
1253// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
1254// the three offsets (voffset, soffset and instoffset)
1255unsigned AMDGPURegisterBankInfo::setBufferOffsets(
1256 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
1257 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) const {
1258 const LLT S32 = LLT::scalar(SizeInBits: 32);
1259 MachineRegisterInfo *MRI = B.getMRI();
1260
1261 if (std::optional<int64_t> Imm =
1262 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI: *MRI)) {
1263 uint32_t SOffset, ImmOffset;
1264 if (TII->splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
1265 VOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1266 SOffsetReg = B.buildConstant(Res: S32, Val: SOffset).getReg(Idx: 0);
1267 InstOffsetVal = ImmOffset;
1268
1269 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1270 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1271 return SOffset + ImmOffset;
1272 }
1273 }
1274
1275 const bool CheckNUW = Subtarget.hasGFX1250Insts();
1276 Register Base;
1277 unsigned Offset;
1278
1279 std::tie(args&: Base, args&: Offset) =
1280 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: CombinedOffset,
1281 /*KnownBits=*/ValueTracking: nullptr,
1282 /*CheckNUW=*/CheckNUW);
1283
1284 uint32_t SOffset, ImmOffset;
1285 if (static_cast<int32_t>(Offset) > 0 &&
1286 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
1287 if (getRegBank(Reg: Base, MRI: *MRI, TRI: *TRI) == &AMDGPU::VGPRRegBank) {
1288 VOffsetReg = Base;
1289 SOffsetReg = B.buildConstant(Res: S32, Val: SOffset).getReg(Idx: 0);
1290 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1291 InstOffsetVal = ImmOffset;
1292 return 0; // XXX - Why is this 0?
1293 }
1294
1295 // If we have SGPR base, we can use it for soffset.
1296 if (SOffset == 0) {
1297 VOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1298 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1299 SOffsetReg = Base;
1300 InstOffsetVal = ImmOffset;
1301 return 0; // XXX - Why is this 0?
1302 }
1303 }
1304
1305 // Handle the variable sgpr + vgpr case.
1306 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI: *MRI);
1307 if (Add && static_cast<int32_t>(Offset) >= 0 &&
1308 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
1309 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI: *MRI);
1310 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI: *MRI);
1311
1312 const RegisterBank *Src0Bank = getRegBank(Reg: Src0, MRI: *MRI, TRI: *TRI);
1313 const RegisterBank *Src1Bank = getRegBank(Reg: Src1, MRI: *MRI, TRI: *TRI);
1314
1315 if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) {
1316 VOffsetReg = Src0;
1317 SOffsetReg = Src1;
1318 return 0;
1319 }
1320
1321 if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) {
1322 VOffsetReg = Src1;
1323 SOffsetReg = Src0;
1324 return 0;
1325 }
1326 }
1327
1328 // Ensure we have a VGPR for the combined offset. This could be an issue if we
1329 // have an SGPR offset and a VGPR resource.
1330 if (getRegBank(Reg: CombinedOffset, MRI: *MRI, TRI: *TRI) == &AMDGPU::VGPRRegBank) {
1331 VOffsetReg = CombinedOffset;
1332 } else {
1333 VOffsetReg = B.buildCopy(Res: S32, Op: CombinedOffset).getReg(Idx: 0);
1334 B.getMRI()->setRegBank(Reg: VOffsetReg, RegBank: AMDGPU::VGPRRegBank);
1335 }
1336
1337 SOffsetReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1338 B.getMRI()->setRegBank(Reg: SOffsetReg, RegBank: AMDGPU::SGPRRegBank);
1339 return 0;
1340}
1341
1342static unsigned getSBufferLoadCorrespondingBufferLoadOpcode(unsigned Opc) {
1343 switch (Opc) {
1344 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
1345 return AMDGPU::G_AMDGPU_BUFFER_LOAD;
1346 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
1347 return AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
1348 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
1349 return AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE;
1350 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
1351 return AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
1352 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
1353 return AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT;
1354 default:
1355 break;
1356 }
1357 llvm_unreachable("Unexpected s_buffer_load opcode");
1358}
1359
1360bool AMDGPURegisterBankInfo::applyMappingSBufferLoad(
1361 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
1362 MachineInstr &MI = OpdMapper.getMI();
1363 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1364
1365 const LLT S32 = LLT::scalar(SizeInBits: 32);
1366 Register Dst = MI.getOperand(i: 0).getReg();
1367 LLT Ty = MRI.getType(Reg: Dst);
1368
1369 const RegisterBank *RSrcBank =
1370 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1371 const RegisterBank *OffsetBank =
1372 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
1373 if (RSrcBank == &AMDGPU::SGPRRegBank &&
1374 OffsetBank == &AMDGPU::SGPRRegBank)
1375 return true; // Legal mapping
1376
1377 // FIXME: 96-bit case was widened during legalize. We need to narrow it back
1378 // here but don't have an MMO.
1379
1380 unsigned LoadSize = Ty.getSizeInBits();
1381 int NumLoads = 1;
1382 if (LoadSize == 256 || LoadSize == 512) {
1383 NumLoads = LoadSize / 128;
1384 Ty = Ty.divide(Factor: NumLoads);
1385 }
1386
1387 // Use the alignment to ensure that the required offsets will fit into the
1388 // immediate offsets.
1389 const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1);
1390
1391 MachineFunction &MF = B.getMF();
1392
1393 Register SOffset;
1394 Register VOffset;
1395 int64_t ImmOffset = 0;
1396
1397 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
1398 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
1399
1400 // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we
1401 // can, but we need to track an MMO for that.
1402 const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8;
1403 const Align MemAlign(4); // FIXME: ABI type alignment?
1404 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(
1405 PtrInfo: MachinePointerInfo(),
1406 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
1407 MachineMemOperand::MOInvariant,
1408 Size: MemSize, BaseAlignment: MemAlign);
1409 if (MMOOffset != 0)
1410 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
1411
1412 // If only the offset is divergent, emit a MUBUF buffer load instead. We can
1413 // assume that the buffer is unswizzled.
1414
1415 Register RSrc = MI.getOperand(i: 1).getReg();
1416 Register VIndex = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1417 B.getMRI()->setRegBank(Reg: VIndex, RegBank: AMDGPU::VGPRRegBank);
1418 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
1419
1420 SmallVector<Register, 4> LoadParts(NumLoads);
1421
1422 MachineBasicBlock::iterator MII = MI.getIterator();
1423 MachineInstrSpan Span(MII, &B.getMBB());
1424
1425 for (int i = 0; i < NumLoads; ++i) {
1426 if (NumLoads == 1) {
1427 LoadParts[i] = Dst;
1428 } else {
1429 LoadParts[i] = MRI.createGenericVirtualRegister(Ty);
1430 MRI.setRegBank(Reg: LoadParts[i], RegBank: AMDGPU::VGPRRegBank);
1431 }
1432
1433 if (i != 0)
1434 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16, Size: MemSize);
1435
1436 B.buildInstr(Opcode: getSBufferLoadCorrespondingBufferLoadOpcode(Opc: MI.getOpcode()))
1437 .addDef(RegNo: LoadParts[i]) // vdata
1438 .addUse(RegNo: RSrc) // rsrc
1439 .addUse(RegNo: VIndex) // vindex
1440 .addUse(RegNo: VOffset) // voffset
1441 .addUse(RegNo: SOffset) // soffset
1442 .addImm(Val: ImmOffset + 16 * i) // offset(imm)
1443 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
1444 .addImm(Val: 0) // idxen(imm)
1445 .addMemOperand(MMO: BaseMMO);
1446 }
1447
1448 // TODO: If only the resource is a VGPR, it may be better to execute the
1449 // scalar load in the waterfall loop if the resource is expected to frequently
1450 // be dynamically uniform.
1451 if (RSrcBank != &AMDGPU::SGPRRegBank) {
1452 // Remove the original instruction to avoid potentially confusing the
1453 // waterfall loop logic.
1454 B.setInstr(*Span.begin());
1455 MI.eraseFromParent();
1456
1457 SmallSet<Register, 4> OpsToWaterfall;
1458
1459 OpsToWaterfall.insert(V: RSrc);
1460 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
1461 SGPROperandRegs&: OpsToWaterfall);
1462 }
1463
1464 if (NumLoads != 1) {
1465 if (Ty.isVector())
1466 B.buildConcatVectors(Res: Dst, Ops: LoadParts);
1467 else
1468 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
1469 }
1470
1471 // We removed the instruction earlier with a waterfall loop.
1472 if (RSrcBank == &AMDGPU::SGPRRegBank)
1473 MI.eraseFromParent();
1474
1475 return true;
1476}
1477
1478bool AMDGPURegisterBankInfo::applyMappingBFE(MachineIRBuilder &B,
1479 const OperandsMapper &OpdMapper,
1480 bool Signed) const {
1481 MachineInstr &MI = OpdMapper.getMI();
1482 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1483
1484 // Insert basic copies
1485 applyDefaultMapping(OpdMapper);
1486
1487 Register DstReg = MI.getOperand(i: 0).getReg();
1488 LLT Ty = MRI.getType(Reg: DstReg);
1489
1490 const LLT S32 = LLT::scalar(SizeInBits: 32);
1491
1492 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
1493 Register SrcReg = MI.getOperand(i: FirstOpnd).getReg();
1494 Register OffsetReg = MI.getOperand(i: FirstOpnd + 1).getReg();
1495 Register WidthReg = MI.getOperand(i: FirstOpnd + 2).getReg();
1496
1497 const RegisterBank *DstBank =
1498 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1499 if (DstBank == &AMDGPU::VGPRRegBank) {
1500 if (Ty == S32)
1501 return true;
1502
1503 // There is no 64-bit vgpr bitfield extract instructions so the operation
1504 // is expanded to a sequence of instructions that implement the operation.
1505 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank);
1506
1507 const LLT S64 = LLT::scalar(SizeInBits: 64);
1508 // Shift the source operand so that extracted bits start at bit 0.
1509 auto ShiftOffset = Signed ? B.buildAShr(Dst: S64, Src0: SrcReg, Src1: OffsetReg)
1510 : B.buildLShr(Dst: S64, Src0: SrcReg, Src1: OffsetReg);
1511 auto UnmergeSOffset = B.buildUnmerge(Res: {S32, S32}, Op: ShiftOffset);
1512
1513 // A 64-bit bitfield extract uses the 32-bit bitfield extract instructions
1514 // if the width is a constant.
1515 if (auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: WidthReg, MRI)) {
1516 // Use the 32-bit bitfield extract instruction if the width is a constant.
1517 // Depending on the width size, use either the low or high 32-bits.
1518 auto Zero = B.buildConstant(Res: S32, Val: 0);
1519 auto WidthImm = ConstWidth->Value.getZExtValue();
1520 if (WidthImm <= 32) {
1521 // Use bitfield extract on the lower 32-bit source, and then sign-extend
1522 // or clear the upper 32-bits.
1523 auto Extract =
1524 Signed ? B.buildSbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 0), LSB: Zero, Width: WidthReg)
1525 : B.buildUbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 0), LSB: Zero, Width: WidthReg);
1526 auto Extend =
1527 Signed ? B.buildAShr(Dst: S32, Src0: Extract, Src1: B.buildConstant(Res: S32, Val: 31)) : Zero;
1528 B.buildMergeLikeInstr(Res: DstReg, Ops: {Extract, Extend});
1529 } else {
1530 // Use bitfield extract on upper 32-bit source, and combine with lower
1531 // 32-bit source.
1532 auto UpperWidth = B.buildConstant(Res: S32, Val: WidthImm - 32);
1533 auto Extract =
1534 Signed
1535 ? B.buildSbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 1), LSB: Zero, Width: UpperWidth)
1536 : B.buildUbfx(Dst: S32, Src: UnmergeSOffset.getReg(Idx: 1), LSB: Zero, Width: UpperWidth);
1537 B.buildMergeLikeInstr(Res: DstReg, Ops: {UnmergeSOffset.getReg(Idx: 0), Extract});
1538 }
1539 MI.eraseFromParent();
1540 return true;
1541 }
1542
1543 // Expand to Src >> Offset << (64 - Width) >> (64 - Width) using 64-bit
1544 // operations.
1545 auto ExtShift = B.buildSub(Dst: S32, Src0: B.buildConstant(Res: S32, Val: 64), Src1: WidthReg);
1546 auto SignBit = B.buildShl(Dst: S64, Src0: ShiftOffset, Src1: ExtShift);
1547 if (Signed)
1548 B.buildAShr(Dst: S64, Src0: SignBit, Src1: ExtShift);
1549 else
1550 B.buildLShr(Dst: S64, Src0: SignBit, Src1: ExtShift);
1551 MI.eraseFromParent();
1552 return true;
1553 }
1554
1555 // The scalar form packs the offset and width in a single operand.
1556
1557 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::SGPRRegBank);
1558
1559 // Ensure the high bits are clear to insert the offset.
1560 auto OffsetMask = B.buildConstant(Res: S32, Val: maskTrailingOnes<unsigned>(N: 6));
1561 auto ClampOffset = B.buildAnd(Dst: S32, Src0: OffsetReg, Src1: OffsetMask);
1562
1563 // Zeros out the low bits, so don't bother clamping the input value.
1564 auto ShiftWidth = B.buildShl(Dst: S32, Src0: WidthReg, Src1: B.buildConstant(Res: S32, Val: 16));
1565
1566 // Transformation function, pack the offset and width of a BFE into
1567 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1568 // source, bits [5:0] contain the offset and bits [22:16] the width.
1569 auto MergedInputs = B.buildOr(Dst: S32, Src0: ClampOffset, Src1: ShiftWidth);
1570
1571 // TODO: It might be worth using a pseudo here to avoid scc clobber and
1572 // register class constraints.
1573 unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) :
1574 (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64);
1575
1576 auto MIB = B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {SrcReg, MergedInputs});
1577 constrainSelectedInstRegOperands(I&: *MIB, TII: *TII, TRI: *TRI, RBI: *this);
1578
1579 MI.eraseFromParent();
1580 return true;
1581}
1582
1583bool AMDGPURegisterBankInfo::applyMappingMAD_64_32(
1584 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
1585 MachineInstr &MI = OpdMapper.getMI();
1586 MachineRegisterInfo &MRI = OpdMapper.getMRI();
1587
1588 // Insert basic copies.
1589 applyDefaultMapping(OpdMapper);
1590
1591 Register Dst0 = MI.getOperand(i: 0).getReg();
1592 Register Dst1 = MI.getOperand(i: 1).getReg();
1593 Register Src0 = MI.getOperand(i: 2).getReg();
1594 Register Src1 = MI.getOperand(i: 3).getReg();
1595 Register Src2 = MI.getOperand(i: 4).getReg();
1596
1597 if (MRI.getRegBankOrNull(Reg: Src0) == &AMDGPU::VGPRRegBank)
1598 return true;
1599
1600 bool IsUnsigned = MI.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
1601 LLT S1 = LLT::scalar(SizeInBits: 1);
1602 LLT S32 = LLT::scalar(SizeInBits: 32);
1603
1604 bool DstOnValu = MRI.getRegBankOrNull(Reg: Src2) == &AMDGPU::VGPRRegBank;
1605 bool Accumulate = true;
1606
1607 if (!DstOnValu) {
1608 if (mi_match(R: Src2, MRI, P: m_ZeroInt()))
1609 Accumulate = false;
1610 }
1611
1612 // Keep the multiplication on the SALU.
1613 Register DstHi;
1614 Register DstLo = B.buildMul(Dst: S32, Src0, Src1).getReg(Idx: 0);
1615 bool MulHiInVgpr = false;
1616
1617 MRI.setRegBank(Reg: DstLo, RegBank: AMDGPU::SGPRRegBank);
1618
1619 if (Subtarget.hasSMulHi()) {
1620 DstHi = IsUnsigned ? B.buildUMulH(Dst: S32, Src0, Src1).getReg(Idx: 0)
1621 : B.buildSMulH(Dst: S32, Src0, Src1).getReg(Idx: 0);
1622 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::SGPRRegBank);
1623 } else {
1624 Register VSrc0 = B.buildCopy(Res: S32, Op: Src0).getReg(Idx: 0);
1625 Register VSrc1 = B.buildCopy(Res: S32, Op: Src1).getReg(Idx: 0);
1626
1627 MRI.setRegBank(Reg: VSrc0, RegBank: AMDGPU::VGPRRegBank);
1628 MRI.setRegBank(Reg: VSrc1, RegBank: AMDGPU::VGPRRegBank);
1629
1630 DstHi = IsUnsigned ? B.buildUMulH(Dst: S32, Src0: VSrc0, Src1: VSrc1).getReg(Idx: 0)
1631 : B.buildSMulH(Dst: S32, Src0: VSrc0, Src1: VSrc1).getReg(Idx: 0);
1632 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::VGPRRegBank);
1633
1634 if (!DstOnValu) {
1635 DstHi = buildReadFirstLane(B, MRI, Src: DstHi);
1636 } else {
1637 MulHiInVgpr = true;
1638 }
1639 }
1640
1641 // Accumulate and produce the "carry-out" bit.
1642 //
1643 // The "carry-out" is defined as bit 64 of the result when computed as a
1644 // big integer. For unsigned multiply-add, this matches the usual definition
1645 // of carry-out. For signed multiply-add, bit 64 is the sign bit of the
1646 // result, which is determined as:
1647 // sign(Src0 * Src1) + sign(Src2) + carry-out from unsigned 64-bit add
1648 LLT CarryType = DstOnValu ? S1 : S32;
1649 const RegisterBank &CarryBank =
1650 DstOnValu ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank;
1651 const RegisterBank &DstBank =
1652 DstOnValu ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank;
1653 Register Carry;
1654 Register Zero;
1655
1656 if (!IsUnsigned) {
1657 Zero = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1658 MRI.setRegBank(Reg: Zero,
1659 RegBank: MulHiInVgpr ? AMDGPU::VGPRRegBank : AMDGPU::SGPRRegBank);
1660
1661 Carry = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: MulHiInVgpr ? S1 : S32, Op0: DstHi, Op1: Zero)
1662 .getReg(Idx: 0);
1663 MRI.setRegBank(Reg: Carry, RegBank: MulHiInVgpr ? AMDGPU::VCCRegBank
1664 : AMDGPU::SGPRRegBank);
1665
1666 if (DstOnValu && !MulHiInVgpr) {
1667 Carry = B.buildTrunc(Res: S1, Op: Carry).getReg(Idx: 0);
1668 MRI.setRegBank(Reg: Carry, RegBank: AMDGPU::VCCRegBank);
1669 }
1670 }
1671
1672 if (Accumulate) {
1673 if (DstOnValu) {
1674 DstLo = B.buildCopy(Res: S32, Op: DstLo).getReg(Idx: 0);
1675 DstHi = B.buildCopy(Res: S32, Op: DstHi).getReg(Idx: 0);
1676 MRI.setRegBank(Reg: DstLo, RegBank: AMDGPU::VGPRRegBank);
1677 MRI.setRegBank(Reg: DstHi, RegBank: AMDGPU::VGPRRegBank);
1678 }
1679
1680 auto Unmerge = B.buildUnmerge(Res: S32, Op: Src2);
1681 Register Src2Lo = Unmerge.getReg(Idx: 0);
1682 Register Src2Hi = Unmerge.getReg(Idx: 1);
1683 MRI.setRegBank(Reg: Src2Lo, RegBank: DstBank);
1684 MRI.setRegBank(Reg: Src2Hi, RegBank: DstBank);
1685
1686 if (!IsUnsigned) {
1687 auto Src2Sign = B.buildICmp(Pred: CmpInst::ICMP_SLT, Res: CarryType, Op0: Src2Hi, Op1: Zero);
1688 MRI.setRegBank(Reg: Src2Sign.getReg(Idx: 0), RegBank: CarryBank);
1689
1690 Carry = B.buildXor(Dst: CarryType, Src0: Carry, Src1: Src2Sign).getReg(Idx: 0);
1691 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1692 }
1693
1694 auto AddLo = B.buildUAddo(Res: S32, CarryOut: CarryType, Op0: DstLo, Op1: Src2Lo);
1695 DstLo = AddLo.getReg(Idx: 0);
1696 Register CarryLo = AddLo.getReg(Idx: 1);
1697 MRI.setRegBank(Reg: DstLo, RegBank: DstBank);
1698 MRI.setRegBank(Reg: CarryLo, RegBank: CarryBank);
1699
1700 auto AddHi = B.buildUAdde(Res: S32, CarryOut: CarryType, Op0: DstHi, Op1: Src2Hi, CarryIn: CarryLo);
1701 DstHi = AddHi.getReg(Idx: 0);
1702 MRI.setRegBank(Reg: DstHi, RegBank: DstBank);
1703
1704 Register CarryHi = AddHi.getReg(Idx: 1);
1705 MRI.setRegBank(Reg: CarryHi, RegBank: CarryBank);
1706
1707 if (IsUnsigned) {
1708 Carry = CarryHi;
1709 } else {
1710 Carry = B.buildXor(Dst: CarryType, Src0: Carry, Src1: CarryHi).getReg(Idx: 0);
1711 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1712 }
1713 } else {
1714 if (IsUnsigned) {
1715 Carry = B.buildConstant(Res: CarryType, Val: 0).getReg(Idx: 0);
1716 MRI.setRegBank(Reg: Carry, RegBank: CarryBank);
1717 }
1718 }
1719
1720 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
1721
1722 if (DstOnValu) {
1723 B.buildCopy(Res: Dst1, Op: Carry);
1724 } else {
1725 B.buildTrunc(Res: Dst1, Op: Carry);
1726 }
1727
1728 MI.eraseFromParent();
1729 return true;
1730}
1731
1732// Return a suitable opcode for extending the operands of Opc when widening.
1733static unsigned getExtendOp(unsigned Opc) {
1734 switch (Opc) {
1735 case TargetOpcode::G_ASHR:
1736 case TargetOpcode::G_SMIN:
1737 case TargetOpcode::G_SMAX:
1738 return TargetOpcode::G_SEXT;
1739 case TargetOpcode::G_LSHR:
1740 case TargetOpcode::G_UMIN:
1741 case TargetOpcode::G_UMAX:
1742 return TargetOpcode::G_ZEXT;
1743 default:
1744 return TargetOpcode::G_ANYEXT;
1745 }
1746}
1747
1748// Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding
1749// any illegal vector extend or unmerge operations.
1750static std::pair<Register, Register>
1751unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) {
1752 const LLT S32 = LLT::scalar(SizeInBits: 32);
1753 auto Bitcast = B.buildBitcast(Dst: S32, Src);
1754
1755 if (ExtOpcode == TargetOpcode::G_SEXT) {
1756 auto ExtLo = B.buildSExtInReg(Res: S32, Op: Bitcast, ImmOp: 16);
1757 auto ShiftHi = B.buildAShr(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 16));
1758 return std::pair(ExtLo.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1759 }
1760
1761 auto ShiftHi = B.buildLShr(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 16));
1762 if (ExtOpcode == TargetOpcode::G_ZEXT) {
1763 auto ExtLo = B.buildAnd(Dst: S32, Src0: Bitcast, Src1: B.buildConstant(Res: S32, Val: 0xffff));
1764 return std::pair(ExtLo.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1765 }
1766
1767 assert(ExtOpcode == TargetOpcode::G_ANYEXT);
1768 return std::pair(Bitcast.getReg(Idx: 0), ShiftHi.getReg(Idx: 0));
1769}
1770
1771// For cases where only a single copy is inserted for matching register banks.
1772// Replace the register in the instruction operand
1773static bool substituteSimpleCopyRegs(
1774 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) {
1775 SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx));
1776 if (!SrcReg.empty()) {
1777 assert(SrcReg.size() == 1);
1778 OpdMapper.getMI().getOperand(i: OpIdx).setReg(SrcReg[0]);
1779 return true;
1780 }
1781
1782 return false;
1783}
1784
1785/// Handle register layout difference for f16 images for some subtargets.
1786Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
1787 MachineRegisterInfo &MRI,
1788 Register Reg) const {
1789 if (!Subtarget.hasUnpackedD16VMem())
1790 return Reg;
1791
1792 const LLT S16 = LLT::scalar(SizeInBits: 16);
1793 LLT StoreVT = MRI.getType(Reg);
1794 if (!StoreVT.isVector() || StoreVT.getElementType() != S16)
1795 return Reg;
1796
1797 auto Unmerge = B.buildUnmerge(Res: S16, Op: Reg);
1798
1799
1800 SmallVector<Register, 4> WideRegs;
1801 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
1802 WideRegs.push_back(Elt: Unmerge.getReg(Idx: I));
1803
1804 const LLT S32 = LLT::scalar(SizeInBits: 32);
1805 int NumElts = StoreVT.getNumElements();
1806
1807 return B.buildMergeLikeInstr(Res: LLT::fixed_vector(NumElements: NumElts, ScalarTy: S32), Ops: WideRegs)
1808 .getReg(Idx: 0);
1809}
1810
1811static std::pair<Register, unsigned>
1812getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
1813 int64_t Const;
1814 if (mi_match(R: Reg, MRI, P: m_ICst(Cst&: Const)))
1815 return std::pair(Register(), Const);
1816
1817 Register Base;
1818 if (mi_match(R: Reg, MRI, P: m_GAdd(L: m_Reg(R&: Base), R: m_ICst(Cst&: Const))))
1819 return std::pair(Base, Const);
1820
1821 // TODO: Handle G_OR used for add case
1822 return std::pair(Reg, 0);
1823}
1824
1825std::pair<Register, unsigned>
1826AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
1827 Register OrigOffset) const {
1828 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: Subtarget);
1829 Register BaseReg;
1830 unsigned ImmOffset;
1831 const LLT S32 = LLT::scalar(SizeInBits: 32);
1832
1833 // TODO: Use AMDGPU::getBaseWithConstantOffset() instead.
1834 std::tie(args&: BaseReg, args&: ImmOffset) = getBaseWithConstantOffset(MRI&: *B.getMRI(),
1835 Reg: OrigOffset);
1836
1837 unsigned C1 = 0;
1838 if (ImmOffset != 0) {
1839 // If the immediate value is too big for the immoffset field, put only bits
1840 // that would normally fit in the immoffset field. The remaining value that
1841 // is copied/added for the voffset field is a large power of 2, and it
1842 // stands more chance of being CSEd with the copy/add for another similar
1843 // load/store.
1844 // However, do not do that rounding down if that is a negative
1845 // number, as it appears to be illegal to have a negative offset in the
1846 // vgpr, even if adding the immediate offset makes it positive.
1847 unsigned Overflow = ImmOffset & ~MaxImm;
1848 ImmOffset -= Overflow;
1849 if (static_cast<int32_t>(Overflow) < 0) {
1850 Overflow += ImmOffset;
1851 ImmOffset = 0;
1852 }
1853
1854 C1 = ImmOffset;
1855 if (Overflow != 0) {
1856 if (!BaseReg)
1857 BaseReg = B.buildConstant(Res: S32, Val: Overflow).getReg(Idx: 0);
1858 else {
1859 auto OverflowVal = B.buildConstant(Res: S32, Val: Overflow);
1860 BaseReg = B.buildAdd(Dst: S32, Src0: BaseReg, Src1: OverflowVal).getReg(Idx: 0);
1861 }
1862 }
1863 }
1864
1865 if (!BaseReg)
1866 BaseReg = B.buildConstant(Res: S32, Val: 0).getReg(Idx: 0);
1867
1868 return {BaseReg, C1};
1869}
1870
1871bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg,
1872 Register SrcReg) const {
1873 MachineRegisterInfo &MRI = *B.getMRI();
1874 LLT SrcTy = MRI.getType(Reg: SrcReg);
1875 if (SrcTy.getSizeInBits() == 32) {
1876 // Use a v_mov_b32 here to make the exec dependency explicit.
1877 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1878 .addDef(RegNo: DstReg)
1879 .addUse(RegNo: SrcReg);
1880 return constrainGenericRegister(Reg: DstReg, RC: AMDGPU::VGPR_32RegClass, MRI) &&
1881 constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI);
1882 }
1883
1884 Register TmpReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1885 Register TmpReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1886
1887 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1888 .addDef(RegNo: TmpReg0)
1889 .addUse(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
1890 B.buildInstr(Opcode: AMDGPU::V_MOV_B32_e32)
1891 .addDef(RegNo: TmpReg1)
1892 .addUse(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
1893 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
1894 .addDef(RegNo: DstReg)
1895 .addUse(RegNo: TmpReg0)
1896 .addImm(Val: AMDGPU::sub0)
1897 .addUse(RegNo: TmpReg1)
1898 .addImm(Val: AMDGPU::sub1);
1899
1900 return constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_64RegClass, MRI) &&
1901 constrainGenericRegister(Reg: DstReg, RC: AMDGPU::VReg_64RegClass, MRI);
1902}
1903
1904/// Utility function for pushing dynamic vector indexes with a constant offset
1905/// into waterfall loops.
1906static void reinsertVectorIndexAdd(MachineIRBuilder &B,
1907 MachineInstr &IdxUseInstr,
1908 unsigned OpIdx,
1909 unsigned ConstOffset) {
1910 MachineRegisterInfo &MRI = *B.getMRI();
1911 const LLT S32 = LLT::scalar(SizeInBits: 32);
1912 Register WaterfallIdx = IdxUseInstr.getOperand(i: OpIdx).getReg();
1913 B.setInsertPt(MBB&: *IdxUseInstr.getParent(), II: IdxUseInstr.getIterator());
1914
1915 auto MaterializedOffset = B.buildConstant(Res: S32, Val: ConstOffset);
1916
1917 auto Add = B.buildAdd(Dst: S32, Src0: WaterfallIdx, Src1: MaterializedOffset);
1918 MRI.setRegBank(Reg: MaterializedOffset.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
1919 MRI.setRegBank(Reg: Add.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
1920 IdxUseInstr.getOperand(i: OpIdx).setReg(Add.getReg(Idx: 0));
1921}
1922
1923/// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the
1924/// original 32-bit source value (to be inserted in the low part of the combined
1925/// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit
1926/// value.
1927static void extendLow32IntoHigh32(MachineIRBuilder &B,
1928 Register Hi32Reg, Register Lo32Reg,
1929 unsigned ExtOpc,
1930 const RegisterBank &RegBank,
1931 bool IsBooleanSrc = false) {
1932 if (ExtOpc == AMDGPU::G_ZEXT) {
1933 B.buildConstant(Res: Hi32Reg, Val: 0);
1934 } else if (ExtOpc == AMDGPU::G_SEXT) {
1935 if (IsBooleanSrc) {
1936 // If we know the original source was an s1, the high half is the same as
1937 // the low.
1938 B.buildCopy(Res: Hi32Reg, Op: Lo32Reg);
1939 } else {
1940 // Replicate sign bit from 32-bit extended part.
1941 auto ShiftAmt = B.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 31);
1942 B.getMRI()->setRegBank(Reg: ShiftAmt.getReg(Idx: 0), RegBank);
1943 B.buildAShr(Dst: Hi32Reg, Src0: Lo32Reg, Src1: ShiftAmt);
1944 }
1945 } else {
1946 assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension");
1947 B.buildUndef(Res: Hi32Reg);
1948 }
1949}
1950
1951bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect(
1952 MachineIRBuilder &B, MachineInstr &MI,
1953 const OperandsMapper &OpdMapper) const {
1954 MachineRegisterInfo &MRI = *B.getMRI();
1955
1956 Register VecReg = MI.getOperand(i: 1).getReg();
1957 Register Idx = MI.getOperand(i: 2).getReg();
1958
1959 const RegisterBank &IdxBank =
1960 *OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
1961
1962 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
1963
1964 LLT VecTy = MRI.getType(Reg: VecReg);
1965 unsigned EltSize = VecTy.getScalarSizeInBits();
1966 unsigned NumElem = VecTy.getNumElements();
1967
1968 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
1969 IsDivergentIdx, Subtarget: &Subtarget))
1970 return false;
1971
1972 LLT S32 = LLT::scalar(SizeInBits: 32);
1973
1974 const RegisterBank &DstBank =
1975 *OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
1976 const RegisterBank &SrcBank =
1977 *OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
1978
1979 const RegisterBank &CCBank =
1980 (DstBank == AMDGPU::SGPRRegBank &&
1981 SrcBank == AMDGPU::SGPRRegBank &&
1982 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
1983 : AMDGPU::VCCRegBank;
1984 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(SizeInBits: 1);
1985
1986 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
1987 Idx = B.buildCopy(Res: S32, Op: Idx)->getOperand(i: 0).getReg();
1988 MRI.setRegBank(Reg: Idx, RegBank: AMDGPU::VGPRRegBank);
1989 }
1990
1991 LLT EltTy = VecTy.getScalarType();
1992 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
1993 unsigned NumLanes = DstRegs.size();
1994 if (!NumLanes)
1995 NumLanes = 1;
1996 else
1997 EltTy = MRI.getType(Reg: DstRegs[0]);
1998
1999 auto UnmergeToEltTy = B.buildUnmerge(Res: EltTy, Op: VecReg);
2000 SmallVector<Register, 2> Res(NumLanes);
2001 for (unsigned L = 0; L < NumLanes; ++L)
2002 Res[L] = UnmergeToEltTy.getReg(Idx: L);
2003
2004 for (unsigned I = 1; I < NumElem; ++I) {
2005 auto IC = B.buildConstant(Res: S32, Val: I);
2006 MRI.setRegBank(Reg: IC->getOperand(i: 0).getReg(), RegBank: AMDGPU::SGPRRegBank);
2007 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CCTy, Op0: Idx, Op1: IC);
2008 MRI.setRegBank(Reg: Cmp->getOperand(i: 0).getReg(), RegBank: CCBank);
2009
2010 for (unsigned L = 0; L < NumLanes; ++L) {
2011 auto S = B.buildSelect(Res: EltTy, Tst: Cmp,
2012 Op0: UnmergeToEltTy.getReg(Idx: I * NumLanes + L), Op1: Res[L]);
2013
2014 for (unsigned N : { 0, 2, 3 })
2015 MRI.setRegBank(Reg: S->getOperand(i: N).getReg(), RegBank: DstBank);
2016
2017 Res[L] = S->getOperand(i: 0).getReg();
2018 }
2019 }
2020
2021 for (unsigned L = 0; L < NumLanes; ++L) {
2022 Register DstReg = (NumLanes == 1) ? MI.getOperand(i: 0).getReg() : DstRegs[L];
2023 B.buildCopy(Res: DstReg, Op: Res[L]);
2024 MRI.setRegBank(Reg: DstReg, RegBank: DstBank);
2025 }
2026
2027 MRI.setRegBank(Reg: MI.getOperand(i: 0).getReg(), RegBank: DstBank);
2028 MI.eraseFromParent();
2029
2030 return true;
2031}
2032
2033// Insert a cross regbank copy for a register if it already has a bank that
2034// differs from the one we want to set.
2035static Register constrainRegToBank(MachineRegisterInfo &MRI,
2036 MachineIRBuilder &B, Register &Reg,
2037 const RegisterBank &Bank) {
2038 const RegisterBank *CurrBank = MRI.getRegBankOrNull(Reg);
2039 if (CurrBank && *CurrBank != Bank) {
2040 Register Copy = B.buildCopy(Res: MRI.getType(Reg), Op: Reg).getReg(Idx: 0);
2041 MRI.setRegBank(Reg: Copy, RegBank: Bank);
2042 return Copy;
2043 }
2044
2045 MRI.setRegBank(Reg, RegBank: Bank);
2046 return Reg;
2047}
2048
2049bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect(
2050 MachineIRBuilder &B, MachineInstr &MI,
2051 const OperandsMapper &OpdMapper) const {
2052
2053 MachineRegisterInfo &MRI = *B.getMRI();
2054 Register VecReg = MI.getOperand(i: 1).getReg();
2055 Register Idx = MI.getOperand(i: 3).getReg();
2056
2057 const RegisterBank &IdxBank =
2058 *OpdMapper.getInstrMapping().getOperandMapping(i: 3).BreakDown[0].RegBank;
2059
2060 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank;
2061
2062 LLT VecTy = MRI.getType(Reg: VecReg);
2063 unsigned EltSize = VecTy.getScalarSizeInBits();
2064 unsigned NumElem = VecTy.getNumElements();
2065
2066 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem,
2067 IsDivergentIdx, Subtarget: &Subtarget))
2068 return false;
2069
2070 LLT S32 = LLT::scalar(SizeInBits: 32);
2071
2072 const RegisterBank &DstBank =
2073 *OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2074 const RegisterBank &SrcBank =
2075 *OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2076 const RegisterBank &InsBank =
2077 *OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2078
2079 const RegisterBank &CCBank =
2080 (DstBank == AMDGPU::SGPRRegBank &&
2081 SrcBank == AMDGPU::SGPRRegBank &&
2082 InsBank == AMDGPU::SGPRRegBank &&
2083 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank
2084 : AMDGPU::VCCRegBank;
2085 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(SizeInBits: 1);
2086
2087 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) {
2088 Idx = B.buildCopy(Res: S32, Op: Idx)->getOperand(i: 0).getReg();
2089 MRI.setRegBank(Reg: Idx, RegBank: AMDGPU::VGPRRegBank);
2090 }
2091
2092 LLT EltTy = VecTy.getScalarType();
2093 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(OpIdx: 2));
2094 unsigned NumLanes = InsRegs.size();
2095 if (!NumLanes) {
2096 NumLanes = 1;
2097 InsRegs.push_back(Elt: MI.getOperand(i: 2).getReg());
2098 } else {
2099 EltTy = MRI.getType(Reg: InsRegs[0]);
2100 }
2101
2102 auto UnmergeToEltTy = B.buildUnmerge(Res: EltTy, Op: VecReg);
2103 SmallVector<Register, 16> Ops(NumElem * NumLanes);
2104
2105 for (unsigned I = 0; I < NumElem; ++I) {
2106 auto IC = B.buildConstant(Res: S32, Val: I);
2107 MRI.setRegBank(Reg: IC->getOperand(i: 0).getReg(), RegBank: AMDGPU::SGPRRegBank);
2108 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CCTy, Op0: Idx, Op1: IC);
2109 MRI.setRegBank(Reg: Cmp->getOperand(i: 0).getReg(), RegBank: CCBank);
2110
2111 for (unsigned L = 0; L < NumLanes; ++L) {
2112 Register Op0 = constrainRegToBank(MRI, B, Reg&: InsRegs[L], Bank: DstBank);
2113 Register Op1 = UnmergeToEltTy.getReg(Idx: I * NumLanes + L);
2114 Op1 = constrainRegToBank(MRI, B, Reg&: Op1, Bank: DstBank);
2115
2116 Register Select = B.buildSelect(Res: EltTy, Tst: Cmp, Op0, Op1).getReg(Idx: 0);
2117 MRI.setRegBank(Reg: Select, RegBank: DstBank);
2118
2119 Ops[I * NumLanes + L] = Select;
2120 }
2121 }
2122
2123 LLT MergeTy = LLT::fixed_vector(NumElements: Ops.size(), ScalarTy: EltTy);
2124 if (MergeTy == MRI.getType(Reg: MI.getOperand(i: 0).getReg())) {
2125 B.buildBuildVector(Res: MI.getOperand(i: 0), Ops);
2126 } else {
2127 auto Vec = B.buildBuildVector(Res: MergeTy, Ops);
2128 MRI.setRegBank(Reg: Vec->getOperand(i: 0).getReg(), RegBank: DstBank);
2129 B.buildBitcast(Dst: MI.getOperand(i: 0).getReg(), Src: Vec);
2130 }
2131
2132 MRI.setRegBank(Reg: MI.getOperand(i: 0).getReg(), RegBank: DstBank);
2133 MI.eraseFromParent();
2134
2135 return true;
2136}
2137
2138// Break s_mul_u64 into 32-bit vector operations.
2139void AMDGPURegisterBankInfo::applyMappingSMULU64(
2140 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
2141 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2142 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(OpIdx: 1));
2143 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2144
2145 // All inputs are SGPRs, nothing special to do.
2146 if (DefRegs.empty()) {
2147 assert(Src0Regs.empty() && Src1Regs.empty());
2148 applyDefaultMapping(OpdMapper);
2149 return;
2150 }
2151
2152 assert(DefRegs.size() == 2);
2153 assert(Src0Regs.size() == Src1Regs.size() &&
2154 (Src0Regs.empty() || Src0Regs.size() == 2));
2155
2156 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2157 MachineInstr &MI = OpdMapper.getMI();
2158 Register DstReg = MI.getOperand(i: 0).getReg();
2159 LLT HalfTy = LLT::scalar(SizeInBits: 32);
2160
2161 // Depending on where the source registers came from, the generic code may
2162 // have decided to split the inputs already or not. If not, we still need to
2163 // extract the values.
2164
2165 if (Src0Regs.empty())
2166 split64BitValueForMapping(B, Regs&: Src0Regs, HalfTy, Reg: MI.getOperand(i: 1).getReg());
2167 else
2168 setRegsToType(MRI, Regs: Src0Regs, NewTy: HalfTy);
2169
2170 if (Src1Regs.empty())
2171 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2172 else
2173 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2174
2175 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2176
2177 // The multiplication is done as follows:
2178 //
2179 // Op1H Op1L
2180 // * Op0H Op0L
2181 // --------------------
2182 // Op1H*Op0L Op1L*Op0L
2183 // + Op1H*Op0H Op1L*Op0H
2184 // -----------------------------------------
2185 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
2186 //
2187 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
2188 // value and that would overflow.
2189 // The low 32-bit value is Op1L*Op0L.
2190 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from
2191 // Op1L*Op0L).
2192
2193 ApplyRegBankMapping ApplyBank(B, *this, MRI, &AMDGPU::VGPRRegBank);
2194
2195 Register Hi = B.buildUMulH(Dst: HalfTy, Src0: Src0Regs[0], Src1: Src1Regs[0]).getReg(Idx: 0);
2196 Register MulLoHi = B.buildMul(Dst: HalfTy, Src0: Src0Regs[0], Src1: Src1Regs[1]).getReg(Idx: 0);
2197 Register Add = B.buildAdd(Dst: HalfTy, Src0: Hi, Src1: MulLoHi).getReg(Idx: 0);
2198 Register MulHiLo = B.buildMul(Dst: HalfTy, Src0: Src0Regs[1], Src1: Src1Regs[0]).getReg(Idx: 0);
2199 B.buildAdd(Dst: DefRegs[1], Src0: Add, Src1: MulHiLo);
2200 B.buildMul(Dst: DefRegs[0], Src0: Src0Regs[0], Src1: Src1Regs[0]);
2201
2202 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2203 MI.eraseFromParent();
2204}
2205
2206void AMDGPURegisterBankInfo::applyMappingImpl(
2207 MachineIRBuilder &B, const OperandsMapper &OpdMapper) const {
2208 MachineInstr &MI = OpdMapper.getMI();
2209 B.setInstrAndDebugLoc(MI);
2210 unsigned Opc = MI.getOpcode();
2211 MachineRegisterInfo &MRI = OpdMapper.getMRI();
2212 switch (Opc) {
2213 case AMDGPU::G_CONSTANT:
2214 case AMDGPU::G_IMPLICIT_DEF: {
2215 Register DstReg = MI.getOperand(i: 0).getReg();
2216 LLT DstTy = MRI.getType(Reg: DstReg);
2217 if (DstTy != LLT::scalar(SizeInBits: 1))
2218 break;
2219
2220 const RegisterBank *DstBank =
2221 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2222 if (DstBank == &AMDGPU::VCCRegBank)
2223 break;
2224 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2225 if (DefRegs.empty())
2226 DefRegs.push_back(Elt: DstReg);
2227
2228 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
2229
2230 Register NewDstReg = MRI.createGenericVirtualRegister(Ty: LLT::scalar(SizeInBits: 32));
2231 LLVMContext &Ctx = B.getMF().getFunction().getContext();
2232
2233 MI.getOperand(i: 0).setReg(NewDstReg);
2234 if (Opc != AMDGPU::G_IMPLICIT_DEF) {
2235 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
2236 MI.getOperand(i: 1).setCImm(
2237 ConstantInt::get(Ty: IntegerType::getInt32Ty(C&: Ctx), V: ConstVal));
2238 }
2239
2240 MRI.setRegBank(Reg: NewDstReg, RegBank: *DstBank);
2241 B.buildTrunc(Res: DefRegs[0], Op: NewDstReg);
2242 return;
2243 }
2244 case AMDGPU::G_PHI: {
2245 Register DstReg = MI.getOperand(i: 0).getReg();
2246 LLT DstTy = MRI.getType(Reg: DstReg);
2247 if (DstTy != LLT::scalar(SizeInBits: 1))
2248 break;
2249
2250 const LLT S32 = LLT::scalar(SizeInBits: 32);
2251 const RegisterBank *DstBank =
2252 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2253 if (DstBank == &AMDGPU::VCCRegBank) {
2254 applyDefaultMapping(OpdMapper);
2255 // The standard handling only considers the result register bank for
2256 // phis. For VCC, blindly inserting a copy when the phi is lowered will
2257 // produce an invalid copy. We can only copy with some kind of compare to
2258 // get a vector boolean result. Insert a register bank copy that will be
2259 // correctly lowered to a compare.
2260 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2261 Register SrcReg = MI.getOperand(i: I).getReg();
2262 const RegisterBank *SrcBank = getRegBank(Reg: SrcReg, MRI, TRI: *TRI);
2263
2264 if (SrcBank != &AMDGPU::VCCRegBank) {
2265 MachineBasicBlock *SrcMBB = MI.getOperand(i: I + 1).getMBB();
2266 B.setInsertPt(MBB&: *SrcMBB, II: SrcMBB->getFirstTerminator());
2267
2268 auto Copy = B.buildCopy(Res: LLT::scalar(SizeInBits: 1), Op: SrcReg);
2269 MRI.setRegBank(Reg: Copy.getReg(Idx: 0), RegBank: AMDGPU::VCCRegBank);
2270 MI.getOperand(i: I).setReg(Copy.getReg(Idx: 0));
2271 }
2272 }
2273
2274 return;
2275 }
2276
2277 // Phi handling is strange and only considers the bank of the destination.
2278 substituteSimpleCopyRegs(OpdMapper, OpIdx: 0);
2279
2280 // Promote SGPR/VGPR booleans to s32
2281 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
2282 B.setInsertPt(MBB&: B.getMBB(), II: MI);
2283 LegalizerHelper Helper(B.getMF(), ApplyBank, B);
2284
2285 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2286 llvm_unreachable("widen scalar should have succeeded");
2287
2288 return;
2289 }
2290 case AMDGPU::G_FCMP:
2291 if (!Subtarget.hasSALUFloatInsts())
2292 break;
2293 [[fallthrough]];
2294 case AMDGPU::G_ICMP:
2295 case AMDGPU::G_UADDO:
2296 case AMDGPU::G_USUBO:
2297 case AMDGPU::G_UADDE:
2298 case AMDGPU::G_SADDE:
2299 case AMDGPU::G_USUBE:
2300 case AMDGPU::G_SSUBE: {
2301 unsigned BoolDstOp =
2302 (Opc == AMDGPU::G_ICMP || Opc == AMDGPU::G_FCMP) ? 0 : 1;
2303 Register DstReg = MI.getOperand(i: BoolDstOp).getReg();
2304
2305 const RegisterBank *DstBank =
2306 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2307 if (DstBank != &AMDGPU::SGPRRegBank)
2308 break;
2309
2310 const bool HasCarryIn = MI.getNumOperands() == 5;
2311
2312 // If this is a scalar compare, promote the result to s32, as the selection
2313 // will end up using a copy to a 32-bit vreg.
2314 const LLT S32 = LLT::scalar(SizeInBits: 32);
2315 Register NewDstReg = MRI.createGenericVirtualRegister(Ty: S32);
2316 MRI.setRegBank(Reg: NewDstReg, RegBank: AMDGPU::SGPRRegBank);
2317 MI.getOperand(i: BoolDstOp).setReg(NewDstReg);
2318
2319 if (HasCarryIn) {
2320 Register NewSrcReg = MRI.createGenericVirtualRegister(Ty: S32);
2321 MRI.setRegBank(Reg: NewSrcReg, RegBank: AMDGPU::SGPRRegBank);
2322 B.buildZExt(Res: NewSrcReg, Op: MI.getOperand(i: 4).getReg());
2323 MI.getOperand(i: 4).setReg(NewSrcReg);
2324 }
2325
2326 MachineBasicBlock *MBB = MI.getParent();
2327 B.setInsertPt(MBB&: *MBB, II: std::next(x: MI.getIterator()));
2328
2329 // If we had a constrained VCC result register, a copy was inserted to VCC
2330 // from SGPR.
2331 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2332 if (DefRegs.empty())
2333 DefRegs.push_back(Elt: DstReg);
2334 B.buildTrunc(Res: DefRegs[0], Op: NewDstReg);
2335 return;
2336 }
2337 case AMDGPU::G_SELECT: {
2338 Register DstReg = MI.getOperand(i: 0).getReg();
2339 LLT DstTy = MRI.getType(Reg: DstReg);
2340
2341 SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(OpIdx: 1));
2342 if (CondRegs.empty())
2343 CondRegs.push_back(Elt: MI.getOperand(i: 1).getReg());
2344 else {
2345 assert(CondRegs.size() == 1);
2346 }
2347
2348 const RegisterBank *CondBank = getRegBank(Reg: CondRegs[0], MRI, TRI: *TRI);
2349 if (CondBank == &AMDGPU::SGPRRegBank) {
2350 const LLT S32 = LLT::scalar(SizeInBits: 32);
2351 Register NewCondReg = MRI.createGenericVirtualRegister(Ty: S32);
2352 MRI.setRegBank(Reg: NewCondReg, RegBank: AMDGPU::SGPRRegBank);
2353
2354 MI.getOperand(i: 1).setReg(NewCondReg);
2355 B.buildZExt(Res: NewCondReg, Op: CondRegs[0]);
2356 }
2357
2358 if (DstTy.getSizeInBits() != 64)
2359 break;
2360
2361 LLT HalfTy = getHalfSizedType(Ty: DstTy);
2362
2363 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2364 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2365 SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(OpIdx: 3));
2366
2367 // All inputs are SGPRs, nothing special to do.
2368 if (DefRegs.empty()) {
2369 assert(Src1Regs.empty() && Src2Regs.empty());
2370 break;
2371 }
2372
2373 if (Src1Regs.empty())
2374 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2375 else {
2376 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2377 }
2378
2379 if (Src2Regs.empty())
2380 split64BitValueForMapping(B, Regs&: Src2Regs, HalfTy, Reg: MI.getOperand(i: 3).getReg());
2381 else
2382 setRegsToType(MRI, Regs: Src2Regs, NewTy: HalfTy);
2383
2384 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2385
2386 auto Flags = MI.getFlags();
2387 B.buildSelect(Res: DefRegs[0], Tst: CondRegs[0], Op0: Src1Regs[0], Op1: Src2Regs[0], Flags);
2388 B.buildSelect(Res: DefRegs[1], Tst: CondRegs[0], Op0: Src1Regs[1], Op1: Src2Regs[1], Flags);
2389
2390 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2391 MI.eraseFromParent();
2392 return;
2393 }
2394 case AMDGPU::G_BRCOND: {
2395 Register CondReg = MI.getOperand(i: 0).getReg();
2396 // FIXME: Should use legalizer helper, but should change bool ext type.
2397 const RegisterBank *CondBank =
2398 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2399
2400 if (CondBank == &AMDGPU::SGPRRegBank) {
2401 const LLT S32 = LLT::scalar(SizeInBits: 32);
2402 Register NewCondReg = MRI.createGenericVirtualRegister(Ty: S32);
2403 MRI.setRegBank(Reg: NewCondReg, RegBank: AMDGPU::SGPRRegBank);
2404
2405 MI.getOperand(i: 0).setReg(NewCondReg);
2406 B.buildZExt(Res: NewCondReg, Op: CondReg);
2407 return;
2408 }
2409
2410 break;
2411 }
2412 case AMDGPU::G_AND:
2413 case AMDGPU::G_OR:
2414 case AMDGPU::G_XOR: {
2415 // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
2416 // there is a VGPR input.
2417 Register DstReg = MI.getOperand(i: 0).getReg();
2418 LLT DstTy = MRI.getType(Reg: DstReg);
2419
2420 const RegisterBank *DstBank =
2421 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2422
2423 if (DstTy.getSizeInBits() == 1) {
2424 if (DstBank == &AMDGPU::VCCRegBank)
2425 break;
2426
2427 MachineFunction *MF = MI.getMF();
2428 ApplyRegBankMapping ApplyBank(B, *this, MRI, DstBank);
2429 LegalizerHelper Helper(*MF, ApplyBank, B);
2430
2431 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: LLT::scalar(SizeInBits: 32)) !=
2432 LegalizerHelper::Legalized)
2433 llvm_unreachable("widen scalar should have succeeded");
2434 return;
2435 }
2436
2437 if (DstTy.getSizeInBits() == 16 && DstBank == &AMDGPU::SGPRRegBank) {
2438 const LLT S32 = LLT::scalar(SizeInBits: 32);
2439 MachineBasicBlock *MBB = MI.getParent();
2440 MachineFunction *MF = MBB->getParent();
2441 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank);
2442 LegalizerHelper Helper(*MF, ApplySALU, B);
2443 // Widen to S32, but handle `G_XOR x, -1` differently. Legalizer widening
2444 // will use a G_ANYEXT to extend the -1 which prevents matching G_XOR -1
2445 // as "not".
2446 if (MI.getOpcode() == AMDGPU::G_XOR &&
2447 mi_match(R: MI.getOperand(i: 2).getReg(), MRI, P: m_SpecificICstOrSplat(RequestedValue: -1))) {
2448 Helper.widenScalarSrc(MI, WideTy: S32, OpIdx: 1, ExtOpcode: AMDGPU::G_ANYEXT);
2449 Helper.widenScalarSrc(MI, WideTy: S32, OpIdx: 2, ExtOpcode: AMDGPU::G_SEXT);
2450 Helper.widenScalarDst(MI, WideTy: S32);
2451 } else {
2452 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2453 llvm_unreachable("widen scalar should have succeeded");
2454 }
2455 return;
2456 }
2457
2458 if (DstTy.getSizeInBits() != 64)
2459 break;
2460
2461 LLT HalfTy = getHalfSizedType(Ty: DstTy);
2462 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2463 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(OpIdx: 1));
2464 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(OpIdx: 2));
2465
2466 // All inputs are SGPRs, nothing special to do.
2467 if (DefRegs.empty()) {
2468 assert(Src0Regs.empty() && Src1Regs.empty());
2469 break;
2470 }
2471
2472 assert(DefRegs.size() == 2);
2473 assert(Src0Regs.size() == Src1Regs.size() &&
2474 (Src0Regs.empty() || Src0Regs.size() == 2));
2475
2476 // Depending on where the source registers came from, the generic code may
2477 // have decided to split the inputs already or not. If not, we still need to
2478 // extract the values.
2479
2480 if (Src0Regs.empty())
2481 split64BitValueForMapping(B, Regs&: Src0Regs, HalfTy, Reg: MI.getOperand(i: 1).getReg());
2482 else
2483 setRegsToType(MRI, Regs: Src0Regs, NewTy: HalfTy);
2484
2485 if (Src1Regs.empty())
2486 split64BitValueForMapping(B, Regs&: Src1Regs, HalfTy, Reg: MI.getOperand(i: 2).getReg());
2487 else
2488 setRegsToType(MRI, Regs: Src1Regs, NewTy: HalfTy);
2489
2490 setRegsToType(MRI, Regs: DefRegs, NewTy: HalfTy);
2491
2492 auto Flags = MI.getFlags();
2493 B.buildInstr(Opc, DstOps: {DefRegs[0]}, SrcOps: {Src0Regs[0], Src1Regs[0]}, Flags);
2494 B.buildInstr(Opc, DstOps: {DefRegs[1]}, SrcOps: {Src0Regs[1], Src1Regs[1]}, Flags);
2495
2496 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2497 MI.eraseFromParent();
2498 return;
2499 }
2500 case AMDGPU::G_ABS: {
2501 Register SrcReg = MI.getOperand(i: 1).getReg();
2502 const RegisterBank *SrcBank = MRI.getRegBankOrNull(Reg: SrcReg);
2503
2504 // There is no VALU abs instruction so we need to replace it with a sub and
2505 // max combination.
2506 if (SrcBank && SrcBank == &AMDGPU::VGPRRegBank) {
2507 MachineFunction *MF = MI.getMF();
2508 ApplyRegBankMapping Apply(B, *this, MRI, &AMDGPU::VGPRRegBank);
2509 LegalizerHelper Helper(*MF, Apply, B);
2510
2511 if (Helper.lowerAbsToMaxNeg(MI) != LegalizerHelper::Legalized)
2512 llvm_unreachable("lowerAbsToMaxNeg should have succeeded");
2513 return;
2514 }
2515 [[fallthrough]];
2516 }
2517 case AMDGPU::G_ADD:
2518 case AMDGPU::G_SUB:
2519 case AMDGPU::G_MUL:
2520 case AMDGPU::G_SHL:
2521 case AMDGPU::G_LSHR:
2522 case AMDGPU::G_ASHR:
2523 case AMDGPU::G_SMIN:
2524 case AMDGPU::G_SMAX:
2525 case AMDGPU::G_UMIN:
2526 case AMDGPU::G_UMAX: {
2527 Register DstReg = MI.getOperand(i: 0).getReg();
2528 LLT DstTy = MRI.getType(Reg: DstReg);
2529
2530 // Special case for s_mul_u64. There is not a vector equivalent of
2531 // s_mul_u64. Hence, we have to break down s_mul_u64 into 32-bit vector
2532 // multiplications.
2533 if (!Subtarget.useVMulU64Inst() && Opc == AMDGPU::G_MUL &&
2534 DstTy.getSizeInBits() == 64) {
2535 applyMappingSMULU64(B, OpdMapper);
2536 return;
2537 }
2538
2539 // 16-bit operations are VALU only, but can be promoted to 32-bit SALU.
2540 // Packed 16-bit operations need to be scalarized and promoted.
2541 if (DstTy != LLT::scalar(SizeInBits: 16) && DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16))
2542 break;
2543
2544 const RegisterBank *DstBank =
2545 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2546 if (DstBank == &AMDGPU::VGPRRegBank)
2547 break;
2548
2549 const LLT S32 = LLT::scalar(SizeInBits: 32);
2550 MachineBasicBlock *MBB = MI.getParent();
2551 MachineFunction *MF = MBB->getParent();
2552 ApplyRegBankMapping ApplySALU(B, *this, MRI, &AMDGPU::SGPRRegBank);
2553
2554 if (DstTy.isVector() && Opc == AMDGPU::G_ABS) {
2555 Register WideSrcLo, WideSrcHi;
2556
2557 std::tie(args&: WideSrcLo, args&: WideSrcHi) =
2558 unpackV2S16ToS32(B, Src: MI.getOperand(i: 1).getReg(), ExtOpcode: TargetOpcode::G_SEXT);
2559 auto Lo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {S32}, SrcOps: {WideSrcLo});
2560 auto Hi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {S32}, SrcOps: {WideSrcHi});
2561 B.buildBuildVectorTrunc(Res: DstReg, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
2562 MI.eraseFromParent();
2563 return;
2564 }
2565
2566 if (DstTy.isVector()) {
2567 Register WideSrc0Lo, WideSrc0Hi;
2568 Register WideSrc1Lo, WideSrc1Hi;
2569
2570 unsigned ExtendOp = getExtendOp(Opc: MI.getOpcode());
2571 std::tie(args&: WideSrc0Lo, args&: WideSrc0Hi)
2572 = unpackV2S16ToS32(B, Src: MI.getOperand(i: 1).getReg(), ExtOpcode: ExtendOp);
2573 std::tie(args&: WideSrc1Lo, args&: WideSrc1Hi)
2574 = unpackV2S16ToS32(B, Src: MI.getOperand(i: 2).getReg(), ExtOpcode: ExtendOp);
2575 auto Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {S32}, SrcOps: {WideSrc0Lo, WideSrc1Lo});
2576 auto Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {S32}, SrcOps: {WideSrc0Hi, WideSrc1Hi});
2577 B.buildBuildVectorTrunc(Res: DstReg, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
2578 MI.eraseFromParent();
2579 } else {
2580 LegalizerHelper Helper(*MF, ApplySALU, B);
2581
2582 if (Helper.widenScalar(MI, TypeIdx: 0, WideTy: S32) != LegalizerHelper::Legalized)
2583 llvm_unreachable("widen scalar should have succeeded");
2584
2585 // FIXME: s16 shift amounts should be legal.
2586 if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR ||
2587 Opc == AMDGPU::G_ASHR) {
2588 B.setInsertPt(MBB&: *MBB, II: MI.getIterator());
2589 if (Helper.widenScalar(MI, TypeIdx: 1, WideTy: S32) != LegalizerHelper::Legalized)
2590 llvm_unreachable("widen scalar should have succeeded");
2591 }
2592 }
2593
2594 return;
2595 }
2596 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
2597 case AMDGPU::G_AMDGPU_S_MUL_U64_U32: {
2598 // This is a special case for s_mul_u64. We use
2599 // G_AMDGPU_S_MUL_I64_I32 opcode to represent an s_mul_u64 operation
2600 // where the 33 higher bits are sign-extended and
2601 // G_AMDGPU_S_MUL_U64_U32 opcode to represent an s_mul_u64 operation
2602 // where the 32 higher bits are zero-extended. In case scalar registers are
2603 // selected, both opcodes are lowered as s_mul_u64. If the vector registers
2604 // are selected, then G_AMDGPU_S_MUL_I64_I32 and
2605 // G_AMDGPU_S_MUL_U64_U32 are lowered with a vector mad instruction.
2606
2607 // Insert basic copies.
2608 applyDefaultMapping(OpdMapper);
2609
2610 Register DstReg = MI.getOperand(i: 0).getReg();
2611 Register SrcReg0 = MI.getOperand(i: 1).getReg();
2612 Register SrcReg1 = MI.getOperand(i: 2).getReg();
2613 const LLT S32 = LLT::scalar(SizeInBits: 32);
2614 const LLT S64 = LLT::scalar(SizeInBits: 64);
2615 assert(MRI.getType(DstReg) == S64 && "This is a special case for s_mul_u64 "
2616 "that handles only 64-bit operands.");
2617 const RegisterBank *DstBank =
2618 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2619
2620 // Replace G_AMDGPU_S_MUL_I64_I32 and G_AMDGPU_S_MUL_U64_U32
2621 // with s_mul_u64 operation.
2622 if (DstBank == &AMDGPU::SGPRRegBank) {
2623 MI.setDesc(TII->get(Opcode: AMDGPU::S_MUL_U64));
2624 MRI.setRegClass(Reg: DstReg, RC: &AMDGPU::SGPR_64RegClass);
2625 MRI.setRegClass(Reg: SrcReg0, RC: &AMDGPU::SGPR_64RegClass);
2626 MRI.setRegClass(Reg: SrcReg1, RC: &AMDGPU::SGPR_64RegClass);
2627 return;
2628 }
2629
2630 // Replace G_AMDGPU_S_MUL_I64_I32 and G_AMDGPU_S_MUL_U64_U32
2631 // with a vector mad.
2632 assert(MRI.getRegBankOrNull(DstReg) == &AMDGPU::VGPRRegBank &&
2633 "The destination operand should be in vector registers.");
2634
2635 // Extract the lower subregister from the first operand.
2636 Register Op0L = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2637 MRI.setRegClass(Reg: Op0L, RC: &AMDGPU::VGPR_32RegClass);
2638 MRI.setType(VReg: Op0L, Ty: S32);
2639 B.buildTrunc(Res: Op0L, Op: SrcReg0);
2640
2641 // Extract the lower subregister from the second operand.
2642 Register Op1L = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2643 MRI.setRegClass(Reg: Op1L, RC: &AMDGPU::VGPR_32RegClass);
2644 MRI.setType(VReg: Op1L, Ty: S32);
2645 B.buildTrunc(Res: Op1L, Op: SrcReg1);
2646
2647 unsigned NewOpc = Opc == AMDGPU::G_AMDGPU_S_MUL_U64_U32
2648 ? AMDGPU::G_AMDGPU_MAD_U64_U32
2649 : AMDGPU::G_AMDGPU_MAD_I64_I32;
2650
2651 MachineIRBuilder B(MI);
2652 Register Zero64 = B.buildConstant(Res: S64, Val: 0).getReg(Idx: 0);
2653 MRI.setRegClass(Reg: Zero64, RC: &AMDGPU::VReg_64RegClass);
2654 Register CarryOut = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
2655 MRI.setRegClass(Reg: CarryOut, RC: &AMDGPU::VReg_64RegClass);
2656 B.buildInstr(Opc: NewOpc, DstOps: {DstReg, CarryOut}, SrcOps: {Op0L, Op1L, Zero64});
2657 MI.eraseFromParent();
2658 return;
2659 }
2660 case AMDGPU::G_SEXT_INREG: {
2661 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
2662 if (SrcRegs.empty())
2663 break; // Nothing to repair
2664
2665 const LLT S32 = LLT::scalar(SizeInBits: 32);
2666 ApplyRegBankMapping O(B, *this, MRI, &AMDGPU::VGPRRegBank);
2667
2668 // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs
2669 // we would need to further expand, and doesn't let us directly set the
2670 // result registers.
2671 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
2672
2673 int Amt = MI.getOperand(i: 2).getImm();
2674 if (Amt <= 32) {
2675 // Downstream users have expectations for the high bit behavior, so freeze
2676 // incoming undefined bits.
2677 if (Amt == 32) {
2678 // The low bits are unchanged.
2679 B.buildFreeze(Dst: DstRegs[0], Src: SrcRegs[0]);
2680 } else {
2681 auto Freeze = B.buildFreeze(Dst: S32, Src: SrcRegs[0]);
2682 // Extend in the low bits and propagate the sign bit to the high half.
2683 B.buildSExtInReg(Res: DstRegs[0], Op: Freeze, ImmOp: Amt);
2684 }
2685
2686 B.buildAShr(Dst: DstRegs[1], Src0: DstRegs[0], Src1: B.buildConstant(Res: S32, Val: 31));
2687 } else {
2688 // The low bits are unchanged, and extend in the high bits.
2689 // No freeze required
2690 B.buildCopy(Res: DstRegs[0], Op: SrcRegs[0]);
2691 B.buildSExtInReg(Res: DstRegs[1], Op: DstRegs[0], ImmOp: Amt - 32);
2692 }
2693
2694 Register DstReg = MI.getOperand(i: 0).getReg();
2695 MRI.setRegBank(Reg: DstReg, RegBank: AMDGPU::VGPRRegBank);
2696 MI.eraseFromParent();
2697 return;
2698 }
2699 case AMDGPU::G_CTPOP:
2700 case AMDGPU::G_BITREVERSE: {
2701 const RegisterBank *DstBank =
2702 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2703 if (DstBank == &AMDGPU::SGPRRegBank)
2704 break;
2705
2706 Register SrcReg = MI.getOperand(i: 1).getReg();
2707 const LLT S32 = LLT::scalar(SizeInBits: 32);
2708 LLT Ty = MRI.getType(Reg: SrcReg);
2709 if (Ty == S32)
2710 break;
2711
2712 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank);
2713
2714 MachineFunction &MF = B.getMF();
2715 LegalizerHelper Helper(MF, ApplyVALU, B);
2716
2717 if (Helper.narrowScalar(MI, TypeIdx: 1, NarrowTy: S32) != LegalizerHelper::Legalized)
2718 llvm_unreachable("narrowScalar should have succeeded");
2719 return;
2720 }
2721 case AMDGPU::G_AMDGPU_FFBH_U32:
2722 case AMDGPU::G_AMDGPU_FFBL_B32:
2723 case AMDGPU::G_CTLZ_ZERO_POISON:
2724 case AMDGPU::G_CTTZ_ZERO_POISON: {
2725 const RegisterBank *DstBank =
2726 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
2727 if (DstBank == &AMDGPU::SGPRRegBank)
2728 break;
2729
2730 Register SrcReg = MI.getOperand(i: 1).getReg();
2731 const LLT S32 = LLT::scalar(SizeInBits: 32);
2732 LLT Ty = MRI.getType(Reg: SrcReg);
2733 if (Ty == S32)
2734 break;
2735
2736 // We can narrow this more efficiently than Helper can by using ffbh/ffbl
2737 // which return -1 when the input is zero:
2738 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
2739 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
2740 // (ffbh hi:lo) -> (umin (ffbh hi), (uaddsat (ffbh lo), 32))
2741 // (ffbl hi:lo) -> (umin (uaddsat (ffbh hi), 32), (ffbh lo))
2742 ApplyRegBankMapping ApplyVALU(B, *this, MRI, &AMDGPU::VGPRRegBank);
2743 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(OpIdx: 1));
2744 unsigned NewOpc = Opc == AMDGPU::G_CTLZ_ZERO_POISON
2745 ? (unsigned)AMDGPU::G_AMDGPU_FFBH_U32
2746 : Opc == AMDGPU::G_CTTZ_ZERO_POISON
2747 ? (unsigned)AMDGPU::G_AMDGPU_FFBL_B32
2748 : Opc;
2749 unsigned Idx = NewOpc == AMDGPU::G_AMDGPU_FFBH_U32;
2750 auto X = B.buildInstr(Opc: NewOpc, DstOps: {S32}, SrcOps: {SrcRegs[Idx]});
2751 auto Y = B.buildInstr(Opc: NewOpc, DstOps: {S32}, SrcOps: {SrcRegs[Idx ^ 1]});
2752 unsigned AddOpc =
2753 Opc == AMDGPU::G_CTLZ_ZERO_POISON || Opc == AMDGPU::G_CTTZ_ZERO_POISON
2754 ? AMDGPU::G_ADD
2755 : AMDGPU::G_UADDSAT;
2756 Y = B.buildInstr(Opc: AddOpc, DstOps: {S32}, SrcOps: {Y, B.buildConstant(Res: S32, Val: 32)});
2757 Register DstReg = MI.getOperand(i: 0).getReg();
2758 B.buildUMin(Dst: DstReg, Src0: X, Src1: Y);
2759 MI.eraseFromParent();
2760 return;
2761 }
2762 case AMDGPU::G_SEXT:
2763 case AMDGPU::G_ZEXT:
2764 case AMDGPU::G_ANYEXT: {
2765 Register SrcReg = MI.getOperand(i: 1).getReg();
2766 LLT SrcTy = MRI.getType(Reg: SrcReg);
2767 const bool Signed = Opc == AMDGPU::G_SEXT;
2768
2769 assert(OpdMapper.getVRegs(1).empty());
2770
2771 const RegisterBank *SrcBank =
2772 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2773
2774 Register DstReg = MI.getOperand(i: 0).getReg();
2775 LLT DstTy = MRI.getType(Reg: DstReg);
2776 if (DstTy.isScalar() &&
2777 SrcBank != &AMDGPU::SGPRRegBank &&
2778 SrcBank != &AMDGPU::VCCRegBank &&
2779 // FIXME: Should handle any type that round to s64 when irregular
2780 // breakdowns supported.
2781 DstTy.getSizeInBits() == 64 &&
2782 SrcTy.getSizeInBits() <= 32) {
2783 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2784
2785 // Extend to 32-bit, and then extend the low half.
2786 if (Signed) {
2787 // TODO: Should really be buildSExtOrCopy
2788 B.buildSExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2789 } else if (Opc == AMDGPU::G_ZEXT) {
2790 B.buildZExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2791 } else {
2792 B.buildAnyExtOrTrunc(Res: DefRegs[0], Op: SrcReg);
2793 }
2794
2795 extendLow32IntoHigh32(B, Hi32Reg: DefRegs[1], Lo32Reg: DefRegs[0], ExtOpc: Opc, RegBank: *SrcBank);
2796 MRI.setRegBank(Reg: DstReg, RegBank: *SrcBank);
2797 MI.eraseFromParent();
2798 return;
2799 }
2800
2801 if (SrcTy != LLT::scalar(SizeInBits: 1))
2802 return;
2803
2804 // It is not legal to have a legalization artifact with a VCC source. Rather
2805 // than introducing a copy, insert the select we would have to select the
2806 // copy to.
2807 if (SrcBank == &AMDGPU::VCCRegBank) {
2808 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(OpIdx: 0));
2809
2810 const RegisterBank *DstBank = &AMDGPU::VGPRRegBank;
2811
2812 unsigned DstSize = DstTy.getSizeInBits();
2813 // 64-bit select is SGPR only
2814 const bool UseSel64 = DstSize > 32 &&
2815 SrcBank->getID() == AMDGPU::SGPRRegBankID;
2816
2817 // TODO: Should s16 select be legal?
2818 LLT SelType = UseSel64 ? LLT::scalar(SizeInBits: 64) : LLT::scalar(SizeInBits: 32);
2819 auto True = B.buildConstant(Res: SelType, Val: Signed ? -1 : 1);
2820 auto False = B.buildConstant(Res: SelType, Val: 0);
2821
2822 MRI.setRegBank(Reg: True.getReg(Idx: 0), RegBank: *DstBank);
2823 MRI.setRegBank(Reg: False.getReg(Idx: 0), RegBank: *DstBank);
2824 MRI.setRegBank(Reg: DstReg, RegBank: *DstBank);
2825
2826 if (DstSize > 32) {
2827 B.buildSelect(Res: DefRegs[0], Tst: SrcReg, Op0: True, Op1: False);
2828 extendLow32IntoHigh32(B, Hi32Reg: DefRegs[1], Lo32Reg: DefRegs[0], ExtOpc: Opc, RegBank: *SrcBank, IsBooleanSrc: true);
2829 } else if (DstSize < 32) {
2830 auto Sel = B.buildSelect(Res: SelType, Tst: SrcReg, Op0: True, Op1: False);
2831 MRI.setRegBank(Reg: Sel.getReg(Idx: 0), RegBank: *DstBank);
2832 B.buildTrunc(Res: DstReg, Op: Sel);
2833 } else {
2834 B.buildSelect(Res: DstReg, Tst: SrcReg, Op0: True, Op1: False);
2835 }
2836
2837 MI.eraseFromParent();
2838 return;
2839 }
2840
2841 break;
2842 }
2843 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
2844 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(OpIdx: 0));
2845
2846 assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty());
2847
2848 Register DstReg = MI.getOperand(i: 0).getReg();
2849 Register SrcReg = MI.getOperand(i: 1).getReg();
2850
2851 const LLT S32 = LLT::scalar(SizeInBits: 32);
2852 LLT DstTy = MRI.getType(Reg: DstReg);
2853 LLT SrcTy = MRI.getType(Reg: SrcReg);
2854
2855 if (foldExtractEltToCmpSelect(B, MI, OpdMapper))
2856 return;
2857
2858 const ValueMapping &DstMapping
2859 = OpdMapper.getInstrMapping().getOperandMapping(i: 0);
2860 const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank;
2861 const RegisterBank *SrcBank =
2862 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
2863 const RegisterBank *IdxBank =
2864 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
2865
2866 Register BaseIdxReg;
2867 unsigned ConstOffset;
2868 std::tie(args&: BaseIdxReg, args&: ConstOffset) =
2869 AMDGPU::getBaseWithConstantOffset(MRI, Reg: MI.getOperand(i: 2).getReg());
2870
2871 // See if the index is an add of a constant which will be foldable by moving
2872 // the base register of the index later if this is going to be executed in a
2873 // waterfall loop. This is essentially to reassociate the add of a constant
2874 // with the readfirstlane.
2875 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
2876 ConstOffset > 0 &&
2877 ConstOffset < SrcTy.getNumElements();
2878
2879 // Move the base register. We'll re-insert the add later.
2880 if (ShouldMoveIndexIntoLoop)
2881 MI.getOperand(i: 2).setReg(BaseIdxReg);
2882
2883 // If this is a VGPR result only because the index was a VGPR result, the
2884 // actual indexing will be done on the SGPR source vector, which will
2885 // produce a scalar result. We need to copy to the VGPR result inside the
2886 // waterfall loop.
2887 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank &&
2888 SrcBank == &AMDGPU::SGPRRegBank;
2889 if (DstRegs.empty()) {
2890 applyDefaultMapping(OpdMapper);
2891
2892 executeInWaterfallLoop(B, MI, OpIndices: {2});
2893
2894 if (NeedCopyToVGPR) {
2895 // We don't want a phi for this temporary reg.
2896 Register TmpReg = MRI.createGenericVirtualRegister(Ty: DstTy);
2897 MRI.setRegBank(Reg: TmpReg, RegBank: AMDGPU::SGPRRegBank);
2898 MI.getOperand(i: 0).setReg(TmpReg);
2899 B.setInsertPt(MBB&: *MI.getParent(), II: ++MI.getIterator());
2900
2901 // Use a v_mov_b32 here to make the exec dependency explicit.
2902 buildVCopy(B, DstReg, SrcReg: TmpReg);
2903 }
2904
2905 // Re-insert the constant offset add inside the waterfall loop.
2906 if (ShouldMoveIndexIntoLoop)
2907 reinsertVectorIndexAdd(B, IdxUseInstr&: MI, OpIdx: 2, ConstOffset);
2908
2909 return;
2910 }
2911
2912 assert(DstTy.getSizeInBits() == 64);
2913
2914 LLT Vec32 = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
2915
2916 auto CastSrc = B.buildBitcast(Dst: Vec32, Src: SrcReg);
2917 auto One = B.buildConstant(Res: S32, Val: 1);
2918
2919 MachineBasicBlock::iterator MII = MI.getIterator();
2920
2921 // Split the vector index into 32-bit pieces. Prepare to move all of the
2922 // new instructions into a waterfall loop if necessary.
2923 //
2924 // Don't put the bitcast or constant in the loop.
2925 MachineInstrSpan Span(MII, &B.getMBB());
2926
2927 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
2928 auto IdxLo = B.buildShl(Dst: S32, Src0: BaseIdxReg, Src1: One);
2929 auto IdxHi = B.buildAdd(Dst: S32, Src0: IdxLo, Src1: One);
2930
2931 auto Extract0 = B.buildExtractVectorElement(Res: DstRegs[0], Val: CastSrc, Idx: IdxLo);
2932 auto Extract1 = B.buildExtractVectorElement(Res: DstRegs[1], Val: CastSrc, Idx: IdxHi);
2933
2934 MRI.setRegBank(Reg: DstReg, RegBank: *DstBank);
2935 MRI.setRegBank(Reg: CastSrc.getReg(Idx: 0), RegBank: *SrcBank);
2936 MRI.setRegBank(Reg: One.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2937 MRI.setRegBank(Reg: IdxLo.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2938 MRI.setRegBank(Reg: IdxHi.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
2939
2940 SmallSet<Register, 4> OpsToWaterfall;
2941 if (!collectWaterfallOperands(SGPROperandRegs&: OpsToWaterfall, MI, MRI, OpIndices: { 2 })) {
2942 MI.eraseFromParent();
2943 return;
2944 }
2945
2946 // Remove the original instruction to avoid potentially confusing the
2947 // waterfall loop logic.
2948 B.setInstr(*Span.begin());
2949 MI.eraseFromParent();
2950 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
2951 SGPROperandRegs&: OpsToWaterfall);
2952
2953 if (NeedCopyToVGPR) {
2954 MachineBasicBlock *LoopBB = Extract1->getParent();
2955 Register TmpReg0 = MRI.createGenericVirtualRegister(Ty: S32);
2956 Register TmpReg1 = MRI.createGenericVirtualRegister(Ty: S32);
2957 MRI.setRegBank(Reg: TmpReg0, RegBank: AMDGPU::SGPRRegBank);
2958 MRI.setRegBank(Reg: TmpReg1, RegBank: AMDGPU::SGPRRegBank);
2959
2960 Extract0->getOperand(i: 0).setReg(TmpReg0);
2961 Extract1->getOperand(i: 0).setReg(TmpReg1);
2962
2963 B.setInsertPt(MBB&: *LoopBB, II: ++Extract1->getIterator());
2964
2965 buildVCopy(B, DstReg: DstRegs[0], SrcReg: TmpReg0);
2966 buildVCopy(B, DstReg: DstRegs[1], SrcReg: TmpReg1);
2967 }
2968
2969 if (ShouldMoveIndexIntoLoop)
2970 reinsertVectorIndexAdd(B, IdxUseInstr&: *IdxLo, OpIdx: 1, ConstOffset);
2971
2972 return;
2973 }
2974 case AMDGPU::G_INSERT_VECTOR_ELT: {
2975 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(OpIdx: 2));
2976
2977 Register DstReg = MI.getOperand(i: 0).getReg();
2978 LLT VecTy = MRI.getType(Reg: DstReg);
2979
2980 assert(OpdMapper.getVRegs(0).empty());
2981 assert(OpdMapper.getVRegs(3).empty());
2982
2983 if (substituteSimpleCopyRegs(OpdMapper, OpIdx: 1))
2984 MRI.setType(VReg: MI.getOperand(i: 1).getReg(), Ty: VecTy);
2985
2986 if (foldInsertEltToCmpSelect(B, MI, OpdMapper))
2987 return;
2988
2989 const RegisterBank *IdxBank =
2990 OpdMapper.getInstrMapping().getOperandMapping(i: 3).BreakDown[0].RegBank;
2991
2992 Register SrcReg = MI.getOperand(i: 1).getReg();
2993 Register InsReg = MI.getOperand(i: 2).getReg();
2994 LLT InsTy = MRI.getType(Reg: InsReg);
2995 (void)InsTy;
2996
2997 Register BaseIdxReg;
2998 unsigned ConstOffset;
2999 std::tie(args&: BaseIdxReg, args&: ConstOffset) =
3000 AMDGPU::getBaseWithConstantOffset(MRI, Reg: MI.getOperand(i: 3).getReg());
3001
3002 // See if the index is an add of a constant which will be foldable by moving
3003 // the base register of the index later if this is going to be executed in a
3004 // waterfall loop. This is essentially to reassociate the add of a constant
3005 // with the readfirstlane.
3006 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank &&
3007 ConstOffset > 0 &&
3008 ConstOffset < VecTy.getNumElements();
3009
3010 // Move the base register. We'll re-insert the add later.
3011 if (ShouldMoveIndexIntoLoop)
3012 MI.getOperand(i: 3).setReg(BaseIdxReg);
3013
3014
3015 if (InsRegs.empty()) {
3016 executeInWaterfallLoop(B, MI, OpIndices: {3});
3017
3018 // Re-insert the constant offset add inside the waterfall loop.
3019 if (ShouldMoveIndexIntoLoop) {
3020 reinsertVectorIndexAdd(B, IdxUseInstr&: MI, OpIdx: 3, ConstOffset);
3021 }
3022
3023 return;
3024 }
3025
3026 assert(InsTy.getSizeInBits() == 64);
3027
3028 const LLT S32 = LLT::scalar(SizeInBits: 32);
3029 LLT Vec32 = LLT::fixed_vector(NumElements: 2 * VecTy.getNumElements(), ScalarSizeInBits: 32);
3030
3031 auto CastSrc = B.buildBitcast(Dst: Vec32, Src: SrcReg);
3032 auto One = B.buildConstant(Res: S32, Val: 1);
3033
3034 // Split the vector index into 32-bit pieces. Prepare to move all of the
3035 // new instructions into a waterfall loop if necessary.
3036 //
3037 // Don't put the bitcast or constant in the loop.
3038 MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB());
3039
3040 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1).
3041 auto IdxLo = B.buildShl(Dst: S32, Src0: BaseIdxReg, Src1: One);
3042 auto IdxHi = B.buildAdd(Dst: S32, Src0: IdxLo, Src1: One);
3043
3044 auto InsLo = B.buildInsertVectorElement(Res: Vec32, Val: CastSrc, Elt: InsRegs[0], Idx: IdxLo);
3045 auto InsHi = B.buildInsertVectorElement(Res: Vec32, Val: InsLo, Elt: InsRegs[1], Idx: IdxHi);
3046
3047 const RegisterBank *DstBank =
3048 OpdMapper.getInstrMapping().getOperandMapping(i: 0).BreakDown[0].RegBank;
3049 const RegisterBank *SrcBank =
3050 OpdMapper.getInstrMapping().getOperandMapping(i: 1).BreakDown[0].RegBank;
3051 const RegisterBank *InsSrcBank =
3052 OpdMapper.getInstrMapping().getOperandMapping(i: 2).BreakDown[0].RegBank;
3053
3054 MRI.setRegBank(Reg: InsReg, RegBank: *InsSrcBank);
3055 MRI.setRegBank(Reg: CastSrc.getReg(Idx: 0), RegBank: *SrcBank);
3056 MRI.setRegBank(Reg: InsLo.getReg(Idx: 0), RegBank: *DstBank);
3057 MRI.setRegBank(Reg: InsHi.getReg(Idx: 0), RegBank: *DstBank);
3058 MRI.setRegBank(Reg: One.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
3059 MRI.setRegBank(Reg: IdxLo.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
3060 MRI.setRegBank(Reg: IdxHi.getReg(Idx: 0), RegBank: AMDGPU::SGPRRegBank);
3061
3062
3063 SmallSet<Register, 4> OpsToWaterfall;
3064 if (!collectWaterfallOperands(SGPROperandRegs&: OpsToWaterfall, MI, MRI, OpIndices: { 3 })) {
3065 B.setInsertPt(MBB&: B.getMBB(), II: MI);
3066 B.buildBitcast(Dst: DstReg, Src: InsHi);
3067 MI.eraseFromParent();
3068 return;
3069 }
3070
3071 B.setInstr(*Span.begin());
3072 MI.eraseFromParent();
3073
3074 // Figure out the point after the waterfall loop before mangling the control
3075 // flow.
3076 executeInWaterfallLoop(B, Range: make_range(x: Span.begin(), y: Span.end()),
3077 SGPROperandRegs&: OpsToWaterfall);
3078
3079 // The insertion point is now right after the original instruction.
3080 //
3081 // Keep the bitcast to the original vector type out of the loop. Doing this
3082 // saved an extra phi we don't need inside the loop.
3083 B.buildBitcast(Dst: DstReg, Src: InsHi);
3084
3085 // Re-insert the constant offset add inside the waterfall loop.
3086 if (ShouldMoveIndexIntoLoop)
3087 reinsertVectorIndexAdd(B, IdxUseInstr&: *IdxLo, OpIdx: 1, ConstOffset);
3088
3089 return;
3090 }
3091 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
3092 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
3093 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
3094 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
3095 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
3096 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
3097 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
3098 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
3099 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
3100 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
3101 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
3102 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
3103 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
3104 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
3105 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
3106 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
3107 case AMDGPU::G_AMDGPU_BUFFER_STORE:
3108 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
3109 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
3110 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
3111 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16:
3112 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
3113 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: {
3114 applyDefaultMapping(OpdMapper);
3115 executeInWaterfallLoop(B, MI, OpIndices: {1, 4});
3116 return;
3117 }
3118 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
3119 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
3120 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
3121 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
3122 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
3123 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
3124 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
3125 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
3126 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
3127 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
3128 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
3129 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
3130 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
3131 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
3132 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
3133 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
3134 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
3135 applyDefaultMapping(OpdMapper);
3136 executeInWaterfallLoop(B, MI, OpIndices: {2, 5});
3137 return;
3138 }
3139 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
3140 applyDefaultMapping(OpdMapper);
3141 executeInWaterfallLoop(B, MI, OpIndices: {3, 6});
3142 return;
3143 }
3144 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
3145 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
3146 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
3147 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
3148 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
3149 applyMappingSBufferLoad(B, OpdMapper);
3150 return;
3151 }
3152 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
3153 constrainOpWithReadfirstlane(B, MI, OpIdx: 0);
3154 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3155 return;
3156 case AMDGPU::G_INTRINSIC:
3157 case AMDGPU::G_INTRINSIC_CONVERGENT: {
3158 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3159 case Intrinsic::amdgcn_readlane: {
3160 substituteSimpleCopyRegs(OpdMapper, OpIdx: 2);
3161
3162 assert(OpdMapper.getVRegs(0).empty());
3163 assert(OpdMapper.getVRegs(3).empty());
3164
3165 // Make sure the index is an SGPR. It doesn't make sense to run this in a
3166 // waterfall loop, so assume it's a uniform value.
3167 constrainOpWithReadfirstlane(B, MI, OpIdx: 3); // Index
3168 return;
3169 }
3170 case Intrinsic::amdgcn_writelane: {
3171 assert(OpdMapper.getVRegs(0).empty());
3172 assert(OpdMapper.getVRegs(2).empty());
3173 assert(OpdMapper.getVRegs(3).empty());
3174
3175 substituteSimpleCopyRegs(OpdMapper, OpIdx: 4); // VGPR input val
3176 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // Source value
3177 constrainOpWithReadfirstlane(B, MI, OpIdx: 3); // Index
3178 return;
3179 }
3180 case Intrinsic::amdgcn_interp_p1:
3181 case Intrinsic::amdgcn_interp_p2:
3182 case Intrinsic::amdgcn_interp_mov:
3183 case Intrinsic::amdgcn_interp_p1_f16:
3184 case Intrinsic::amdgcn_interp_p2_f16:
3185 case Intrinsic::amdgcn_lds_param_load: {
3186 applyDefaultMapping(OpdMapper);
3187
3188 // Readlane for m0 value, which is always the last operand.
3189 // FIXME: Should this be a waterfall loop instead?
3190 constrainOpWithReadfirstlane(B, MI, OpIdx: MI.getNumOperands() - 1); // Index
3191 return;
3192 }
3193 case Intrinsic::amdgcn_interp_inreg_p10:
3194 case Intrinsic::amdgcn_interp_inreg_p2:
3195 case Intrinsic::amdgcn_interp_inreg_p10_f16:
3196 case Intrinsic::amdgcn_interp_inreg_p2_f16:
3197 case Intrinsic::amdgcn_interp_p10_rtz_f16:
3198 case Intrinsic::amdgcn_interp_p2_rtz_f16:
3199 case Intrinsic::amdgcn_permlane16_swap:
3200 case Intrinsic::amdgcn_permlane32_swap:
3201 applyDefaultMapping(OpdMapper);
3202 return;
3203 case Intrinsic::amdgcn_permlane16:
3204 case Intrinsic::amdgcn_permlanex16: {
3205 // Doing a waterfall loop over these wouldn't make any sense.
3206 substituteSimpleCopyRegs(OpdMapper, OpIdx: 2);
3207 substituteSimpleCopyRegs(OpdMapper, OpIdx: 3);
3208 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3209 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3210 return;
3211 }
3212 case Intrinsic::amdgcn_permlane_bcast:
3213 case Intrinsic::amdgcn_permlane_up:
3214 case Intrinsic::amdgcn_permlane_down:
3215 case Intrinsic::amdgcn_permlane_xor:
3216 // Doing a waterfall loop over these wouldn't make any sense.
3217 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3218 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3219 return;
3220 case Intrinsic::amdgcn_permlane_idx_gen: {
3221 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3222 return;
3223 }
3224 case Intrinsic::amdgcn_sbfe:
3225 applyMappingBFE(B, OpdMapper, Signed: true);
3226 return;
3227 case Intrinsic::amdgcn_ubfe:
3228 applyMappingBFE(B, OpdMapper, Signed: false);
3229 return;
3230 case Intrinsic::amdgcn_inverse_ballot:
3231 case Intrinsic::amdgcn_s_bitreplicate:
3232 case Intrinsic::amdgcn_s_quadmask:
3233 case Intrinsic::amdgcn_s_wqm:
3234 applyDefaultMapping(OpdMapper);
3235 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // Mask
3236 return;
3237 case Intrinsic::amdgcn_ballot:
3238 // Use default handling and insert copy to vcc source.
3239 break;
3240 }
3241 break;
3242 }
3243 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
3244 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
3245 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
3246 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
3247 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
3248 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3249 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
3250 assert(RSrcIntrin && RSrcIntrin->IsImage);
3251 // Non-images can have complications from operands that allow both SGPR
3252 // and VGPR. For now it's too complicated to figure out the final opcode
3253 // to derive the register bank from the MCInstrDesc.
3254 applyMappingImage(B, MI, OpdMapper, RsrcIdx: RSrcIntrin->RsrcArg);
3255 return;
3256 }
3257 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
3258 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
3259 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
3260 bool IsDualOrBVH8 =
3261 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
3262 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
3263 unsigned NumMods = IsDualOrBVH8 ? 0 : 1; // Has A16 modifier
3264 unsigned LastRegOpIdx = MI.getNumExplicitOperands() - 1 - NumMods;
3265 applyDefaultMapping(OpdMapper);
3266 executeInWaterfallLoop(B, MI, OpIndices: {LastRegOpIdx});
3267 return;
3268 }
3269 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
3270 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
3271 auto IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3272 switch (IntrID) {
3273 case Intrinsic::amdgcn_ds_ordered_add:
3274 case Intrinsic::amdgcn_ds_ordered_swap: {
3275 // This is only allowed to execute with 1 lane, so readfirstlane is safe.
3276 assert(OpdMapper.getVRegs(0).empty());
3277 substituteSimpleCopyRegs(OpdMapper, OpIdx: 3);
3278 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3279 return;
3280 }
3281 case Intrinsic::amdgcn_ds_gws_init:
3282 case Intrinsic::amdgcn_ds_gws_barrier:
3283 case Intrinsic::amdgcn_ds_gws_sema_br: {
3284 // Only the first lane is executes, so readfirstlane is safe.
3285 substituteSimpleCopyRegs(OpdMapper, OpIdx: 1);
3286 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3287 return;
3288 }
3289 case Intrinsic::amdgcn_ds_gws_sema_v:
3290 case Intrinsic::amdgcn_ds_gws_sema_p:
3291 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
3292 // Only the first lane is executes, so readfirstlane is safe.
3293 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // M0
3294 return;
3295 }
3296 case Intrinsic::amdgcn_ds_append:
3297 case Intrinsic::amdgcn_ds_consume: {
3298 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3299 return;
3300 }
3301 case Intrinsic::amdgcn_s_alloc_vgpr:
3302 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3303 return;
3304 case Intrinsic::amdgcn_s_sendmsg:
3305 case Intrinsic::amdgcn_s_sendmsghalt: {
3306 // FIXME: Should this use a waterfall loop?
3307 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3308 return;
3309 }
3310 case Intrinsic::amdgcn_s_setreg: {
3311 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3312 return;
3313 }
3314 case Intrinsic::amdgcn_s_ttracedata:
3315 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // M0
3316 return;
3317 case Intrinsic::amdgcn_raw_buffer_load_lds:
3318 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3319 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
3320 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: {
3321 applyDefaultMapping(OpdMapper);
3322 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // rsrc
3323 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3324 constrainOpWithReadfirstlane(B, MI, OpIdx: 5); // soffset
3325 return;
3326 }
3327 case Intrinsic::amdgcn_struct_buffer_load_lds:
3328 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3329 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
3330 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
3331 applyDefaultMapping(OpdMapper);
3332 constrainOpWithReadfirstlane(B, MI, OpIdx: 1); // rsrc
3333 constrainOpWithReadfirstlane(B, MI, OpIdx: 2); // M0
3334 constrainOpWithReadfirstlane(B, MI, OpIdx: 6); // soffset
3335 return;
3336 }
3337 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
3338 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
3339 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
3340 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
3341 applyDefaultMapping(OpdMapper);
3342 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3343 return;
3344 }
3345 case Intrinsic::amdgcn_load_to_lds:
3346 case Intrinsic::amdgcn_load_async_to_lds:
3347 case Intrinsic::amdgcn_global_load_lds:
3348 case Intrinsic::amdgcn_global_load_async_lds: {
3349 applyDefaultMapping(OpdMapper);
3350 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3351 return;
3352 }
3353 case Intrinsic::amdgcn_lds_direct_load: {
3354 applyDefaultMapping(OpdMapper);
3355 // Readlane for m0 value, which is always the last operand.
3356 constrainOpWithReadfirstlane(B, MI, OpIdx: MI.getNumOperands() - 1); // Index
3357 return;
3358 }
3359 case Intrinsic::amdgcn_exp_row:
3360 applyDefaultMapping(OpdMapper);
3361 constrainOpWithReadfirstlane(B, MI, OpIdx: 8); // M0
3362 return;
3363 case Intrinsic::amdgcn_cluster_load_b32:
3364 case Intrinsic::amdgcn_cluster_load_b64:
3365 case Intrinsic::amdgcn_cluster_load_b128: {
3366 applyDefaultMapping(OpdMapper);
3367 constrainOpWithReadfirstlane(B, MI, OpIdx: 4); // M0
3368 return;
3369 }
3370 case Intrinsic::amdgcn_s_sleep_var:
3371 assert(OpdMapper.getVRegs(1).empty());
3372 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3373 return;
3374 case Intrinsic::amdgcn_s_barrier_join:
3375 case Intrinsic::amdgcn_s_wakeup_barrier:
3376 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3377 return;
3378 case Intrinsic::amdgcn_s_barrier_init:
3379 case Intrinsic::amdgcn_s_barrier_signal_var:
3380 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3381 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3382 return;
3383 case Intrinsic::amdgcn_s_get_barrier_state:
3384 case Intrinsic::amdgcn_s_get_named_barrier_state: {
3385 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3386 return;
3387 }
3388 case Intrinsic::amdgcn_s_prefetch_data:
3389 case Intrinsic::amdgcn_s_prefetch_inst: {
3390 Register PtrReg = MI.getOperand(i: 1).getReg();
3391 unsigned AS = MRI.getType(Reg: PtrReg).getAddressSpace();
3392 if (AMDGPU::isFlatGlobalAddrSpace(AS)) {
3393 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3394 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3395 } else
3396 MI.eraseFromParent();
3397 return;
3398 }
3399 case Intrinsic::amdgcn_tensor_load_to_lds:
3400 case Intrinsic::amdgcn_tensor_store_from_lds: {
3401 constrainOpWithReadfirstlane(B, MI, OpIdx: 1);
3402 constrainOpWithReadfirstlane(B, MI, OpIdx: 2);
3403 constrainOpWithReadfirstlane(B, MI, OpIdx: 3);
3404 constrainOpWithReadfirstlane(B, MI, OpIdx: 4);
3405 constrainOpWithReadfirstlane(B, MI, OpIdx: 5);
3406 return;
3407 }
3408 default: {
3409 if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
3410 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
3411 // Non-images can have complications from operands that allow both SGPR
3412 // and VGPR. For now it's too complicated to figure out the final opcode
3413 // to derive the register bank from the MCInstrDesc.
3414 if (RSrcIntrin->IsImage) {
3415 applyMappingImage(B, MI, OpdMapper, RsrcIdx: RSrcIntrin->RsrcArg);
3416 return;
3417 }
3418 }
3419
3420 break;
3421 }
3422 }
3423 break;
3424 }
3425 case AMDGPU::G_SI_CALL: {
3426 // Use a set to avoid extra readfirstlanes in the case where multiple
3427 // operands are the same register.
3428 SmallSet<Register, 4> SGPROperandRegs;
3429
3430 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI, OpIndices: {1}))
3431 break;
3432
3433 // Move all copies to physical SGPRs that are used by the call instruction
3434 // into the loop block. Start searching for these copies until the
3435 // ADJCALLSTACKUP.
3436 unsigned FrameSetupOpcode = AMDGPU::ADJCALLSTACKUP;
3437 unsigned FrameDestroyOpcode = AMDGPU::ADJCALLSTACKDOWN;
3438
3439 // Move all non-copies before the copies, so that a complete range can be
3440 // moved into the waterfall loop.
3441 SmallVector<MachineInstr *, 4> NonCopyInstrs;
3442 // Count of NonCopyInstrs found until the current LastCopy.
3443 unsigned NonCopyInstrsLen = 0;
3444 MachineBasicBlock::iterator Start(&MI);
3445 MachineBasicBlock::iterator LastCopy = Start;
3446 MachineBasicBlock *MBB = MI.getParent();
3447 const SIMachineFunctionInfo *Info =
3448 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
3449 while (Start->getOpcode() != FrameSetupOpcode) {
3450 --Start;
3451 bool IsCopy = false;
3452 if (Start->getOpcode() == AMDGPU::COPY) {
3453 auto &Dst = Start->getOperand(i: 0);
3454 if (Dst.isReg()) {
3455 Register Reg = Dst.getReg();
3456 if (Reg.isPhysical() && MI.readsRegister(Reg, TRI)) {
3457 IsCopy = true;
3458 } else {
3459 // Also move the copy from the scratch rsrc descriptor into the loop
3460 // to allow it to be optimized away.
3461 auto &Src = Start->getOperand(i: 1);
3462 if (Src.isReg()) {
3463 Reg = Src.getReg();
3464 IsCopy = Info->getScratchRSrcReg() == Reg;
3465 }
3466 }
3467 }
3468 }
3469
3470 if (IsCopy) {
3471 LastCopy = Start;
3472 NonCopyInstrsLen = NonCopyInstrs.size();
3473 } else {
3474 NonCopyInstrs.push_back(Elt: &*Start);
3475 }
3476 }
3477 NonCopyInstrs.resize(N: NonCopyInstrsLen);
3478
3479 for (auto *NonCopy : reverse(C&: NonCopyInstrs)) {
3480 MBB->splice(Where: LastCopy, Other: MBB, From: NonCopy->getIterator());
3481 }
3482 Start = LastCopy;
3483
3484 // Do the same for copies after the loop
3485 NonCopyInstrs.clear();
3486 NonCopyInstrsLen = 0;
3487 MachineBasicBlock::iterator End(&MI);
3488 LastCopy = End;
3489 while (End->getOpcode() != FrameDestroyOpcode) {
3490 ++End;
3491 bool IsCopy = false;
3492 if (End->getOpcode() == AMDGPU::COPY) {
3493 auto &Src = End->getOperand(i: 1);
3494 if (Src.isReg()) {
3495 Register Reg = Src.getReg();
3496 IsCopy = Reg.isPhysical() && MI.modifiesRegister(Reg, TRI);
3497 }
3498 }
3499
3500 if (IsCopy) {
3501 LastCopy = End;
3502 NonCopyInstrsLen = NonCopyInstrs.size();
3503 } else {
3504 NonCopyInstrs.push_back(Elt: &*End);
3505 }
3506 }
3507 NonCopyInstrs.resize(N: NonCopyInstrsLen);
3508
3509 End = LastCopy;
3510 ++LastCopy;
3511 for (auto *NonCopy : reverse(C&: NonCopyInstrs)) {
3512 MBB->splice(Where: LastCopy, Other: MBB, From: NonCopy->getIterator());
3513 }
3514
3515 ++End;
3516 B.setInsertPt(MBB&: B.getMBB(), II: Start);
3517 executeInWaterfallLoop(B, Range: make_range(x: Start, y: End), SGPROperandRegs);
3518 break;
3519 }
3520 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
3521 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR:
3522 case AMDGPU::G_LOAD:
3523 case AMDGPU::G_ZEXTLOAD:
3524 case AMDGPU::G_SEXTLOAD: {
3525 if (applyMappingLoad(B, OpdMapper, MI))
3526 return;
3527 break;
3528 }
3529 case AMDGPU::G_DYN_STACKALLOC:
3530 applyMappingDynStackAlloc(B, OpdMapper, MI);
3531 return;
3532 case AMDGPU::G_STACKRESTORE: {
3533 applyDefaultMapping(OpdMapper);
3534 constrainOpWithReadfirstlane(B, MI, OpIdx: 0);
3535 return;
3536 }
3537 case AMDGPU::G_SBFX:
3538 applyMappingBFE(B, OpdMapper, /*Signed*/ true);
3539 return;
3540 case AMDGPU::G_UBFX:
3541 applyMappingBFE(B, OpdMapper, /*Signed*/ false);
3542 return;
3543 case AMDGPU::G_AMDGPU_MAD_U64_U32:
3544 case AMDGPU::G_AMDGPU_MAD_I64_I32:
3545 applyMappingMAD_64_32(B, OpdMapper);
3546 return;
3547 case AMDGPU::G_PREFETCH: {
3548 if (!Subtarget.hasSafeSmemPrefetch() && !Subtarget.hasVmemPrefInsts()) {
3549 MI.eraseFromParent();
3550 return;
3551 }
3552 Register PtrReg = MI.getOperand(i: 0).getReg();
3553 unsigned PtrBank = getRegBankID(Reg: PtrReg, MRI, Default: AMDGPU::SGPRRegBankID);
3554 if (PtrBank == AMDGPU::VGPRRegBankID &&
3555 (!Subtarget.hasVmemPrefInsts() || !MI.getOperand(i: 3).getImm())) {
3556 // Cannot do I$ prefetch with divergent pointer.
3557 MI.eraseFromParent();
3558 return;
3559 }
3560 unsigned AS = MRI.getType(Reg: PtrReg).getAddressSpace();
3561 if ((!AMDGPU::isFlatGlobalAddrSpace(AS) &&
3562 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
3563 (!Subtarget.hasSafeSmemPrefetch() &&
3564 (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
3565 !MI.getOperand(i: 3).getImm() /* I$ prefetch */))) {
3566 MI.eraseFromParent();
3567 return;
3568 }
3569 applyDefaultMapping(OpdMapper);
3570 return;
3571 }
3572 default:
3573 break;
3574 }
3575
3576 return applyDefaultMapping(OpdMapper);
3577}
3578
3579// vgpr, sgpr -> vgpr
3580// vgpr, agpr -> vgpr
3581// agpr, agpr -> agpr
3582// agpr, sgpr -> vgpr
3583static unsigned regBankUnion(unsigned RB0, unsigned RB1) {
3584 if (RB0 == AMDGPU::InvalidRegBankID)
3585 return RB1;
3586 if (RB1 == AMDGPU::InvalidRegBankID)
3587 return RB0;
3588
3589 if (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID)
3590 return AMDGPU::SGPRRegBankID;
3591
3592 if (RB0 == AMDGPU::AGPRRegBankID && RB1 == AMDGPU::AGPRRegBankID)
3593 return AMDGPU::AGPRRegBankID;
3594
3595 return AMDGPU::VGPRRegBankID;
3596}
3597
3598static unsigned regBankBoolUnion(unsigned RB0, unsigned RB1) {
3599 if (RB0 == AMDGPU::InvalidRegBankID)
3600 return RB1;
3601 if (RB1 == AMDGPU::InvalidRegBankID)
3602 return RB0;
3603
3604 // vcc, vcc -> vcc
3605 // vcc, sgpr -> vcc
3606 // vcc, vgpr -> vcc
3607 if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID)
3608 return AMDGPU::VCCRegBankID;
3609
3610 // vcc, vgpr -> vgpr
3611 return regBankUnion(RB0, RB1);
3612}
3613
3614unsigned AMDGPURegisterBankInfo::getMappingType(const MachineRegisterInfo &MRI,
3615 const MachineInstr &MI) const {
3616 unsigned RegBank = AMDGPU::InvalidRegBankID;
3617
3618 for (const MachineOperand &MO : MI.operands()) {
3619 if (!MO.isReg())
3620 continue;
3621 Register Reg = MO.getReg();
3622 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
3623 RegBank = regBankUnion(RB0: RegBank, RB1: Bank->getID());
3624 if (RegBank == AMDGPU::VGPRRegBankID)
3625 break;
3626 }
3627 }
3628
3629 return RegBank;
3630}
3631
3632bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const {
3633 const MachineFunction &MF = *MI.getMF();
3634 const MachineRegisterInfo &MRI = MF.getRegInfo();
3635 for (const MachineOperand &MO : MI.operands()) {
3636 if (!MO.isReg())
3637 continue;
3638 Register Reg = MO.getReg();
3639 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
3640 if (Bank->getID() != AMDGPU::SGPRRegBankID)
3641 return false;
3642 }
3643 }
3644 return true;
3645}
3646
3647const RegisterBankInfo::InstructionMapping &
3648AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const {
3649 const MachineFunction &MF = *MI.getMF();
3650 const MachineRegisterInfo &MRI = MF.getRegInfo();
3651 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3652
3653 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3654 const MachineOperand &SrcOp = MI.getOperand(i);
3655 if (!SrcOp.isReg())
3656 continue;
3657
3658 unsigned Size = getSizeInBits(Reg: SrcOp.getReg(), MRI, TRI: *TRI);
3659 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
3660 }
3661 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3662 NumOperands: MI.getNumOperands());
3663}
3664
3665const RegisterBankInfo::InstructionMapping &
3666AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
3667 const MachineFunction &MF = *MI.getMF();
3668 const MachineRegisterInfo &MRI = MF.getRegInfo();
3669 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3670
3671 // Even though we technically could use SGPRs, this would require knowledge of
3672 // the constant bus restriction. Force all sources to VGPR (except for VCC).
3673 //
3674 // TODO: Unary ops are trivially OK, so accept SGPRs?
3675 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3676 const MachineOperand &Src = MI.getOperand(i);
3677 if (!Src.isReg())
3678 continue;
3679
3680 unsigned Size = getSizeInBits(Reg: Src.getReg(), MRI, TRI: *TRI);
3681 unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
3682 OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
3683 }
3684
3685 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3686 NumOperands: MI.getNumOperands());
3687}
3688
3689const RegisterBankInfo::InstructionMapping &
3690AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
3691 const MachineFunction &MF = *MI.getMF();
3692 const MachineRegisterInfo &MRI = MF.getRegInfo();
3693 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3694
3695 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
3696 const MachineOperand &Op = MI.getOperand(i: I);
3697 if (!Op.isReg())
3698 continue;
3699
3700 unsigned Size = getSizeInBits(Reg: Op.getReg(), MRI, TRI: *TRI);
3701 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3702 }
3703
3704 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping),
3705 NumOperands: MI.getNumOperands());
3706}
3707
3708const RegisterBankInfo::InstructionMapping &
3709AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
3710 const MachineInstr &MI,
3711 int RsrcIdx) const {
3712 // The reported argument index is relative to the IR intrinsic call arguments,
3713 // so we need to shift by the number of defs and the intrinsic ID.
3714 RsrcIdx += MI.getNumExplicitDefs() + 1;
3715
3716 const int NumOps = MI.getNumOperands();
3717 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps);
3718
3719 // TODO: Should packed/unpacked D16 difference be reported here as part of
3720 // the value mapping?
3721 for (int I = 0; I != NumOps; ++I) {
3722 if (!MI.getOperand(i: I).isReg())
3723 continue;
3724
3725 Register OpReg = MI.getOperand(i: I).getReg();
3726 // We replace some dead address operands with $noreg
3727 if (!OpReg)
3728 continue;
3729
3730 unsigned Size = getSizeInBits(Reg: OpReg, MRI, TRI: *TRI);
3731
3732 // FIXME: Probably need a new intrinsic register bank searchable table to
3733 // handle arbitrary intrinsics easily.
3734 //
3735 // If this has a sampler, it immediately follows rsrc.
3736 const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1;
3737
3738 if (MustBeSGPR) {
3739 // If this must be an SGPR, so we must report whatever it is as legal.
3740 unsigned NewBank = getRegBankID(Reg: OpReg, MRI, Default: AMDGPU::SGPRRegBankID);
3741 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: NewBank, Size);
3742 } else {
3743 // Some operands must be VGPR, and these are easy to copy to.
3744 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3745 }
3746 }
3747
3748 return getInstructionMapping(ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping), NumOperands: NumOps);
3749}
3750
3751/// Return the mapping for a pointer argument.
3752const RegisterBankInfo::ValueMapping *
3753AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI,
3754 Register PtrReg) const {
3755 LLT PtrTy = MRI.getType(Reg: PtrReg);
3756 unsigned Size = PtrTy.getSizeInBits();
3757 if (Subtarget.useFlatForGlobal() ||
3758 !AMDGPU::isFlatGlobalAddrSpace(AS: PtrTy.getAddressSpace()))
3759 return AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3760
3761 // If we're using MUBUF instructions for global memory, an SGPR base register
3762 // is possible. Otherwise this needs to be a VGPR.
3763 const RegisterBank *PtrBank = getRegBank(Reg: PtrReg, MRI, TRI: *TRI);
3764 return AMDGPU::getValueMapping(BankID: PtrBank->getID(), Size);
3765}
3766
3767const RegisterBankInfo::InstructionMapping &
3768AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
3769
3770 const MachineFunction &MF = *MI.getMF();
3771 const MachineRegisterInfo &MRI = MF.getRegInfo();
3772 SmallVector<const ValueMapping*, 2> OpdsMapping(2);
3773 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3774 Register PtrReg = MI.getOperand(i: 1).getReg();
3775 LLT PtrTy = MRI.getType(Reg: PtrReg);
3776 unsigned AS = PtrTy.getAddressSpace();
3777 unsigned PtrSize = PtrTy.getSizeInBits();
3778
3779 const ValueMapping *ValMapping;
3780 const ValueMapping *PtrMapping;
3781
3782 const RegisterBank *PtrBank = getRegBank(Reg: PtrReg, MRI, TRI: *TRI);
3783
3784 if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) {
3785 if (isScalarLoadLegal(MI)) {
3786 // We have a uniform instruction so we want to use an SMRD load
3787 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
3788 PtrMapping = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: PtrSize);
3789 } else {
3790 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3791
3792 // If we're using MUBUF instructions for global memory, an SGPR base
3793 // register is possible. Otherwise this needs to be a VGPR.
3794 unsigned PtrBankID = Subtarget.useFlatForGlobal() ?
3795 AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID;
3796
3797 PtrMapping = AMDGPU::getValueMapping(BankID: PtrBankID, Size: PtrSize);
3798 }
3799 } else {
3800 ValMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3801 PtrMapping = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize);
3802 }
3803
3804 OpdsMapping[0] = ValMapping;
3805 OpdsMapping[1] = PtrMapping;
3806 const RegisterBankInfo::InstructionMapping &Mapping = getInstructionMapping(
3807 ID: 1, Cost: 1, OperandsMapping: getOperandsMapping(OpdsMapping), NumOperands: MI.getNumOperands());
3808 return Mapping;
3809
3810 // FIXME: Do we want to add a mapping for FLAT load, or should we just
3811 // handle that during instruction selection?
3812}
3813
3814unsigned
3815AMDGPURegisterBankInfo::getRegBankID(Register Reg,
3816 const MachineRegisterInfo &MRI,
3817 unsigned Default) const {
3818 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
3819 return Bank ? Bank->getID() : Default;
3820}
3821
3822const RegisterBankInfo::ValueMapping *
3823AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
3824 const MachineRegisterInfo &MRI,
3825 const TargetRegisterInfo &TRI) const {
3826 // Lie and claim anything is legal, even though this needs to be an SGPR
3827 // applyMapping will have to deal with it as a waterfall loop.
3828 unsigned Bank = getRegBankID(Reg, MRI, Default: AMDGPU::SGPRRegBankID);
3829 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3830 return AMDGPU::getValueMapping(BankID: Bank, Size);
3831}
3832
3833const RegisterBankInfo::ValueMapping *
3834AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
3835 const MachineRegisterInfo &MRI,
3836 const TargetRegisterInfo &TRI) const {
3837 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3838 return AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
3839}
3840
3841const RegisterBankInfo::ValueMapping *
3842AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg,
3843 const MachineRegisterInfo &MRI,
3844 const TargetRegisterInfo &TRI) const {
3845 unsigned Size = getSizeInBits(Reg, MRI, TRI);
3846 return AMDGPU::getValueMapping(BankID: AMDGPU::AGPRRegBankID, Size);
3847}
3848
3849///
3850/// This function must return a legal mapping, because
3851/// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called
3852/// in RegBankSelect::Mode::Fast. Any mapping that would cause a
3853/// VGPR to SGPR generated is illegal.
3854///
3855// Operands that must be SGPRs must accept potentially divergent VGPRs as
3856// legal. These will be dealt with in applyMappingImpl.
3857//
3858const RegisterBankInfo::InstructionMapping &
3859AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
3860 const MachineFunction &MF = *MI.getMF();
3861 const MachineRegisterInfo &MRI = MF.getRegInfo();
3862
3863 if (MI.isCopy() || MI.getOpcode() == AMDGPU::G_FREEZE) {
3864 Register DstReg = MI.getOperand(i: 0).getReg();
3865 Register SrcReg = MI.getOperand(i: 1).getReg();
3866
3867 // The default logic bothers to analyze impossible alternative mappings. We
3868 // want the most straightforward mapping, so just directly handle this.
3869 const RegisterBank *DstBank = getRegBank(Reg: DstReg, MRI, TRI: *TRI);
3870 const RegisterBank *SrcBank = getRegBank(Reg: SrcReg, MRI, TRI: *TRI);
3871
3872 // For COPY between a physical reg and an s1, there is no type associated so
3873 // we need to take the virtual register's type as a hint on how to interpret
3874 // s1 values.
3875 unsigned Size;
3876 if (!SrcReg.isVirtual() && !DstBank &&
3877 MRI.getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1)) {
3878 DstBank = &AMDGPU::VCCRegBank;
3879 Size = 1;
3880 } else if (!DstReg.isVirtual() && MRI.getType(Reg: SrcReg) == LLT::scalar(SizeInBits: 1)) {
3881 DstBank = &AMDGPU::VCCRegBank;
3882 Size = 1;
3883 } else {
3884 Size = getSizeInBits(Reg: DstReg, MRI, TRI: *TRI);
3885 }
3886
3887 if (!DstBank)
3888 DstBank = SrcBank;
3889 else if (!SrcBank)
3890 SrcBank = DstBank;
3891
3892 if (MI.getOpcode() != AMDGPU::G_FREEZE &&
3893 cannotCopy(Dst: *DstBank, Src: *SrcBank, Size: TypeSize::getFixed(ExactSize: Size)))
3894 return getInvalidInstructionMapping();
3895
3896 const ValueMapping &ValMap = getValueMapping(StartIdx: 0, Length: Size, RegBank: *DstBank);
3897 unsigned OpdsMappingSize = MI.isCopy() ? 1 : 2;
3898 SmallVector<const ValueMapping *, 1> OpdsMapping(OpdsMappingSize);
3899 OpdsMapping[0] = &ValMap;
3900 if (MI.getOpcode() == AMDGPU::G_FREEZE)
3901 OpdsMapping[1] = &ValMap;
3902
3903 return getInstructionMapping(
3904 ID: 1, /*Cost*/ 1,
3905 /*OperandsMapping*/ getOperandsMapping(OpdsMapping), NumOperands: OpdsMappingSize);
3906 }
3907
3908 if (MI.isRegSequence()) {
3909 // If any input is a VGPR, the result must be a VGPR. The default handling
3910 // assumes any copy between banks is legal.
3911 unsigned BankID = AMDGPU::SGPRRegBankID;
3912
3913 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3914 auto OpBank = getRegBankID(Reg: MI.getOperand(i: I).getReg(), MRI);
3915 // It doesn't make sense to use vcc or scc banks here, so just ignore
3916 // them.
3917 if (OpBank != AMDGPU::SGPRRegBankID) {
3918 BankID = AMDGPU::VGPRRegBankID;
3919 break;
3920 }
3921 }
3922 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3923
3924 const ValueMapping &ValMap = getValueMapping(StartIdx: 0, Length: Size, RegBank: getRegBank(ID: BankID));
3925 return getInstructionMapping(
3926 ID: 1, /*Cost*/ 1,
3927 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: {&ValMap}), NumOperands: 1);
3928 }
3929
3930 // The default handling is broken and doesn't handle illegal SGPR->VGPR copies
3931 // properly.
3932 //
3933 // TODO: There are additional exec masking dependencies to analyze.
3934 if (auto *PHI = dyn_cast<GPhi>(Val: &MI)) {
3935 unsigned ResultBank = AMDGPU::InvalidRegBankID;
3936 Register DstReg = PHI->getReg(Idx: 0);
3937
3938 // Sometimes the result may have already been assigned a bank.
3939 if (const RegisterBank *DstBank = getRegBank(Reg: DstReg, MRI, TRI: *TRI))
3940 ResultBank = DstBank->getID();
3941
3942 for (unsigned I = 0; I < PHI->getNumIncomingValues(); ++I) {
3943 Register Reg = PHI->getIncomingValue(I);
3944 const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI);
3945
3946 // FIXME: Assuming VGPR for any undetermined inputs.
3947 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) {
3948 ResultBank = AMDGPU::VGPRRegBankID;
3949 break;
3950 }
3951
3952 // FIXME: Need to promote SGPR case to s32
3953 unsigned OpBank = Bank->getID();
3954 ResultBank = regBankBoolUnion(RB0: ResultBank, RB1: OpBank);
3955 }
3956
3957 assert(ResultBank != AMDGPU::InvalidRegBankID);
3958
3959 unsigned Size = MRI.getType(Reg: DstReg).getSizeInBits();
3960
3961 const ValueMapping &ValMap =
3962 getValueMapping(StartIdx: 0, Length: Size, RegBank: getRegBank(ID: ResultBank));
3963 return getInstructionMapping(
3964 ID: 1, /*Cost*/ 1,
3965 /*OperandsMapping*/ getOperandsMapping(OpdsMapping: {&ValMap}), NumOperands: 1);
3966 }
3967
3968 const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI);
3969 if (Mapping.isValid())
3970 return Mapping;
3971
3972 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
3973
3974 switch (MI.getOpcode()) {
3975 default:
3976 return getInvalidInstructionMapping();
3977
3978 case AMDGPU::G_AND:
3979 case AMDGPU::G_OR:
3980 case AMDGPU::G_XOR:
3981 case AMDGPU::G_MUL: {
3982 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
3983 if (Size == 1) {
3984 const RegisterBank *DstBank
3985 = getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
3986
3987 unsigned TargetBankID = AMDGPU::InvalidRegBankID;
3988 unsigned BankLHS = AMDGPU::InvalidRegBankID;
3989 unsigned BankRHS = AMDGPU::InvalidRegBankID;
3990 if (DstBank) {
3991 TargetBankID = DstBank->getID();
3992 if (DstBank == &AMDGPU::VCCRegBank) {
3993 TargetBankID = AMDGPU::VCCRegBankID;
3994 BankLHS = AMDGPU::VCCRegBankID;
3995 BankRHS = AMDGPU::VCCRegBankID;
3996 } else {
3997 BankLHS = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
3998 Default: AMDGPU::SGPRRegBankID);
3999 BankRHS = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
4000 Default: AMDGPU::SGPRRegBankID);
4001 }
4002 } else {
4003 BankLHS = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
4004 Default: AMDGPU::VCCRegBankID);
4005 BankRHS = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
4006 Default: AMDGPU::VCCRegBankID);
4007
4008 // Both inputs should be true booleans to produce a boolean result.
4009 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) {
4010 TargetBankID = AMDGPU::VGPRRegBankID;
4011 } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) {
4012 TargetBankID = AMDGPU::VCCRegBankID;
4013 BankLHS = AMDGPU::VCCRegBankID;
4014 BankRHS = AMDGPU::VCCRegBankID;
4015 } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) {
4016 TargetBankID = AMDGPU::SGPRRegBankID;
4017 }
4018 }
4019
4020 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: TargetBankID, Size);
4021 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: BankLHS, Size);
4022 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: BankRHS, Size);
4023 break;
4024 }
4025
4026 if (Size == 64) {
4027
4028 if (isSALUMapping(MI)) {
4029 OpdsMapping[0] = getValueMappingSGPR64Only(BankID: AMDGPU::SGPRRegBankID, Size);
4030 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
4031 } else {
4032 if (MI.getOpcode() == AMDGPU::G_MUL && Subtarget.useVMulU64Inst())
4033 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4034 else
4035 OpdsMapping[0] =
4036 getValueMappingSGPR64Only(BankID: AMDGPU::VGPRRegBankID, Size);
4037 unsigned Bank1 = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI /*, DefaultBankID*/);
4038 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank1, Size);
4039
4040 unsigned Bank2 = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI /*, DefaultBankID*/);
4041 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank2, Size);
4042 }
4043
4044 break;
4045 }
4046
4047 [[fallthrough]];
4048 }
4049 case AMDGPU::G_PTR_ADD:
4050 case AMDGPU::G_PTRMASK:
4051 case AMDGPU::G_ADD:
4052 case AMDGPU::G_SUB:
4053 case AMDGPU::G_SHL:
4054 case AMDGPU::G_LSHR:
4055 case AMDGPU::G_ASHR:
4056 case AMDGPU::G_UADDO:
4057 case AMDGPU::G_USUBO:
4058 case AMDGPU::G_UADDE:
4059 case AMDGPU::G_SADDE:
4060 case AMDGPU::G_USUBE:
4061 case AMDGPU::G_SSUBE:
4062 case AMDGPU::G_ABS:
4063 case AMDGPU::G_SHUFFLE_VECTOR:
4064 case AMDGPU::G_SBFX:
4065 case AMDGPU::G_UBFX:
4066 case AMDGPU::G_AMDGPU_S_MUL_I64_I32:
4067 case AMDGPU::G_AMDGPU_S_MUL_U64_U32:
4068 if (isSALUMapping(MI)) {
4069 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4070 unsigned Size = Ty.getSizeInBits();
4071 // Packed add and sub are VALU only.
4072 if (Subtarget.hasAnyPackedU64Ops() && Ty.isVector() && Size == 128)
4073 return getDefaultMappingVOP(MI);
4074 return getDefaultMappingSOP(MI);
4075 }
4076 return getDefaultMappingVOP(MI);
4077 case AMDGPU::G_SMIN:
4078 case AMDGPU::G_SMAX:
4079 case AMDGPU::G_UMIN:
4080 case AMDGPU::G_UMAX:
4081 if (isSALUMapping(MI)) {
4082 // There are no scalar 64-bit min and max, use vector instruction instead.
4083 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits() == 64 &&
4084 Subtarget.useMinMaxI64Insts())
4085 return getDefaultMappingVOP(MI);
4086 return getDefaultMappingSOP(MI);
4087 }
4088 return getDefaultMappingVOP(MI);
4089 case AMDGPU::G_FADD:
4090 case AMDGPU::G_FSUB:
4091 case AMDGPU::G_FMUL:
4092 case AMDGPU::G_FMA:
4093 case AMDGPU::G_FFLOOR:
4094 case AMDGPU::G_FCEIL:
4095 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
4096 case AMDGPU::G_FMINNUM:
4097 case AMDGPU::G_FMAXNUM:
4098 case AMDGPU::G_FMINIMUMNUM:
4099 case AMDGPU::G_FMAXIMUMNUM:
4100 case AMDGPU::G_INTRINSIC_TRUNC:
4101 case AMDGPU::G_STRICT_FADD:
4102 case AMDGPU::G_STRICT_FSUB:
4103 case AMDGPU::G_STRICT_FMUL:
4104 case AMDGPU::G_STRICT_FMA: {
4105 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4106 unsigned Size = Ty.getSizeInBits();
4107 if (Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
4108 (Size == 32 || Size == 16) && isSALUMapping(MI))
4109 return getDefaultMappingSOP(MI);
4110 return getDefaultMappingVOP(MI);
4111 }
4112 case AMDGPU::G_FMINIMUM:
4113 case AMDGPU::G_FMAXIMUM: {
4114 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4115 unsigned Size = Ty.getSizeInBits();
4116 if (Subtarget.hasSALUMinimumMaximumInsts() && Ty.isScalar() &&
4117 (Size == 32 || Size == 16) && isSALUMapping(MI))
4118 return getDefaultMappingSOP(MI);
4119 return getDefaultMappingVOP(MI);
4120 }
4121 case AMDGPU::G_FPTOSI:
4122 case AMDGPU::G_FPTOUI:
4123 case AMDGPU::G_FPTOSI_SAT:
4124 case AMDGPU::G_FPTOUI_SAT:
4125 case AMDGPU::G_SITOFP:
4126 case AMDGPU::G_UITOFP: {
4127 unsigned SizeDst = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4128 unsigned SizeSrc = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4129 if (Subtarget.hasSALUFloatInsts() && SizeDst == 32 && SizeSrc == 32 &&
4130 isSALUMapping(MI))
4131 return getDefaultMappingSOP(MI);
4132 return getDefaultMappingVOP(MI);
4133 }
4134 case AMDGPU::G_FPTRUNC:
4135 case AMDGPU::G_FPEXT: {
4136 unsigned SizeDst = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4137 unsigned SizeSrc = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4138 if (Subtarget.hasSALUFloatInsts() && SizeDst != 64 && SizeSrc != 64 &&
4139 isSALUMapping(MI))
4140 return getDefaultMappingSOP(MI);
4141 return getDefaultMappingVOP(MI);
4142 }
4143 case AMDGPU::G_FSQRT:
4144 case AMDGPU::G_FEXP2:
4145 case AMDGPU::G_FLOG2: {
4146 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4147 if (Subtarget.hasPseudoScalarTrans() && (Size == 16 || Size == 32) &&
4148 isSALUMapping(MI))
4149 return getDefaultMappingSOP(MI);
4150 return getDefaultMappingVOP(MI);
4151 }
4152 case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU
4153 case AMDGPU::G_SSUBSAT:
4154 case AMDGPU::G_UADDSAT:
4155 case AMDGPU::G_USUBSAT:
4156 case AMDGPU::G_FMAD:
4157 case AMDGPU::G_FLDEXP:
4158 case AMDGPU::G_FMINNUM_IEEE:
4159 case AMDGPU::G_FMAXNUM_IEEE:
4160 case AMDGPU::G_FCANONICALIZE:
4161 case AMDGPU::G_STRICT_FLDEXP:
4162 case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar?
4163 case AMDGPU::G_FSHR: // TODO: Expand for scalar
4164 case AMDGPU::G_AMDGPU_FMIN_LEGACY:
4165 case AMDGPU::G_AMDGPU_FMAX_LEGACY:
4166 case AMDGPU::G_AMDGPU_RCP_IFLAG:
4167 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
4168 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
4169 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
4170 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
4171 case AMDGPU::G_AMDGPU_CVT_PK_I16_I32:
4172 case AMDGPU::G_AMDGPU_SMED3:
4173 case AMDGPU::G_AMDGPU_FMED3:
4174 return getDefaultMappingVOP(MI);
4175 case AMDGPU::G_UMULH:
4176 case AMDGPU::G_SMULH: {
4177 if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI))
4178 return getDefaultMappingSOP(MI);
4179 return getDefaultMappingVOP(MI);
4180 }
4181 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4182 case AMDGPU::G_AMDGPU_MAD_I64_I32: {
4183 // Three possible mappings:
4184 //
4185 // - Default SOP
4186 // - Default VOP
4187 // - Scalar multiply: src0 and src1 are SGPRs, the rest is VOP.
4188 //
4189 // This allows instruction selection to keep the multiplication part of the
4190 // instruction on the SALU.
4191 bool AllSalu = true;
4192 bool MulSalu = true;
4193 for (unsigned i = 0; i < 5; ++i) {
4194 Register Reg = MI.getOperand(i).getReg();
4195 if (const RegisterBank *Bank = getRegBank(Reg, MRI, TRI: *TRI)) {
4196 if (Bank->getID() != AMDGPU::SGPRRegBankID) {
4197 AllSalu = false;
4198 if (i == 2 || i == 3) {
4199 MulSalu = false;
4200 break;
4201 }
4202 }
4203 }
4204 }
4205
4206 if (AllSalu)
4207 return getDefaultMappingSOP(MI);
4208
4209 // If the multiply-add is full-rate in VALU, use that even if the
4210 // multiplication part is scalar. Accumulating separately on the VALU would
4211 // take two instructions.
4212 if (!MulSalu || Subtarget.hasFullRate64Ops())
4213 return getDefaultMappingVOP(MI);
4214
4215 // Keep the multiplication on the SALU, then accumulate on the VALU.
4216 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 64);
4217 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4218 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4219 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4220 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 64);
4221 break;
4222 }
4223 case AMDGPU::G_IMPLICIT_DEF: {
4224 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4225 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4226 break;
4227 }
4228 case AMDGPU::G_FCONSTANT:
4229 case AMDGPU::G_CONSTANT:
4230 case AMDGPU::G_GLOBAL_VALUE:
4231 case AMDGPU::G_FRAME_INDEX:
4232 case AMDGPU::G_BLOCK_ADDR:
4233 case AMDGPU::G_READSTEADYCOUNTER:
4234 case AMDGPU::G_READCYCLECOUNTER: {
4235 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4236 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4237 break;
4238 }
4239 case AMDGPU::G_DYN_STACKALLOC: {
4240 // Result is always uniform, and a wave reduction is needed for the source.
4241 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4242 unsigned SrcBankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4243 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: SrcBankID, Size: 32);
4244 break;
4245 }
4246 case AMDGPU::G_AMDGPU_WAVE_ADDRESS: {
4247 // This case is weird because we expect a physical register in the source,
4248 // but need to set a bank anyway.
4249 //
4250 // TODO: We could select the result to SGPR or VGPR
4251 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4252 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
4253 break;
4254 }
4255 case AMDGPU::G_INSERT: {
4256 unsigned BankID = getMappingType(MRI, MI);
4257 unsigned DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4258 unsigned SrcSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4259 unsigned EltSize = getSizeInBits(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4260 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: DstSize);
4261 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size: SrcSize);
4262 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, Size: EltSize);
4263 OpdsMapping[3] = nullptr;
4264 break;
4265 }
4266 case AMDGPU::G_EXTRACT: {
4267 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4268 unsigned DstSize = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4269 unsigned SrcSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4270 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: DstSize);
4271 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size: SrcSize);
4272 OpdsMapping[2] = nullptr;
4273 break;
4274 }
4275 case AMDGPU::G_BUILD_VECTOR:
4276 case AMDGPU::G_BUILD_VECTOR_TRUNC: {
4277 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4278 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16)) {
4279 unsigned DstSize = DstTy.getSizeInBits();
4280 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4281 unsigned Src0BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4282 unsigned Src1BankID = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4283 unsigned DstBankID = regBankUnion(RB0: Src0BankID, RB1: Src1BankID);
4284
4285 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: DstBankID, Size: DstSize);
4286 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Src0BankID, Size: SrcSize);
4287 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Src1BankID, Size: SrcSize);
4288 break;
4289 }
4290
4291 [[fallthrough]];
4292 }
4293 case AMDGPU::G_MERGE_VALUES:
4294 case AMDGPU::G_CONCAT_VECTORS: {
4295 unsigned Bank = getMappingType(MRI, MI);
4296 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4297 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4298
4299 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: DstSize);
4300 // Op1 and Dst should use the same register bank.
4301 for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i)
4302 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: Bank, Size: SrcSize);
4303 break;
4304 }
4305 case AMDGPU::G_BITREVERSE:
4306 case AMDGPU::G_BITCAST:
4307 case AMDGPU::G_INTTOPTR:
4308 case AMDGPU::G_PTRTOINT:
4309 case AMDGPU::G_FABS:
4310 case AMDGPU::G_FNEG: {
4311 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4312 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4313 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
4314 break;
4315 }
4316 case AMDGPU::G_AMDGPU_FFBH_U32:
4317 case AMDGPU::G_AMDGPU_FFBL_B32:
4318 case AMDGPU::G_CTLZ_ZERO_POISON:
4319 case AMDGPU::G_CTTZ_ZERO_POISON: {
4320 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4321 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4322 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: 32);
4323 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID, Size);
4324 break;
4325 }
4326 case AMDGPU::G_CTPOP: {
4327 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4328 unsigned BankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4329 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, Size: 32);
4330
4331 // This should really be getValueMappingSGPR64Only, but allowing the generic
4332 // code to handle the register split just makes using LegalizerHelper more
4333 // difficult.
4334 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size);
4335 break;
4336 }
4337 case AMDGPU::G_TRUNC: {
4338 Register Dst = MI.getOperand(i: 0).getReg();
4339 Register Src = MI.getOperand(i: 1).getReg();
4340 unsigned Bank = getRegBankID(Reg: Src, MRI);
4341 unsigned DstSize = getSizeInBits(Reg: Dst, MRI, TRI: *TRI);
4342 unsigned SrcSize = getSizeInBits(Reg: Src, MRI, TRI: *TRI);
4343 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: DstSize);
4344 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: SrcSize);
4345 break;
4346 }
4347 case AMDGPU::G_ZEXT:
4348 case AMDGPU::G_SEXT:
4349 case AMDGPU::G_ANYEXT:
4350 case AMDGPU::G_SEXT_INREG: {
4351 Register Dst = MI.getOperand(i: 0).getReg();
4352 Register Src = MI.getOperand(i: 1).getReg();
4353 unsigned DstSize = getSizeInBits(Reg: Dst, MRI, TRI: *TRI);
4354 unsigned SrcSize = getSizeInBits(Reg: Src, MRI, TRI: *TRI);
4355
4356 unsigned DstBank;
4357 const RegisterBank *SrcBank = getRegBank(Reg: Src, MRI, TRI: *TRI);
4358 assert(SrcBank);
4359 switch (SrcBank->getID()) {
4360 case AMDGPU::SGPRRegBankID:
4361 DstBank = AMDGPU::SGPRRegBankID;
4362 break;
4363 default:
4364 DstBank = AMDGPU::VGPRRegBankID;
4365 break;
4366 }
4367
4368 // Scalar extend can use 64-bit BFE, but VGPRs require extending to
4369 // 32-bits, and then to 64.
4370 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: DstBank, Size: DstSize);
4371 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(BankID: SrcBank->getID(),
4372 Size: SrcSize);
4373 break;
4374 }
4375 case AMDGPU::G_IS_FPCLASS: {
4376 Register SrcReg = MI.getOperand(i: 1).getReg();
4377 unsigned SrcSize = MRI.getType(Reg: SrcReg).getSizeInBits();
4378 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4379 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: DstSize);
4380 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4381 break;
4382 }
4383 case AMDGPU::G_STORE: {
4384 assert(MI.getOperand(0).isReg());
4385 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4386
4387 // FIXME: We need to specify a different reg bank once scalar stores are
4388 // supported.
4389 const ValueMapping *ValMapping =
4390 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
4391 OpdsMapping[0] = ValMapping;
4392 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
4393 break;
4394 }
4395 case AMDGPU::G_ICMP:
4396 case AMDGPU::G_FCMP: {
4397 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4398
4399 // See if the result register has already been constrained to vcc, which may
4400 // happen due to control flow intrinsic lowering.
4401 unsigned DstBank = getRegBankID(Reg: MI.getOperand(i: 0).getReg(), MRI,
4402 Default: AMDGPU::SGPRRegBankID);
4403 unsigned Op2Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4404 unsigned Op3Bank = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI);
4405
4406 auto canUseSCCICMP = [&]() {
4407 auto Pred =
4408 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
4409 return Size == 32 ||
4410 (Size == 64 &&
4411 (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) &&
4412 Subtarget.hasScalarCompareEq64());
4413 };
4414 auto canUseSCCFCMP = [&]() {
4415 return Subtarget.hasSALUFloatInsts() && (Size == 32 || Size == 16);
4416 };
4417
4418 bool isICMP = MI.getOpcode() == AMDGPU::G_ICMP;
4419 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID &&
4420 Op2Bank == AMDGPU::SGPRRegBankID &&
4421 Op3Bank == AMDGPU::SGPRRegBankID &&
4422 (isICMP ? canUseSCCICMP() : canUseSCCFCMP());
4423
4424 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
4425 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4426
4427 // TODO: Use 32-bit for scalar output size.
4428 // SCC results will need to be copied to a 32-bit SGPR virtual register.
4429 const unsigned ResultSize = 1;
4430
4431 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: DstBank, Size: ResultSize);
4432 OpdsMapping[1] = nullptr; // Predicate Operand.
4433 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: SrcBank, Size);
4434 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: SrcBank, Size);
4435 break;
4436 }
4437 case AMDGPU::G_EXTRACT_VECTOR_ELT: {
4438 // VGPR index can be used for waterfall when indexing a SGPR vector.
4439 unsigned SrcBankID = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI);
4440 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4441 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4442 unsigned IdxSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4443 unsigned IdxBank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4444 unsigned OutputBankID = regBankUnion(RB0: SrcBankID, RB1: IdxBank);
4445
4446 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: OutputBankID, Size: DstSize);
4447 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: SrcBankID, Size: SrcSize);
4448
4449 // The index can be either if the source vector is VGPR.
4450 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4451 break;
4452 }
4453 case AMDGPU::G_INSERT_VECTOR_ELT: {
4454 unsigned OutputBankID = isSALUMapping(MI) ?
4455 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
4456
4457 unsigned VecSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4458 unsigned InsertSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4459 unsigned IdxSize = MRI.getType(Reg: MI.getOperand(i: 3).getReg()).getSizeInBits();
4460 unsigned InsertEltBankID = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI);
4461 unsigned IdxBankID = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI);
4462
4463 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: OutputBankID, Size: VecSize);
4464 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: OutputBankID, Size: VecSize);
4465
4466 // This is a weird case, because we need to break down the mapping based on
4467 // the register bank of a different operand.
4468 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) {
4469 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(BankID: InsertEltBankID,
4470 Size: InsertSize);
4471 } else {
4472 assert(InsertSize == 32 || InsertSize == 64);
4473 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: InsertEltBankID, Size: InsertSize);
4474 }
4475
4476 // The index can be either if the source vector is VGPR.
4477 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBankID, Size: IdxSize);
4478 break;
4479 }
4480 case AMDGPU::G_UNMERGE_VALUES: {
4481 unsigned Bank = getMappingType(MRI, MI);
4482
4483 // Op1 and Dst should use the same register bank.
4484 // FIXME: Shouldn't this be the default? Why do we need to handle this?
4485 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
4486 unsigned Size = getSizeInBits(Reg: MI.getOperand(i).getReg(), MRI, TRI: *TRI);
4487 OpdsMapping[i] = AMDGPU::getValueMapping(BankID: Bank, Size);
4488 }
4489 break;
4490 }
4491 case AMDGPU::G_AMDGPU_BUFFER_LOAD:
4492 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
4493 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
4494 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
4495 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
4496 case AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE:
4497 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE:
4498 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE_TFE:
4499 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE:
4500 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT_TFE:
4501 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT:
4502 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE:
4503 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16:
4504 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE:
4505 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT:
4506 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16:
4507 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT:
4508 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16:
4509 case AMDGPU::G_AMDGPU_BUFFER_STORE:
4510 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE:
4511 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT:
4512 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT:
4513 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: {
4514 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4515
4516 // rsrc
4517 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4518
4519 // vindex
4520 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4521
4522 // voffset
4523 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4524
4525 // soffset
4526 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4527
4528 // Any remaining operands are immediates and were correctly null
4529 // initialized.
4530 break;
4531 }
4532 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP:
4533 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD:
4534 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB:
4535 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN:
4536 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN:
4537 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX:
4538 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX:
4539 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND:
4540 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR:
4541 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR:
4542 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC:
4543 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC:
4544 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32:
4545 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32:
4546 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD:
4547 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN:
4548 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX: {
4549 // vdata_out
4550 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4551
4552 // vdata_in
4553 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4554
4555 // rsrc
4556 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4557
4558 // vindex
4559 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4560
4561 // voffset
4562 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4563
4564 // soffset
4565 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
4566
4567 // Any remaining operands are immediates and were correctly null
4568 // initialized.
4569 break;
4570 }
4571 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: {
4572 // vdata_out
4573 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4574
4575 // vdata_in
4576 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4577
4578 // cmp
4579 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4580
4581 // rsrc
4582 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
4583
4584 // vindex
4585 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
4586
4587 // voffset
4588 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
4589
4590 // soffset
4591 OpdsMapping[6] = getSGPROpMapping(Reg: MI.getOperand(i: 6).getReg(), MRI, TRI: *TRI);
4592
4593 // Any remaining operands are immediates and were correctly null
4594 // initialized.
4595 break;
4596 }
4597 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD:
4598 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
4599 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
4600 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
4601 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT: {
4602 // Lie and claim everything is legal, even though some need to be
4603 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
4604 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
4605 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4606
4607 // We need to convert this to a MUBUF if either the resource of offset is
4608 // VGPR.
4609 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID();
4610 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID();
4611 unsigned ResultBank = regBankUnion(RB0: RSrcBank, RB1: OffsetBank);
4612
4613 unsigned Size0 = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4614 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: ResultBank, Size: Size0);
4615 break;
4616 }
4617 case AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH:
4618 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
4619 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
4620 break;
4621 case AMDGPU::G_AMDGPU_SPONENTRY: {
4622 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4623 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4624 break;
4625 }
4626 case AMDGPU::G_INTRINSIC:
4627 case AMDGPU::G_INTRINSIC_CONVERGENT: {
4628 switch (cast<GIntrinsic>(Val: MI).getIntrinsicID()) {
4629 default:
4630 return getInvalidInstructionMapping();
4631 case Intrinsic::amdgcn_div_fmas:
4632 case Intrinsic::amdgcn_div_fixup:
4633 case Intrinsic::amdgcn_trig_preop:
4634 case Intrinsic::amdgcn_sin:
4635 case Intrinsic::amdgcn_cos:
4636 case Intrinsic::amdgcn_log_clamp:
4637 case Intrinsic::amdgcn_rcp_legacy:
4638 case Intrinsic::amdgcn_rsq_legacy:
4639 case Intrinsic::amdgcn_rsq_clamp:
4640 case Intrinsic::amdgcn_tanh:
4641 case Intrinsic::amdgcn_fmul_legacy:
4642 case Intrinsic::amdgcn_fma_legacy:
4643 case Intrinsic::amdgcn_frexp_mant:
4644 case Intrinsic::amdgcn_frexp_exp:
4645 case Intrinsic::amdgcn_fract:
4646 case Intrinsic::amdgcn_cvt_pknorm_i16:
4647 case Intrinsic::amdgcn_cvt_pknorm_u16:
4648 case Intrinsic::amdgcn_cvt_pk_i16:
4649 case Intrinsic::amdgcn_cvt_pk_u16:
4650 case Intrinsic::amdgcn_cvt_sr_pk_f16_f32:
4651 case Intrinsic::amdgcn_cvt_sr_pk_bf16_f32:
4652 case Intrinsic::amdgcn_cvt_pk_f16_fp8:
4653 case Intrinsic::amdgcn_cvt_pk_f16_bf8:
4654 case Intrinsic::amdgcn_cvt_pk_fp8_f16:
4655 case Intrinsic::amdgcn_cvt_pk_bf8_f16:
4656 case Intrinsic::amdgcn_cvt_sr_fp8_f16:
4657 case Intrinsic::amdgcn_cvt_sr_bf8_f16:
4658 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
4659 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
4660 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
4661 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
4662 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
4663 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
4664 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
4665 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
4666 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
4667 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
4668 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
4669 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
4670 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
4671 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
4672 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
4673 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_bf16:
4674 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_bf16:
4675 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f16:
4676 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f16:
4677 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp8_f32:
4678 case Intrinsic::amdgcn_cvt_scalef32_pk8_bf8_f32:
4679 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f32:
4680 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_f16:
4681 case Intrinsic::amdgcn_cvt_scalef32_pk8_fp4_bf16:
4682 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f32:
4683 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f32:
4684 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_f16:
4685 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_f16:
4686 case Intrinsic::amdgcn_cvt_scalef32_pk16_fp6_bf16:
4687 case Intrinsic::amdgcn_cvt_scalef32_pk16_bf6_bf16:
4688 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_bf16:
4689 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_bf16:
4690 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f16:
4691 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f16:
4692 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp8_f32:
4693 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_bf8_f32:
4694 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f32:
4695 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_f16:
4696 case Intrinsic::amdgcn_cvt_scalef32_sr_pk8_fp4_bf16:
4697 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f32:
4698 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f32:
4699 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_f16:
4700 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_f16:
4701 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_fp6_bf16:
4702 case Intrinsic::amdgcn_cvt_scalef32_sr_pk16_bf6_bf16:
4703 case Intrinsic::amdgcn_sat_pk4_i4_i8:
4704 case Intrinsic::amdgcn_sat_pk4_u4_u8:
4705 case Intrinsic::amdgcn_fmed3:
4706 case Intrinsic::amdgcn_cubeid:
4707 case Intrinsic::amdgcn_cubema:
4708 case Intrinsic::amdgcn_cubesc:
4709 case Intrinsic::amdgcn_cubetc:
4710 case Intrinsic::amdgcn_sffbh:
4711 case Intrinsic::amdgcn_fmad_ftz:
4712 case Intrinsic::amdgcn_mbcnt_lo:
4713 case Intrinsic::amdgcn_mbcnt_hi:
4714 case Intrinsic::amdgcn_mul_u24:
4715 case Intrinsic::amdgcn_mul_i24:
4716 case Intrinsic::amdgcn_mulhi_u24:
4717 case Intrinsic::amdgcn_mulhi_i24:
4718 case Intrinsic::amdgcn_lerp:
4719 case Intrinsic::amdgcn_sad_u8:
4720 case Intrinsic::amdgcn_msad_u8:
4721 case Intrinsic::amdgcn_sad_hi_u8:
4722 case Intrinsic::amdgcn_sad_u16:
4723 case Intrinsic::amdgcn_qsad_pk_u16_u8:
4724 case Intrinsic::amdgcn_mqsad_pk_u16_u8:
4725 case Intrinsic::amdgcn_mqsad_u32_u8:
4726 case Intrinsic::amdgcn_cvt_pk_u8_f32:
4727 case Intrinsic::amdgcn_alignbyte:
4728 case Intrinsic::amdgcn_perm:
4729 case Intrinsic::amdgcn_prng_b32:
4730 case Intrinsic::amdgcn_fdot2:
4731 case Intrinsic::amdgcn_sdot2:
4732 case Intrinsic::amdgcn_udot2:
4733 case Intrinsic::amdgcn_sdot4:
4734 case Intrinsic::amdgcn_udot4:
4735 case Intrinsic::amdgcn_sdot8:
4736 case Intrinsic::amdgcn_udot8:
4737 case Intrinsic::amdgcn_fdot2_bf16_bf16:
4738 case Intrinsic::amdgcn_fdot2_f16_f16:
4739 case Intrinsic::amdgcn_fdot2_f32_bf16:
4740 case Intrinsic::amdgcn_fdot2c_f32_bf16:
4741 case Intrinsic::amdgcn_sudot4:
4742 case Intrinsic::amdgcn_sudot8:
4743 case Intrinsic::amdgcn_dot4_f32_fp8_bf8:
4744 case Intrinsic::amdgcn_dot4_f32_bf8_fp8:
4745 case Intrinsic::amdgcn_dot4_f32_fp8_fp8:
4746 case Intrinsic::amdgcn_dot4_f32_bf8_bf8:
4747 case Intrinsic::amdgcn_cvt_f32_fp8:
4748 case Intrinsic::amdgcn_cvt_f32_fp8_e5m3:
4749 case Intrinsic::amdgcn_cvt_f32_bf8:
4750 case Intrinsic::amdgcn_cvt_off_f32_i4:
4751 case Intrinsic::amdgcn_cvt_pk_f32_fp8:
4752 case Intrinsic::amdgcn_cvt_pk_f32_bf8:
4753 case Intrinsic::amdgcn_cvt_pk_fp8_f32:
4754 case Intrinsic::amdgcn_cvt_pk_fp8_f32_e5m3:
4755 case Intrinsic::amdgcn_cvt_pk_bf8_f32:
4756 case Intrinsic::amdgcn_cvt_sr_fp8_f32:
4757 case Intrinsic::amdgcn_cvt_sr_fp8_f32_e5m3:
4758 case Intrinsic::amdgcn_cvt_sr_bf8_f32:
4759 case Intrinsic::amdgcn_cvt_sr_bf16_f32:
4760 case Intrinsic::amdgcn_cvt_sr_f16_f32:
4761 case Intrinsic::amdgcn_cvt_f16_fp8:
4762 case Intrinsic::amdgcn_cvt_f16_bf8:
4763 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f16:
4764 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f16:
4765 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_bf16:
4766 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_bf16:
4767 case Intrinsic::amdgcn_cvt_scalef32_pk32_fp6_f32:
4768 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf6_f32:
4769 case Intrinsic::amdgcn_cvt_scalef32_f16_fp8:
4770 case Intrinsic::amdgcn_cvt_scalef32_f16_bf8:
4771 case Intrinsic::amdgcn_cvt_scalef32_f32_fp8:
4772 case Intrinsic::amdgcn_cvt_scalef32_f32_bf8:
4773 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f32:
4774 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f32:
4775 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp8:
4776 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_bf8:
4777 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_f16:
4778 case Intrinsic::amdgcn_cvt_scalef32_pk_fp8_bf16:
4779 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_f16:
4780 case Intrinsic::amdgcn_cvt_scalef32_pk_bf8_bf16:
4781 case Intrinsic::amdgcn_cvt_scalef32_pk_f32_fp4:
4782 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f32:
4783 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp4:
4784 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp4:
4785 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_fp6:
4786 case Intrinsic::amdgcn_cvt_scalef32_pk32_f32_bf6:
4787 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_bf6:
4788 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_bf6:
4789 case Intrinsic::amdgcn_cvt_scalef32_pk32_f16_fp6:
4790 case Intrinsic::amdgcn_cvt_scalef32_pk32_bf16_fp6:
4791 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_bf8:
4792 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_bf8:
4793 case Intrinsic::amdgcn_cvt_scalef32_pk_f16_fp8:
4794 case Intrinsic::amdgcn_cvt_scalef32_pk_bf16_fp8:
4795 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_f16:
4796 case Intrinsic::amdgcn_cvt_scalef32_pk_fp4_bf16:
4797 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f16:
4798 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_bf16:
4799 case Intrinsic::amdgcn_cvt_scalef32_sr_pk_fp4_f32:
4800 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_bf16:
4801 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f16:
4802 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_bf6_f32:
4803 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_bf16:
4804 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f16:
4805 case Intrinsic::amdgcn_cvt_scalef32_sr_pk32_fp6_f32:
4806 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_bf16:
4807 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f16:
4808 case Intrinsic::amdgcn_cvt_scalef32_sr_bf8_f32:
4809 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_bf16:
4810 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f16:
4811 case Intrinsic::amdgcn_cvt_scalef32_sr_fp8_f32:
4812 case Intrinsic::amdgcn_ashr_pk_i8_i32:
4813 case Intrinsic::amdgcn_ashr_pk_u8_i32:
4814 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_fp6_f32:
4815 case Intrinsic::amdgcn_cvt_scalef32_2xpk16_bf6_f32:
4816 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16:
4817 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16:
4818 case Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16_tied:
4819 case Intrinsic::amdgcn_wmma_f16_16x16x16_f16_tied:
4820 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf16:
4821 case Intrinsic::amdgcn_wmma_f32_16x16x16_f16:
4822 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu4:
4823 case Intrinsic::amdgcn_wmma_i32_16x16x16_iu8:
4824 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_fp8:
4825 case Intrinsic::amdgcn_wmma_f32_16x16x16_fp8_bf8:
4826 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_fp8:
4827 case Intrinsic::amdgcn_wmma_f32_16x16x16_bf8_bf8:
4828 case Intrinsic::amdgcn_wmma_i32_16x16x32_iu4:
4829 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
4830 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
4831 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
4832 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
4833 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
4834 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
4835 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4:
4836 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
4837 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
4838 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
4839 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8:
4840 case Intrinsic::amdgcn_wmma_f64_16x16x4_f64:
4841 case Intrinsic::amdgcn_wmma_f32_16x16x4_f32:
4842 case Intrinsic::amdgcn_wmma_f32_16x16x32_bf16:
4843 case Intrinsic::amdgcn_wmma_f32_16x16x32_f16:
4844 case Intrinsic::amdgcn_wmma_f16_16x16x32_f16:
4845 case Intrinsic::amdgcn_wmma_bf16_16x16x32_bf16:
4846 case Intrinsic::amdgcn_wmma_bf16f32_16x16x32_bf16:
4847 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_fp8:
4848 case Intrinsic::amdgcn_wmma_f32_16x16x64_fp8_bf8:
4849 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_fp8:
4850 case Intrinsic::amdgcn_wmma_f32_16x16x64_bf8_bf8:
4851 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_fp8:
4852 case Intrinsic::amdgcn_wmma_f16_16x16x64_fp8_bf8:
4853 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_fp8:
4854 case Intrinsic::amdgcn_wmma_f16_16x16x64_bf8_bf8:
4855 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_fp8:
4856 case Intrinsic::amdgcn_wmma_f16_16x16x128_fp8_bf8:
4857 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_fp8:
4858 case Intrinsic::amdgcn_wmma_f16_16x16x128_bf8_bf8:
4859 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_fp8:
4860 case Intrinsic::amdgcn_wmma_f32_16x16x128_fp8_bf8:
4861 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_fp8:
4862 case Intrinsic::amdgcn_wmma_f32_16x16x128_bf8_bf8:
4863 case Intrinsic::amdgcn_wmma_i32_16x16x64_iu8:
4864 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
4865 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
4866 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4:
4867 case Intrinsic::amdgcn_wmma_f32_32x16x128_f4:
4868 case Intrinsic::amdgcn_wmma_scale_f32_32x16x128_f4:
4869 case Intrinsic::amdgcn_wmma_scale16_f32_32x16x128_f4:
4870 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
4871 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
4872 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
4873 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
4874 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
4875 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
4876 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
4877 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
4878 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
4879 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
4880 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
4881 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
4882 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8:
4883 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
4884 case Intrinsic::amdgcn_perm_pk16_b4_u4:
4885 case Intrinsic::amdgcn_perm_pk16_b6_u4:
4886 case Intrinsic::amdgcn_perm_pk16_b8_u4:
4887 case Intrinsic::amdgcn_add_max_i32:
4888 case Intrinsic::amdgcn_add_max_u32:
4889 case Intrinsic::amdgcn_add_min_i32:
4890 case Intrinsic::amdgcn_add_min_u32:
4891 case Intrinsic::amdgcn_pk_add_max_i16:
4892 case Intrinsic::amdgcn_pk_add_max_u16:
4893 case Intrinsic::amdgcn_pk_add_min_i16:
4894 case Intrinsic::amdgcn_pk_add_min_u16:
4895 return getDefaultMappingVOP(MI);
4896 case Intrinsic::amdgcn_log:
4897 case Intrinsic::amdgcn_exp2:
4898 case Intrinsic::amdgcn_rcp:
4899 case Intrinsic::amdgcn_rsq:
4900 case Intrinsic::amdgcn_sqrt: {
4901 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
4902 unsigned Size = Ty.getSizeInBits();
4903 // There is no pseudo scalar transcendental instruction for bf16.
4904 if (Subtarget.hasPseudoScalarTrans() && !Ty.isBFloat16() &&
4905 (Size == 16 || Size == 32) && isSALUMapping(MI))
4906 return getDefaultMappingSOP(MI);
4907 return getDefaultMappingVOP(MI);
4908 }
4909 case Intrinsic::amdgcn_sbfe:
4910 case Intrinsic::amdgcn_ubfe:
4911 if (isSALUMapping(MI))
4912 return getDefaultMappingSOP(MI);
4913 return getDefaultMappingVOP(MI);
4914 case Intrinsic::amdgcn_ds_swizzle:
4915 case Intrinsic::amdgcn_ds_permute:
4916 case Intrinsic::amdgcn_ds_bpermute:
4917 case Intrinsic::amdgcn_update_dpp:
4918 case Intrinsic::amdgcn_mov_dpp8:
4919 case Intrinsic::amdgcn_mov_dpp:
4920 case Intrinsic::amdgcn_strict_wwm:
4921 case Intrinsic::amdgcn_wwm:
4922 case Intrinsic::amdgcn_strict_wqm:
4923 case Intrinsic::amdgcn_wqm:
4924 case Intrinsic::amdgcn_softwqm:
4925 case Intrinsic::amdgcn_set_inactive:
4926 case Intrinsic::amdgcn_set_inactive_chain_arg:
4927 case Intrinsic::amdgcn_permlane64:
4928 case Intrinsic::amdgcn_ds_bpermute_fi_b32:
4929 return getDefaultMappingAllVGPR(MI);
4930 case Intrinsic::amdgcn_cvt_pkrtz:
4931 if (Subtarget.hasSALUFloatInsts() && isSALUMapping(MI))
4932 return getDefaultMappingSOP(MI);
4933 return getDefaultMappingVOP(MI);
4934 case Intrinsic::amdgcn_kernarg_segment_ptr:
4935 case Intrinsic::amdgcn_s_getpc:
4936 case Intrinsic::amdgcn_groupstaticsize:
4937 case Intrinsic::amdgcn_reloc_constant:
4938 case Intrinsic::returnaddress: {
4939 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4940 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
4941 break;
4942 }
4943 case Intrinsic::amdgcn_wqm_vote: {
4944 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4945 OpdsMapping[0] = OpdsMapping[2]
4946 = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size);
4947 break;
4948 }
4949 case Intrinsic::amdgcn_ps_live: {
4950 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
4951 break;
4952 }
4953 case Intrinsic::amdgcn_div_scale: {
4954 unsigned Dst0Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4955 unsigned Dst1Size = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits();
4956 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Dst0Size);
4957 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: Dst1Size);
4958
4959 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 3).getReg()).getSizeInBits();
4960 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4961 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4962 break;
4963 }
4964 case Intrinsic::amdgcn_class: {
4965 Register Src0Reg = MI.getOperand(i: 2).getReg();
4966 Register Src1Reg = MI.getOperand(i: 3).getReg();
4967 unsigned Src0Size = MRI.getType(Reg: Src0Reg).getSizeInBits();
4968 unsigned Src1Size = MRI.getType(Reg: Src1Reg).getSizeInBits();
4969 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4970 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: DstSize);
4971 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Src0Size);
4972 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: Src1Size);
4973 break;
4974 }
4975 case Intrinsic::amdgcn_readlane: {
4976 // This must be an SGPR, but accept a VGPR.
4977 Register IdxReg = MI.getOperand(i: 3).getReg();
4978 unsigned IdxSize = MRI.getType(Reg: IdxReg).getSizeInBits();
4979 unsigned IdxBank = getRegBankID(Reg: IdxReg, MRI, Default: AMDGPU::SGPRRegBankID);
4980 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
4981 [[fallthrough]];
4982 }
4983 case Intrinsic::amdgcn_readfirstlane: {
4984 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4985 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
4986 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
4987 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
4988 break;
4989 }
4990 case Intrinsic::amdgcn_writelane: {
4991 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
4992 Register SrcReg = MI.getOperand(i: 2).getReg();
4993 unsigned SrcSize = MRI.getType(Reg: SrcReg).getSizeInBits();
4994 unsigned SrcBank = getRegBankID(Reg: SrcReg, MRI, Default: AMDGPU::SGPRRegBankID);
4995 Register IdxReg = MI.getOperand(i: 3).getReg();
4996 unsigned IdxSize = MRI.getType(Reg: IdxReg).getSizeInBits();
4997 unsigned IdxBank = getRegBankID(Reg: IdxReg, MRI, Default: AMDGPU::SGPRRegBankID);
4998 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
4999
5000 // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted
5001 // to legalize.
5002 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: SrcBank, Size: SrcSize);
5003 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: IdxBank, Size: IdxSize);
5004 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: SrcSize);
5005 break;
5006 }
5007 case Intrinsic::amdgcn_if_break: {
5008 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5009 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5010 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5011 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5012 break;
5013 }
5014 case Intrinsic::amdgcn_permlane16:
5015 case Intrinsic::amdgcn_permlanex16: {
5016 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5017 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5018 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5019 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5020 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5021 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5022 break;
5023 }
5024 case Intrinsic::amdgcn_permlane_bcast:
5025 case Intrinsic::amdgcn_permlane_up:
5026 case Intrinsic::amdgcn_permlane_down:
5027 case Intrinsic::amdgcn_permlane_xor: {
5028 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5029 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5030 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5031 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5032 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5033 break;
5034 }
5035 case Intrinsic::amdgcn_permlane_idx_gen: {
5036 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5037 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5038 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5039 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5040 break;
5041 }
5042 case Intrinsic::amdgcn_permlane16_var:
5043 case Intrinsic::amdgcn_permlanex16_var: {
5044 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5045 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5046 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5047 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5048 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5049 break;
5050 }
5051 case Intrinsic::amdgcn_mfma_f32_4x4x1f32:
5052 case Intrinsic::amdgcn_mfma_f32_4x4x4f16:
5053 case Intrinsic::amdgcn_mfma_i32_4x4x4i8:
5054 case Intrinsic::amdgcn_mfma_f32_4x4x2bf16:
5055 case Intrinsic::amdgcn_mfma_f32_16x16x1f32:
5056 case Intrinsic::amdgcn_mfma_f32_16x16x4f32:
5057 case Intrinsic::amdgcn_mfma_f32_16x16x4f16:
5058 case Intrinsic::amdgcn_mfma_f32_16x16x16f16:
5059 case Intrinsic::amdgcn_mfma_i32_16x16x4i8:
5060 case Intrinsic::amdgcn_mfma_i32_16x16x16i8:
5061 case Intrinsic::amdgcn_mfma_f32_16x16x2bf16:
5062 case Intrinsic::amdgcn_mfma_f32_16x16x8bf16:
5063 case Intrinsic::amdgcn_mfma_f32_32x32x1f32:
5064 case Intrinsic::amdgcn_mfma_f32_32x32x2f32:
5065 case Intrinsic::amdgcn_mfma_f32_32x32x4f16:
5066 case Intrinsic::amdgcn_mfma_f32_32x32x8f16:
5067 case Intrinsic::amdgcn_mfma_i32_32x32x4i8:
5068 case Intrinsic::amdgcn_mfma_i32_32x32x8i8:
5069 case Intrinsic::amdgcn_mfma_f32_32x32x2bf16:
5070 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16:
5071 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16_1k:
5072 case Intrinsic::amdgcn_mfma_f32_16x16x4bf16_1k:
5073 case Intrinsic::amdgcn_mfma_f32_4x4x4bf16_1k:
5074 case Intrinsic::amdgcn_mfma_f32_32x32x8bf16_1k:
5075 case Intrinsic::amdgcn_mfma_f32_16x16x16bf16_1k:
5076 case Intrinsic::amdgcn_mfma_f64_16x16x4f64:
5077 case Intrinsic::amdgcn_mfma_f64_4x4x4f64:
5078 case Intrinsic::amdgcn_mfma_i32_16x16x32_i8:
5079 case Intrinsic::amdgcn_mfma_i32_32x32x16_i8:
5080 case Intrinsic::amdgcn_mfma_f32_16x16x8_xf32:
5081 case Intrinsic::amdgcn_mfma_f32_32x32x4_xf32:
5082 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_bf8:
5083 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf8_fp8:
5084 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_bf8:
5085 case Intrinsic::amdgcn_mfma_f32_16x16x32_fp8_fp8:
5086 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_bf8:
5087 case Intrinsic::amdgcn_mfma_f32_32x32x16_bf8_fp8:
5088 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_bf8:
5089 case Intrinsic::amdgcn_mfma_f32_32x32x16_fp8_fp8:
5090 case Intrinsic::amdgcn_mfma_f32_16x16x32_f16:
5091 case Intrinsic::amdgcn_mfma_f32_32x32x16_f16:
5092 case Intrinsic::amdgcn_mfma_i32_16x16x64_i8:
5093 case Intrinsic::amdgcn_mfma_i32_32x32x32_i8:
5094 case Intrinsic::amdgcn_mfma_f32_16x16x32_bf16: {
5095 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5096 unsigned MinNumRegsRequired = DstSize / 32;
5097
5098 // Default for MAI intrinsics.
5099 // srcC can also be an immediate which can be folded later.
5100 // FIXME: Should we eventually add an alternative mapping with AGPR src
5101 // for srcA/srcB?
5102 //
5103 // vdst, srcA, srcB, srcC
5104 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5105
5106 bool UseAGPRForm = !Subtarget.hasGFX90AInsts() ||
5107 Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5108
5109 OpdsMapping[0] =
5110 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI)
5111 : getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5112 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5113 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5114 OpdsMapping[4] =
5115 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5116 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5117 break;
5118 }
5119 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
5120 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
5121 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5122 unsigned MinNumRegsRequired = DstSize / 32;
5123
5124 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5125 bool UseAGPRForm = Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5126
5127 OpdsMapping[0] =
5128 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI)
5129 : getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5130
5131 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5132 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5133 OpdsMapping[4] =
5134 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5135 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5136
5137 OpdsMapping[8] = getVGPROpMapping(Reg: MI.getOperand(i: 8).getReg(), MRI, TRI: *TRI);
5138 OpdsMapping[10] = getVGPROpMapping(Reg: MI.getOperand(i: 10).getReg(), MRI, TRI: *TRI);
5139 break;
5140 }
5141 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
5142 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
5143 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
5144 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
5145 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
5146 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
5147 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
5148 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
5149 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
5150 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
5151 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
5152 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
5153 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
5154 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
5155 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
5156 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
5157 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
5158 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
5159 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
5160 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
5161 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
5162 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
5163 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
5164 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
5165 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
5166 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
5167 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
5168 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8: {
5169 Register DstReg = MI.getOperand(i: 0).getReg();
5170 unsigned DstSize = MRI.getType(Reg: DstReg).getSizeInBits();
5171 unsigned MinNumRegsRequired = DstSize / 32;
5172 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5173 bool UseAGPRForm = Info->selectAGPRFormMFMA(NumRegs: MinNumRegsRequired);
5174
5175 // vdst, srcA, srcB, srcC, idx
5176 OpdsMapping[0] = UseAGPRForm ? getAGPROpMapping(Reg: DstReg, MRI, TRI: *TRI)
5177 : getVGPROpMapping(Reg: DstReg, MRI, TRI: *TRI);
5178
5179 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5180 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5181 OpdsMapping[4] =
5182 UseAGPRForm ? getAGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI)
5183 : getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5184 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5185 break;
5186 }
5187 case Intrinsic::amdgcn_interp_p1:
5188 case Intrinsic::amdgcn_interp_p2:
5189 case Intrinsic::amdgcn_interp_mov:
5190 case Intrinsic::amdgcn_interp_p1_f16:
5191 case Intrinsic::amdgcn_interp_p2_f16:
5192 case Intrinsic::amdgcn_lds_param_load: {
5193 const int M0Idx = MI.getNumOperands() - 1;
5194 Register M0Reg = MI.getOperand(i: M0Idx).getReg();
5195 unsigned M0Bank = getRegBankID(Reg: M0Reg, MRI, Default: AMDGPU::SGPRRegBankID);
5196 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5197
5198 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5199 for (int I = 2; I != M0Idx && MI.getOperand(i: I).isReg(); ++I)
5200 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5201
5202 // Must be SGPR, but we must take whatever the original bank is and fix it
5203 // later.
5204 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5205 break;
5206 }
5207 case Intrinsic::amdgcn_interp_inreg_p10:
5208 case Intrinsic::amdgcn_interp_inreg_p2:
5209 case Intrinsic::amdgcn_interp_inreg_p10_f16:
5210 case Intrinsic::amdgcn_interp_inreg_p2_f16:
5211 case Intrinsic::amdgcn_interp_p10_rtz_f16:
5212 case Intrinsic::amdgcn_interp_p2_rtz_f16: {
5213 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5214 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5215 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5216 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5217 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5218 break;
5219 }
5220 case Intrinsic::amdgcn_permlane16_swap:
5221 case Intrinsic::amdgcn_permlane32_swap: {
5222 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5223 OpdsMapping[0] = OpdsMapping[1] = OpdsMapping[3] = OpdsMapping[4] =
5224 AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5225 break;
5226 }
5227 case Intrinsic::amdgcn_ballot: {
5228 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5229 unsigned SrcSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5230 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
5231 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: SrcSize);
5232 break;
5233 }
5234 case Intrinsic::amdgcn_inverse_ballot: {
5235 // This must be an SGPR, but accept a VGPR.
5236 Register MaskReg = MI.getOperand(i: 2).getReg();
5237 unsigned MaskSize = MRI.getType(Reg: MaskReg).getSizeInBits();
5238 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5239 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5240 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: MaskSize);
5241 break;
5242 }
5243 case Intrinsic::amdgcn_bitop3: {
5244 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5245 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5246 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5247 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5248 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5249 break;
5250 }
5251 case Intrinsic::amdgcn_s_quadmask:
5252 case Intrinsic::amdgcn_s_wqm: {
5253 Register MaskReg = MI.getOperand(i: 2).getReg();
5254 unsigned MaskSize = MRI.getType(Reg: MaskReg).getSizeInBits();
5255 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5256 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: MaskSize);
5257 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: MaskSize);
5258 break;
5259 }
5260 case Intrinsic::amdgcn_wave_reduce_add:
5261 case Intrinsic::amdgcn_wave_reduce_fadd:
5262 case Intrinsic::amdgcn_wave_reduce_sub:
5263 case Intrinsic::amdgcn_wave_reduce_fsub:
5264 case Intrinsic::amdgcn_wave_reduce_min:
5265 case Intrinsic::amdgcn_wave_reduce_umin:
5266 case Intrinsic::amdgcn_wave_reduce_fmin:
5267 case Intrinsic::amdgcn_wave_reduce_max:
5268 case Intrinsic::amdgcn_wave_reduce_umax:
5269 case Intrinsic::amdgcn_wave_reduce_fmax:
5270 case Intrinsic::amdgcn_wave_reduce_and:
5271 case Intrinsic::amdgcn_wave_reduce_or:
5272 case Intrinsic::amdgcn_wave_reduce_xor: {
5273 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5274 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: DstSize);
5275 unsigned OpSize = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5276 auto regBankID =
5277 isSALUMapping(MI) ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5278 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: regBankID, Size: OpSize);
5279 break;
5280 }
5281 case Intrinsic::amdgcn_s_bitreplicate: {
5282 Register MaskReg = MI.getOperand(i: 2).getReg();
5283 unsigned MaskBank = getRegBankID(Reg: MaskReg, MRI, Default: AMDGPU::SGPRRegBankID);
5284 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 64);
5285 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: MaskBank, Size: 32);
5286 break;
5287 }
5288 case Intrinsic::amdgcn_wave_shuffle: {
5289 unsigned OpSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5290 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5291 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5292 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: OpSize);
5293 break;
5294 }
5295 }
5296 break;
5297 }
5298 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
5299 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
5300 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
5301 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
5302 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
5303 auto IntrID = AMDGPU::getIntrinsicID(I: MI);
5304 const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(Intr: IntrID);
5305 assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic");
5306 // Non-images can have complications from operands that allow both SGPR
5307 // and VGPR. For now it's too complicated to figure out the final opcode
5308 // to derive the register bank from the MCInstrDesc.
5309 assert(RSrcIntrin->IsImage);
5310 return getImageMapping(MRI, MI, RsrcIdx: RSrcIntrin->RsrcArg);
5311 }
5312 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
5313 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
5314 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY: {
5315 bool IsDualOrBVH8 =
5316 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY ||
5317 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY;
5318 unsigned NumMods = IsDualOrBVH8 ? 0 : 1; // Has A16 modifier
5319 unsigned LastRegOpIdx = MI.getNumExplicitOperands() - 1 - NumMods;
5320 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5321 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5322 if (IsDualOrBVH8) {
5323 OpdsMapping[1] = AMDGPU::getValueMapping(
5324 BankID: AMDGPU::VGPRRegBankID,
5325 Size: MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getSizeInBits());
5326 OpdsMapping[2] = AMDGPU::getValueMapping(
5327 BankID: AMDGPU::VGPRRegBankID,
5328 Size: MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits());
5329 }
5330 OpdsMapping[LastRegOpIdx] =
5331 getSGPROpMapping(Reg: MI.getOperand(i: LastRegOpIdx).getReg(), MRI, TRI: *TRI);
5332 if (LastRegOpIdx == 3) {
5333 // Sequential form: all operands combined into VGPR256/VGPR512
5334 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getSizeInBits();
5335 if (Size > 256)
5336 Size = 512;
5337 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5338 } else {
5339 // NSA form
5340 unsigned FirstSrcOpIdx = IsDualOrBVH8 ? 4 : 2;
5341 for (unsigned I = FirstSrcOpIdx; I < LastRegOpIdx; ++I) {
5342 unsigned Size = MRI.getType(Reg: MI.getOperand(i: I).getReg()).getSizeInBits();
5343 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5344 }
5345 }
5346 break;
5347 }
5348 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS:
5349 case AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {
5350 auto IntrID = cast<GIntrinsic>(Val: MI).getIntrinsicID();
5351 switch (IntrID) {
5352 case Intrinsic::amdgcn_s_getreg:
5353 case Intrinsic::amdgcn_s_memtime:
5354 case Intrinsic::amdgcn_s_memrealtime:
5355 case Intrinsic::amdgcn_s_get_waveid_in_workgroup:
5356 case Intrinsic::amdgcn_s_sendmsg_rtn: {
5357 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5358 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5359 break;
5360 }
5361 case Intrinsic::amdgcn_global_atomic_fmin_num:
5362 case Intrinsic::amdgcn_global_atomic_fmax_num:
5363 case Intrinsic::amdgcn_flat_atomic_fmin_num:
5364 case Intrinsic::amdgcn_flat_atomic_fmax_num:
5365 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
5366 case Intrinsic::amdgcn_global_load_tr_b64:
5367 case Intrinsic::amdgcn_global_load_tr_b128:
5368 case Intrinsic::amdgcn_global_load_tr4_b64:
5369 case Intrinsic::amdgcn_global_load_tr6_b96:
5370 case Intrinsic::amdgcn_ds_load_tr8_b64:
5371 case Intrinsic::amdgcn_ds_load_tr16_b128:
5372 case Intrinsic::amdgcn_ds_load_tr4_b64:
5373 case Intrinsic::amdgcn_ds_load_tr6_b96:
5374 case Intrinsic::amdgcn_ds_read_tr4_b64:
5375 case Intrinsic::amdgcn_ds_read_tr6_b96:
5376 case Intrinsic::amdgcn_ds_read_tr8_b64:
5377 case Intrinsic::amdgcn_ds_read_tr16_b64:
5378 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
5379 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
5380 return getDefaultMappingAllVGPR(MI);
5381 case Intrinsic::amdgcn_ds_ordered_add:
5382 case Intrinsic::amdgcn_ds_ordered_swap: {
5383 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5384 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5385 unsigned M0Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5386 Default: AMDGPU::SGPRRegBankID);
5387 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5388 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5389 break;
5390 }
5391 case Intrinsic::amdgcn_ds_append:
5392 case Intrinsic::amdgcn_ds_consume: {
5393 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5394 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5395 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5396 break;
5397 }
5398 case Intrinsic::amdgcn_exp_compr:
5399 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5400 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5401 break;
5402 case Intrinsic::amdgcn_exp:
5403 // FIXME: Could we support packed types here?
5404 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5405 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5406 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5407 OpdsMapping[6] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5408 break;
5409 case Intrinsic::amdgcn_exp_row:
5410 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5411 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5412 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5413 OpdsMapping[6] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5414 OpdsMapping[8] = getSGPROpMapping(Reg: MI.getOperand(i: 8).getReg(), MRI, TRI: *TRI);
5415 break;
5416 case Intrinsic::amdgcn_s_alloc_vgpr:
5417 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 1);
5418 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 32);
5419 break;
5420 case Intrinsic::amdgcn_s_sendmsg:
5421 case Intrinsic::amdgcn_s_sendmsghalt: {
5422 // This must be an SGPR, but accept a VGPR.
5423 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5424 Default: AMDGPU::SGPRRegBankID);
5425 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5426 break;
5427 }
5428 case Intrinsic::amdgcn_s_setreg: {
5429 // This must be an SGPR, but accept a VGPR.
5430 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5431 Default: AMDGPU::SGPRRegBankID);
5432 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5433 break;
5434 }
5435 case Intrinsic::amdgcn_s_ttracedata: {
5436 // This must be an SGPR, but accept a VGPR.
5437 unsigned Bank =
5438 getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5439 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5440 break;
5441 }
5442 case Intrinsic::amdgcn_end_cf: {
5443 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5444 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5445 break;
5446 }
5447 case Intrinsic::amdgcn_else: {
5448 unsigned WaveSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5449 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5450 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: WaveSize);
5451 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: WaveSize);
5452 break;
5453 }
5454 case Intrinsic::amdgcn_init_whole_wave:
5455 case Intrinsic::amdgcn_live_mask: {
5456 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5457 break;
5458 }
5459 case Intrinsic::amdgcn_wqm_demote:
5460 case Intrinsic::amdgcn_kill: {
5461 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5462 break;
5463 }
5464 case Intrinsic::amdgcn_ptr_s_buffer_load: {
5465 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5466 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5467 OpdsMapping[3] = getSGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5468 break;
5469 }
5470 case Intrinsic::amdgcn_raw_buffer_load:
5471 case Intrinsic::amdgcn_raw_ptr_buffer_load:
5472 case Intrinsic::amdgcn_raw_atomic_buffer_load:
5473 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
5474 case Intrinsic::amdgcn_raw_tbuffer_load:
5475 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
5476 // FIXME: Should make intrinsic ID the last operand of the instruction,
5477 // then this would be the same as store
5478 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5479 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5480 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5481 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5482 break;
5483 }
5484 case Intrinsic::amdgcn_raw_buffer_load_lds:
5485 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
5486 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
5487 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: {
5488 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5489 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5490 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5491 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5492 break;
5493 }
5494 case Intrinsic::amdgcn_raw_buffer_store:
5495 case Intrinsic::amdgcn_raw_ptr_buffer_store:
5496 case Intrinsic::amdgcn_raw_buffer_store_format:
5497 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
5498 case Intrinsic::amdgcn_raw_tbuffer_store:
5499 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
5500 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5501 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5502 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5503 OpdsMapping[4] = getSGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5504 break;
5505 }
5506 case Intrinsic::amdgcn_struct_buffer_load:
5507 case Intrinsic::amdgcn_struct_ptr_buffer_load:
5508 case Intrinsic::amdgcn_struct_tbuffer_load:
5509 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
5510 case Intrinsic::amdgcn_struct_atomic_buffer_load:
5511 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
5512 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5513 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5514 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5515 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5516 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5517 break;
5518 }
5519 case Intrinsic::amdgcn_struct_buffer_load_lds:
5520 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
5521 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
5522 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
5523 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5524 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5525 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5526 OpdsMapping[5] = getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5527 OpdsMapping[6] = getSGPROpMapping(Reg: MI.getOperand(i: 6).getReg(), MRI, TRI: *TRI);
5528 break;
5529 }
5530 case Intrinsic::amdgcn_struct_buffer_store:
5531 case Intrinsic::amdgcn_struct_ptr_buffer_store:
5532 case Intrinsic::amdgcn_struct_tbuffer_store:
5533 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
5534 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5535 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5536 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5537 OpdsMapping[4] = getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI);
5538 OpdsMapping[5] = getSGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI);
5539 break;
5540 }
5541 case Intrinsic::amdgcn_init_exec_from_input: {
5542 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5543 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size);
5544 break;
5545 }
5546 case Intrinsic::amdgcn_ds_gws_init:
5547 case Intrinsic::amdgcn_ds_gws_barrier:
5548 case Intrinsic::amdgcn_ds_gws_sema_br: {
5549 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5550
5551 // This must be an SGPR, but accept a VGPR.
5552 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5553 Default: AMDGPU::SGPRRegBankID);
5554 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5555 break;
5556 }
5557 case Intrinsic::amdgcn_ds_gws_sema_v:
5558 case Intrinsic::amdgcn_ds_gws_sema_p:
5559 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
5560 // This must be an SGPR, but accept a VGPR.
5561 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
5562 Default: AMDGPU::SGPRRegBankID);
5563 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: Bank, Size: 32);
5564 break;
5565 }
5566 case Intrinsic::amdgcn_cluster_load_b32:
5567 case Intrinsic::amdgcn_cluster_load_b64:
5568 case Intrinsic::amdgcn_cluster_load_b128: {
5569 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5570 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5571 unsigned M0Bank =
5572 getRegBankID(Reg: MI.getOperand(i: 4).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5573 OpdsMapping[4] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5574 break;
5575 }
5576 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
5577 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
5578 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
5579 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
5580 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5581 // LDS address goes into $vdst (VGPR).
5582 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5583 unsigned M0Bank =
5584 getRegBankID(Reg: MI.getOperand(i: 5).getReg(), MRI, Default: AMDGPU::SGPRRegBankID);
5585 OpdsMapping[5] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5586 break;
5587 }
5588 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
5589 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
5590 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
5591 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
5592 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
5593 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
5594 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
5595 case Intrinsic::amdgcn_global_load_async_to_lds_b128: {
5596 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5597 // LDS address goes into $vdst/$vdata (VGPR).
5598 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5599 break;
5600 }
5601 case Intrinsic::amdgcn_load_to_lds:
5602 case Intrinsic::amdgcn_load_async_to_lds:
5603 case Intrinsic::amdgcn_global_load_lds:
5604 case Intrinsic::amdgcn_global_load_async_lds: {
5605 OpdsMapping[1] = getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5606 // LDS address goes into M0 (SGPR).
5607 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5608 break;
5609 }
5610 case Intrinsic::amdgcn_lds_direct_load: {
5611 const int M0Idx = MI.getNumOperands() - 1;
5612 Register M0Reg = MI.getOperand(i: M0Idx).getReg();
5613 unsigned M0Bank = getRegBankID(Reg: M0Reg, MRI, Default: AMDGPU::SGPRRegBankID);
5614 unsigned DstSize = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5615
5616 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: DstSize);
5617 for (int I = 2; I != M0Idx && MI.getOperand(i: I).isReg(); ++I)
5618 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: 32);
5619
5620 // Must be SGPR, but we must take whatever the original bank is and fix it
5621 // later.
5622 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(BankID: M0Bank, Size: 32);
5623 break;
5624 }
5625 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
5626 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn:
5627 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5628 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5629 break;
5630 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
5631 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
5632 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
5633 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
5634 OpdsMapping[0] =
5635 getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI); // %vdst
5636 OpdsMapping[1] =
5637 getVGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI); // %addr
5638 OpdsMapping[3] =
5639 getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI); // %addr
5640 OpdsMapping[4] =
5641 getVGPROpMapping(Reg: MI.getOperand(i: 4).getReg(), MRI, TRI: *TRI); // %data0
5642 OpdsMapping[5] =
5643 getVGPROpMapping(Reg: MI.getOperand(i: 5).getReg(), MRI, TRI: *TRI); // %data1
5644 break;
5645 }
5646 case Intrinsic::amdgcn_s_sleep_var:
5647 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5648 break;
5649 case Intrinsic::amdgcn_s_barrier_join:
5650 case Intrinsic::amdgcn_s_wakeup_barrier:
5651 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5652 break;
5653 case Intrinsic::amdgcn_s_barrier_init:
5654 case Intrinsic::amdgcn_s_barrier_signal_var:
5655 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5656 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5657 break;
5658 case Intrinsic::amdgcn_s_barrier_signal_isfirst: {
5659 const unsigned ResultSize = 1;
5660 OpdsMapping[0] =
5661 AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: ResultSize);
5662 break;
5663 }
5664 case Intrinsic::amdgcn_s_get_barrier_state:
5665 case Intrinsic::amdgcn_s_get_named_barrier_state: {
5666 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5667 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5668 break;
5669 }
5670 case Intrinsic::amdgcn_pops_exiting_wave_id:
5671 return getDefaultMappingSOP(MI);
5672 case Intrinsic::amdgcn_tensor_load_to_lds:
5673 case Intrinsic::amdgcn_tensor_store_from_lds: {
5674 // Lie and claim everything is legal, even all operands need to be
5675 // SGPRs. applyMapping will have to deal with it with readfirstlane.
5676 for (unsigned I = 1; I < MI.getNumOperands(); ++I) {
5677 if (MI.getOperand(i: I).isReg()) {
5678 Register Reg = MI.getOperand(i: I).getReg();
5679 auto OpBank = getRegBankID(Reg, MRI);
5680 unsigned Size = getSizeInBits(Reg, MRI, TRI: *TRI);
5681 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: OpBank, Size);
5682 }
5683 }
5684 break;
5685 }
5686 case Intrinsic::amdgcn_s_prefetch_data:
5687 case Intrinsic::amdgcn_s_prefetch_inst: {
5688 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5689 OpdsMapping[2] = getSGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5690 break;
5691 }
5692 case Intrinsic::amdgcn_flat_prefetch:
5693 case Intrinsic::amdgcn_global_prefetch:
5694 return getDefaultMappingVOP(MI);
5695 default:
5696 return getInvalidInstructionMapping();
5697 }
5698 break;
5699 }
5700 case AMDGPU::G_SELECT: {
5701 unsigned Size = MRI.getType(Reg: MI.getOperand(i: 0).getReg()).getSizeInBits();
5702 unsigned Op2Bank = getRegBankID(Reg: MI.getOperand(i: 2).getReg(), MRI,
5703 Default: AMDGPU::SGPRRegBankID);
5704 unsigned Op3Bank = getRegBankID(Reg: MI.getOperand(i: 3).getReg(), MRI,
5705 Default: AMDGPU::SGPRRegBankID);
5706 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID &&
5707 Op3Bank == AMDGPU::SGPRRegBankID;
5708
5709 unsigned CondBankDefault = SGPRSrcs ?
5710 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5711 unsigned CondBank = getRegBankID(Reg: MI.getOperand(i: 1).getReg(), MRI,
5712 Default: CondBankDefault);
5713 if (CondBank == AMDGPU::SGPRRegBankID)
5714 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID;
5715 else if (CondBank == AMDGPU::VGPRRegBankID)
5716 CondBank = AMDGPU::VCCRegBankID;
5717
5718 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ?
5719 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
5720
5721 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID);
5722
5723 // TODO: Should report 32-bit for scalar condition type.
5724 if (Size == 64) {
5725 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5726 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: CondBank, Size: 1);
5727 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5728 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(BankID: Bank, Size);
5729 } else {
5730 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size);
5731 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: CondBank, Size: 1);
5732 OpdsMapping[2] = AMDGPU::getValueMapping(BankID: Bank, Size);
5733 OpdsMapping[3] = AMDGPU::getValueMapping(BankID: Bank, Size);
5734 }
5735
5736 break;
5737 }
5738
5739 case AMDGPU::G_SI_CALL: {
5740 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::SGPRRegBankID, Size: 64);
5741 // Lie and claim everything is legal, even though some need to be
5742 // SGPRs. applyMapping will have to deal with it as a waterfall loop.
5743 OpdsMapping[1] = getSGPROpMapping(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5744
5745 // Allow anything for implicit arguments
5746 for (unsigned I = 4; I < MI.getNumOperands(); ++I) {
5747 if (MI.getOperand(i: I).isReg()) {
5748 Register Reg = MI.getOperand(i: I).getReg();
5749 auto OpBank = getRegBankID(Reg, MRI);
5750 unsigned Size = getSizeInBits(Reg, MRI, TRI: *TRI);
5751 OpdsMapping[I] = AMDGPU::getValueMapping(BankID: OpBank, Size);
5752 }
5753 }
5754 break;
5755 }
5756 case AMDGPU::G_LOAD:
5757 case AMDGPU::G_ZEXTLOAD:
5758 case AMDGPU::G_SEXTLOAD:
5759 return getInstrMappingForLoad(MI);
5760
5761 case AMDGPU::G_ATOMICRMW_XCHG:
5762 case AMDGPU::G_ATOMICRMW_ADD:
5763 case AMDGPU::G_ATOMICRMW_SUB:
5764 case AMDGPU::G_ATOMICRMW_AND:
5765 case AMDGPU::G_ATOMICRMW_OR:
5766 case AMDGPU::G_ATOMICRMW_XOR:
5767 case AMDGPU::G_ATOMICRMW_MAX:
5768 case AMDGPU::G_ATOMICRMW_MIN:
5769 case AMDGPU::G_ATOMICRMW_UMAX:
5770 case AMDGPU::G_ATOMICRMW_UMIN:
5771 case AMDGPU::G_ATOMICRMW_FADD:
5772 case AMDGPU::G_ATOMICRMW_FMIN:
5773 case AMDGPU::G_ATOMICRMW_FMAX:
5774 case AMDGPU::G_ATOMICRMW_UINC_WRAP:
5775 case AMDGPU::G_ATOMICRMW_UDEC_WRAP:
5776 case AMDGPU::G_ATOMICRMW_USUB_COND:
5777 case AMDGPU::G_ATOMICRMW_USUB_SAT:
5778 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: {
5779 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5780 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
5781 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5782 break;
5783 }
5784 case AMDGPU::G_ATOMIC_CMPXCHG: {
5785 OpdsMapping[0] = getVGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5786 OpdsMapping[1] = getValueMappingForPtr(MRI, PtrReg: MI.getOperand(i: 1).getReg());
5787 OpdsMapping[2] = getVGPROpMapping(Reg: MI.getOperand(i: 2).getReg(), MRI, TRI: *TRI);
5788 OpdsMapping[3] = getVGPROpMapping(Reg: MI.getOperand(i: 3).getReg(), MRI, TRI: *TRI);
5789 break;
5790 }
5791 case AMDGPU::G_BRCOND: {
5792 unsigned Bank = getRegBankID(Reg: MI.getOperand(i: 0).getReg(), MRI,
5793 Default: AMDGPU::SGPRRegBankID);
5794 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
5795 if (Bank != AMDGPU::SGPRRegBankID)
5796 Bank = AMDGPU::VCCRegBankID;
5797
5798 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: Bank, Size: 1);
5799 break;
5800 }
5801 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
5802 return getDefaultMappingVOP(MI);
5803 case AMDGPU::G_PREFETCH:
5804 OpdsMapping[0] = getSGPROpMapping(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5805 break;
5806 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP:
5807 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN:
5808 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VCCRegBankID, Size: 1);
5809 break;
5810 case AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR:
5811 case AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR: {
5812 unsigned Size = getSizeInBits(Reg: MI.getOperand(i: 0).getReg(), MRI, TRI: *TRI);
5813 unsigned PtrSize = getSizeInBits(Reg: MI.getOperand(i: 1).getReg(), MRI, TRI: *TRI);
5814 OpdsMapping[0] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size);
5815 OpdsMapping[1] = AMDGPU::getValueMapping(BankID: AMDGPU::VGPRRegBankID, Size: PtrSize);
5816 break;
5817 }
5818 }
5819
5820 return getInstructionMapping(/*ID*/1, /*Cost*/1,
5821 OperandsMapping: getOperandsMapping(OpdsMapping),
5822 NumOperands: MI.getNumOperands());
5823}
5824