1//===-- AMDGPURegBankLegalizeHelper.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Implements actual lowering algorithms for each ID that can be used in
10/// Rule.OperandMapping. Similar to legalizer helper but with register banks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPURegBankLegalizeHelper.h"
15#include "AMDGPUGlobalISelUtils.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPURegBankLegalizeRules.h"
19#include "AMDGPURegisterBankInfo.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineInstr.h"
27#include "llvm/CodeGen/MachineUniformityAnalysis.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29
30#define DEBUG_TYPE "amdgpu-reg-bank-legalize"
31
32using namespace llvm;
33using namespace AMDGPU;
34
35RegBankLegalizeHelper::RegBankLegalizeHelper(
36 MachineIRBuilder &B, const MachineUniformityInfo &MUI,
37 GISelValueTracking *VT, const RegisterBankInfo &RBI,
38 const RegBankLegalizeRules &RBLRules)
39 : MF(B.getMF()), MFI(MF.getInfo<SIMachineFunctionInfo>()),
40 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()), B(B),
41 MRI(*B.getMRI()), MUI(MUI), VT(VT), RBI(RBI), MORE(MF, nullptr),
42 RBLRules(RBLRules), IsWave32(ST.isWave32()),
43 SgprRB(&RBI.getRegBank(ID: AMDGPU::SGPRRegBankID)),
44 VgprRB(&RBI.getRegBank(ID: AMDGPU::VGPRRegBankID)),
45 AgprRB(&RBI.getRegBank(ID: AMDGPU::AGPRRegBankID)),
46 VccRB(&RBI.getRegBank(ID: AMDGPU::VCCRegBankID)) {}
47
48bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
49 const SetOfRulesForOpcode *RuleSet = RBLRules.getRulesForOpc(MI);
50 if (!RuleSet) {
51 reportGISelFailure(MF, MORE, DEBUG_TYPE,
52 Msg: "No AMDGPU RegBankLegalize rules defined for opcode",
53 MI);
54 return false;
55 }
56
57 const RegBankLLTMapping *Mapping = RuleSet->findMappingForMI(MI, MRI, MUI);
58 if (!Mapping) {
59 reportGISelFailure(MF, MORE, DEBUG_TYPE,
60 Msg: "AMDGPU RegBankLegalize: none of the rules defined with "
61 "'Any' for MI's opcode matched MI",
62 MI);
63 return false;
64 }
65
66 WaterfallInfo WFI;
67 unsigned OpIdx = 0;
68 if (!Mapping->DstOpMapping.empty()) {
69 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
70 if (!applyMappingDst(MI, OpIdx, MethodIDs: Mapping->DstOpMapping))
71 return false;
72 }
73 if (!Mapping->SrcOpMapping.empty()) {
74 B.setInstr(MI);
75 if (!applyMappingSrc(MI, OpIdx, MethodIDs: Mapping->SrcOpMapping, WFI))
76 return false;
77 }
78
79 if (!lower(MI, Mapping: *Mapping, WFI))
80 return false;
81
82 if (!WFI.SgprWaterfallOperandRegs.empty()) {
83 if (!executeInWaterfallLoop(B, WFI))
84 return false;
85 }
86
87 return true;
88}
89
90bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
91 const WaterfallInfo &WFI) {
92 assert(WFI.Start.isValid() && WFI.End.isValid() &&
93 "Waterfall range not initialized");
94
95 // Track use registers which have already been expanded with a readfirstlane
96 // sequence. This may have multiple uses if moving a sequence.
97 DenseMap<Register, Register> WaterfalledRegMap;
98
99 MachineBasicBlock &MBB = B.getMBB();
100 MachineFunction &MF = B.getMF();
101
102 MachineBasicBlock::iterator BeginIt = WFI.Start;
103 MachineBasicBlock::iterator EndIt = WFI.End;
104
105 const SIRegisterInfo *TRI = ST.getRegisterInfo();
106 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
107 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
108
109#ifndef NDEBUG
110 const int OrigRangeSize = std::distance(BeginIt, EndIt);
111#endif
112
113 MachineRegisterInfo &MRI = *B.getMRI();
114 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
115 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
116
117 // Don't bother using generic instructions/registers for the exec mask.
118 B.setInstr(*WFI.Start);
119 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: InitSaveExecReg);
120
121 Register SavedExec = MRI.createVirtualRegister(RegClass: WaveRC);
122
123 // To insert the loop we need to split the block. Move everything before
124 // this point to a new block, and insert a new empty block before this
125 // instruction.
126 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
127 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
128 MachineBasicBlock *RestoreExecBB = MF.CreateMachineBasicBlock();
129 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
130 MachineFunction::iterator MBBI(MBB);
131 ++MBBI;
132 MF.insert(MBBI, MBB: LoopBB);
133 MF.insert(MBBI, MBB: BodyBB);
134 MF.insert(MBBI, MBB: RestoreExecBB);
135 MF.insert(MBBI, MBB: RemainderBB);
136
137 LoopBB->addSuccessor(Succ: BodyBB);
138 BodyBB->addSuccessor(Succ: RestoreExecBB);
139 BodyBB->addSuccessor(Succ: LoopBB);
140
141 // Move the rest of the block into a new block.
142 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
143 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: EndIt, To: MBB.end());
144
145 MBB.addSuccessor(Succ: LoopBB);
146 RestoreExecBB->addSuccessor(Succ: RemainderBB);
147
148 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
149
150 // +-MBB:------------+
151 // | ... |
152 // | %0 = G_INST_1 |
153 // | %Dst = MI %Vgpr |
154 // | %1 = G_INST_2 |
155 // | ... |
156 // +-----------------+
157 // ->
158 // +-MBB-------------------------------+
159 // | ... |
160 // | %0 = G_INST_1 |
161 // | %SaveExecReg = S_MOV_B32 $exec_lo |
162 // +----------------|------------------+
163 // | /------------------------------|
164 // V V |
165 // +-LoopBB---------------------------------------------------------------+ |
166 // | %CurrentLaneReg:sgpr(s32) = READFIRSTLANE %Vgpr | |
167 // | instead of executing for each lane, see if other lanes had | |
168 // | same value for %Vgpr and execute for them also. | |
169 // | %CondReg:vcc(s1) = G_ICMP eq %CurrentLaneReg, %Vgpr | |
170 // | %CondRegLM:sreg_32 = ballot %CondReg // copy vcc to sreg32 lane mask | |
171 // | %SavedExec = S_AND_SAVEEXEC_B32 %CondRegLM | |
172 // | exec is active for lanes with the same "CurrentLane value" in Vgpr | |
173 // +----------------|-----------------------------------------------------+ |
174 // V |
175 // +-BodyBB------------------------------------------------------------+ |
176 // | %Dst = MI %CurrentLaneReg:sgpr(s32) | |
177 // | executed only for active lanes and written to Dst | |
178 // | $exec = S_XOR_B32 $exec, %SavedExec | |
179 // | set active lanes to 0 in SavedExec, lanes that did not write to | |
180 // | Dst yet, and set this as new exec (for READFIRSTLANE and ICMP) | |
181 // | SI_WATERFALL_LOOP LoopBB |-----|
182 // +----------------|--------------------------------------------------+
183 // V
184 // +-RestoreExecBB--------------------------+
185 // | $exec_lo = S_MOV_B32_term %SaveExecReg |
186 // +----------------|-----------------------+
187 // V
188 // +-RemainderBB:----------------------+
189 // | %1 = G_INST_2 |
190 // | ... |
191 // +---------------------------------- +
192
193 // Move the instruction into the loop body. Note we moved everything after
194 // Range.end() already into a new block, so Range.end() is no longer valid.
195 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: BeginIt, To: MBB.end());
196
197 // Figure out the iterator range after splicing the instructions.
198 MachineBasicBlock::iterator NewBegin = BeginIt;
199 auto NewEnd = BodyBB->end();
200 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
201
202 B.setMBB(*LoopBB);
203 Register CondReg;
204
205 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
206 for (MachineOperand &Op : MI.all_uses()) {
207 Register OldReg = Op.getReg();
208 if (!WFI.SgprWaterfallOperandRegs.count(V: OldReg))
209 continue;
210
211 // See if we already processed this register in another instruction in
212 // the sequence.
213 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
214 if (OldVal != WaterfalledRegMap.end()) {
215 Op.setReg(OldVal->second);
216 continue;
217 }
218
219 Register OpReg = Op.getReg();
220 LLT OpTy = MRI.getType(Reg: OpReg);
221
222 // TODO: support for agpr
223 assert(MRI.getRegBank(OpReg) == VgprRB);
224 Register CurrentLaneReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: OpTy});
225 buildReadFirstLane(B, SgprDst: CurrentLaneReg, VgprSrc: OpReg, RBI);
226
227 // Build the comparison(s), CurrentLaneReg == OpReg.
228 unsigned OpSize = OpTy.getSizeInBits();
229 unsigned PartSize = (OpSize % 64 == 0) ? 64 : 32;
230 LLT PartTy = LLT::integer(SizeInBits: PartSize);
231 unsigned NumParts = OpSize / PartSize;
232 SmallVector<Register, 8> OpParts;
233 SmallVector<Register, 8> CurrentLaneParts;
234
235 if (NumParts == 1) {
236 OpParts.push_back(Elt: OpReg);
237 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
238 } else {
239 auto UnmergeOp = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: PartTy}, Op: OpReg);
240 auto UnmergeCurrLane = B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: PartTy}, Op: CurrentLaneReg);
241 for (unsigned i = 0; i < NumParts; ++i) {
242 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
243 CurrentLaneParts.push_back(Elt: UnmergeCurrLane.getReg(Idx: i));
244 }
245 }
246
247 for (unsigned i = 0; i < NumParts; ++i) {
248 Register CmpReg = MRI.createVirtualRegister(RegAttr: VccRB_S1);
249 B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpReg, Op0: CurrentLaneParts[i], Op1: OpParts[i]);
250
251 if (!CondReg)
252 CondReg = CmpReg;
253 else
254 CondReg = B.buildAnd(Dst: VccRB_S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
255 }
256
257 Op.setReg(CurrentLaneReg);
258
259 // Make sure we don't re-process this register again.
260 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
261 }
262 }
263
264 // Copy vcc to sgpr32/64, ballot becomes a no-op during instruction selection.
265 Register CondRegLM =
266 MRI.createVirtualRegister(RegAttr: {.RCOrRB: WaveRC, .Ty: LLT::integer(SizeInBits: IsWave32 ? 32 : 64)});
267 B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot, Res: CondRegLM).addReg(RegNo: CondReg);
268
269 // Update EXEC, save the original EXEC value to SavedExec.
270 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
271 .addDef(RegNo: SavedExec)
272 .addReg(RegNo: CondRegLM)
273 .setOperandDead(3);
274 MRI.setSimpleHint(VReg: SavedExec, PrefReg: CondRegLM);
275
276 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
277
278 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
279 B.buildInstr(Opcode: LMC.XorTermOpc)
280 .addDef(RegNo: LMC.ExecReg)
281 .addReg(RegNo: LMC.ExecReg)
282 .addReg(RegNo: SavedExec)
283 .setOperandDead(3);
284
285 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
286 // s_cbranch_scc0?
287
288 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
289 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
290
291 // Save the EXEC mask before the loop.
292 B.setInsertPt(MBB, II: MBB.end());
293 B.buildInstr(Opcode: LMC.MovOpc).addDef(RegNo: SaveExecReg).addReg(RegNo: LMC.ExecReg);
294
295 // Restore the EXEC mask after the loop.
296 B.setInsertPt(MBB&: *RestoreExecBB, II: RestoreExecBB->begin());
297 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
298
299 // Set the insert point after the original instruction, so any new
300 // instructions will be in the remainder.
301 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
302
303 return true;
304}
305
306// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
307// the three offsets (voffset, soffset and instoffset)
308unsigned RegBankLegalizeHelper::setBufferOffsets(
309 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
310 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) {
311 if (std::optional<int64_t> Imm =
312 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI)) {
313 uint32_t SOffset, ImmOffset;
314 if (TII.splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
315 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
316 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
317 InstOffsetVal = ImmOffset;
318 return SOffset + ImmOffset;
319 }
320 }
321 const bool CheckNUW = ST.hasGFX1250Insts();
322 auto [Base, Offset] = AMDGPU::getBaseWithConstantOffset(
323 MRI, Reg: CombinedOffset, /*KnownBits=*/ValueTracking: nullptr,
324 /*CheckNUW=*/CheckNUW);
325 uint32_t SOffset, ImmOffset;
326 if (static_cast<int32_t>(Offset) > 0 &&
327 TII.splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
328 if (Base.isValid() && MRI.getRegBank(Reg: Base) == VgprRB) {
329 VOffsetReg = Base;
330 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
331 InstOffsetVal = ImmOffset;
332 return 0;
333 }
334 // If we have SGPR base, we can use it for soffset.
335 if (SOffset == 0) {
336 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
337 SOffsetReg = Base;
338 InstOffsetVal = ImmOffset;
339 return 0;
340 }
341 }
342 // Handle the variable sgpr + vgpr case.
343 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI);
344 if (Add && static_cast<int32_t>(Offset) >= 0 &&
345 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
346 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI);
347 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI);
348 const RegisterBank *Src0Bank = MRI.getRegBank(Reg: Src0);
349 const RegisterBank *Src1Bank = MRI.getRegBank(Reg: Src1);
350 if (Src0Bank == VgprRB && Src1Bank == SgprRB) {
351 VOffsetReg = Src0;
352 SOffsetReg = Src1;
353 return 0;
354 }
355 if (Src0Bank == SgprRB && Src1Bank == VgprRB) {
356 VOffsetReg = Src1;
357 SOffsetReg = Src0;
358 return 0;
359 }
360 }
361 // Ensure we have a VGPR for the combined offset. This could be an issue if we
362 // have an SGPR offset and a VGPR resource.
363 if (MRI.getRegBank(Reg: CombinedOffset) == VgprRB) {
364 VOffsetReg = CombinedOffset;
365 } else {
366 VOffsetReg = B.buildCopy(Res: VgprRB_I32, Op: CombinedOffset).getReg(Idx: 0);
367 }
368 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: 0).getReg(Idx: 0);
369 return 0;
370}
371
372bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
373 ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
374 MachineFunction &MF = B.getMF();
375 assert(MI.getNumMemOperands() == 1);
376 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
377 Register Dst = MI.getOperand(i: 0).getReg();
378 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
379 Register Base = MI.getOperand(i: 1).getReg();
380 LLT PtrTy = MRI.getType(Reg: Base);
381 const RegisterBank *PtrRB = MRI.getRegBankOrNull(Reg: Base);
382 LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
383 SmallVector<Register, 4> LoadPartRegs;
384
385 unsigned ByteOffset = 0;
386 for (LLT PartTy : LLTBreakdown) {
387 Register BasePlusOffset;
388 if (ByteOffset == 0) {
389 BasePlusOffset = Base;
390 } else {
391 auto Offset = B.buildConstant(Res: {PtrRB, OffsetTy}, Val: ByteOffset);
392 BasePlusOffset =
393 B.buildObjectPtrOffset(Res: {PtrRB, PtrTy}, Op0: Base, Op1: Offset).getReg(Idx: 0);
394 }
395 auto *OffsetMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: ByteOffset, Ty: PartTy);
396 auto LoadPart = B.buildLoad(Res: {DstRB, PartTy}, Addr: BasePlusOffset, MMO&: *OffsetMMO);
397 LoadPartRegs.push_back(Elt: LoadPart.getReg(Idx: 0));
398 ByteOffset += PartTy.getSizeInBytes();
399 }
400
401 if (!MergeTy.isValid()) {
402 // Loads are of same size, concat or merge them together.
403 B.buildMergeLikeInstr(Res: Dst, Ops: LoadPartRegs);
404 } else {
405 // Loads are not all of same size, need to unmerge them to smaller pieces
406 // of MergeTy type, then merge pieces to Dst.
407 SmallVector<Register, 4> MergeTyParts;
408 for (Register Reg : LoadPartRegs) {
409 if (MRI.getType(Reg) == MergeTy) {
410 MergeTyParts.push_back(Elt: Reg);
411 } else {
412 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: Reg);
413 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i)
414 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
415 }
416 }
417 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
418 }
419 MI.eraseFromParent();
420 return true;
421}
422
423bool RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
424 LLT MergeTy) {
425 MachineFunction &MF = B.getMF();
426 assert(MI.getNumMemOperands() == 1);
427 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
428 Register Dst = MI.getOperand(i: 0).getReg();
429 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
430 Register Base = MI.getOperand(i: 1).getReg();
431
432 MachineMemOperand *WideMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: 0, Ty: WideTy);
433 auto WideLoad = B.buildLoad(Res: {DstRB, WideTy}, Addr: Base, MMO&: *WideMMO);
434
435 if (WideTy.isScalar()) {
436 B.buildTrunc(Res: Dst, Op: WideLoad);
437 } else {
438 SmallVector<Register, 4> MergeTyParts;
439 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: WideLoad);
440
441 LLT DstTy = MRI.getType(Reg: Dst);
442 unsigned NumElts = DstTy.getSizeInBits() / MergeTy.getSizeInBits();
443 for (unsigned i = 0; i < NumElts; ++i) {
444 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
445 }
446 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
447 }
448 MI.eraseFromParent();
449 return true;
450}
451
452bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
453 Register Dst = MI.getDstReg();
454 Register Ptr = MI.getPointerReg();
455 MachineMemOperand &MMO = MI.getMMO();
456 unsigned MemSize = 8 * MMO.getSize().getValue();
457
458 MachineMemOperand *WideMMO = B.getMF().getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: S32);
459
460 if (MI.getOpcode() == G_LOAD) {
461 B.buildLoad(Res: Dst, Addr: Ptr, MMO&: *WideMMO);
462 } else {
463 auto Load = B.buildLoad(Res: SgprRB_I32, Addr: Ptr, MMO&: *WideMMO);
464
465 if (MI.getOpcode() == G_ZEXTLOAD) {
466 APInt Mask = APInt::getLowBitsSet(numBits: S32.getSizeInBits(), loBitsSet: MemSize);
467 auto MaskCst = B.buildConstant(Res: SgprRB_I32, Val: Mask);
468 B.buildAnd(Dst, Src0: Load, Src1: MaskCst);
469 } else {
470 assert(MI.getOpcode() == G_SEXTLOAD);
471 B.buildSExtInReg(Res: Dst, Op: Load, ImmOp: MemSize);
472 }
473 }
474
475 MI.eraseFromParent();
476 return true;
477}
478
479bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
480 Register Dst = MI.getOperand(i: 0).getReg();
481 LLT Ty = MRI.getType(Reg: Dst);
482 Register Src = MI.getOperand(i: 1).getReg();
483 unsigned Opc = MI.getOpcode();
484 int TrueExtCst = Opc == G_SEXT ? -1 : 1;
485 if (Ty == S32 || Ty == S16) {
486 auto True = B.buildConstant(Res: {VgprRB, Ty}, Val: TrueExtCst);
487 auto False = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
488 B.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
489 } else if (Ty == S64) {
490 auto True = B.buildConstant(Res: {VgprRB_I32}, Val: TrueExtCst);
491 auto False = B.buildConstant(Res: {VgprRB_I32}, Val: 0);
492 auto Lo = B.buildSelect(Res: {VgprRB_I32}, Tst: Src, Op0: True, Op1: False);
493 MachineInstrBuilder Hi;
494 switch (Opc) {
495 case G_SEXT:
496 Hi = Lo;
497 break;
498 case G_ZEXT:
499 Hi = False;
500 break;
501 case G_ANYEXT:
502 Hi = B.buildUndef(Res: {VgprRB_I32});
503 break;
504 default:
505 reportGISelFailure(
506 MF, MORE, DEBUG_TYPE,
507 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Opcode not supported", MI);
508 return false;
509 }
510
511 B.buildMergeValues(Res: Dst, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
512 } else {
513 reportGISelFailure(
514 MF, MORE, DEBUG_TYPE,
515 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Type not supported", MI);
516 return false;
517 }
518
519 MI.eraseFromParent();
520 return true;
521}
522
523std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
524 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
525 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: 0x0000ffff);
526 auto Lo = B.buildAnd(Dst: SgprRB_I32, Src0: PackedI32, Src1: Mask);
527 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
528 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
529}
530
531std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
532 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
533 auto Lo = B.buildSExtInReg(Res: SgprRB_I32, Op: PackedI32, ImmOp: 16);
534 auto Hi = B.buildAShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
535 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
536}
537
538std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
539 Register RegI32 = Reg;
540 if (MRI.getType(Reg) != I32)
541 RegI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg).getReg(Idx: 0);
542
543 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: RegI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
544 return {RegI32, Hi.getReg(Idx: 0)};
545}
546
547std::pair<Register, Register>
548RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
549 auto [Lo32, Hi32] = unpackAExt(Reg);
550 LLT EltTy = MRI.getType(Reg).getElementType();
551 return {B.buildTrunc(Res: {SgprRB, EltTy}, Op: Lo32).getReg(Idx: 0),
552 B.buildTrunc(Res: {SgprRB, EltTy}, Op: Hi32).getReg(Idx: 0)};
553}
554
555bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
556 Register Lo, Hi;
557 switch (MI.getOpcode()) {
558 case AMDGPU::G_SHL: {
559 auto [Val0, Val1] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
560 auto [Amt0, Amt1] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
561 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
562 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
563 break;
564 }
565 case AMDGPU::G_LSHR: {
566 auto [Val0, Val1] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
567 auto [Amt0, Amt1] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
568 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
569 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
570 break;
571 }
572 case AMDGPU::G_ASHR: {
573 auto [Val0, Val1] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
574 auto [Amt0, Amt1] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
575 Lo = B.buildAShr(Dst: SgprRB_I32, Src0: Val0, Src1: Amt0).getReg(Idx: 0);
576 Hi = B.buildAShr(Dst: SgprRB_I32, Src0: Val1, Src1: Amt1).getReg(Idx: 0);
577 break;
578 }
579 default:
580 reportGISelFailure(
581 MF, MORE, DEBUG_TYPE,
582 Msg: "AMDGPU RegBankLegalize: lowerUnpackBitShift, case not implemented",
583 MI);
584 return false;
585 }
586 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
587 MI.eraseFromParent();
588 return true;
589}
590
591bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
592 Register Lo, Hi;
593 switch (MI.getOpcode()) {
594 case AMDGPU::G_SMIN:
595 case AMDGPU::G_SMAX: {
596 // For signed operations, use sign extension
597 auto [Val0_Lo, Val0_Hi] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
598 auto [Val1_Lo, Val1_Hi] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
599 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
600 .getReg(Idx: 0);
601 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
602 .getReg(Idx: 0);
603 break;
604 }
605 case AMDGPU::G_UMIN:
606 case AMDGPU::G_UMAX: {
607 // For unsigned operations, use zero extension
608 auto [Val0_Lo, Val0_Hi] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
609 auto [Val1_Lo, Val1_Hi] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
610 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
611 .getReg(Idx: 0);
612 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
613 .getReg(Idx: 0);
614 break;
615 }
616 default:
617 reportGISelFailure(
618 MF, MORE, DEBUG_TYPE,
619 Msg: "AMDGPU RegBankLegalize: lowerUnpackMinMax, case not implemented", MI);
620 return false;
621 }
622 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
623 MI.eraseFromParent();
624 return true;
625}
626
627bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
628 auto [Op1Lo, Op1Hi] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
629 auto [Op2Lo, Op2Hi] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
630 auto ResLo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Lo, Op2Lo});
631 auto ResHi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Hi, Op2Hi});
632 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
633 Ops: {ResLo.getReg(Idx: 0), ResHi.getReg(Idx: 0)});
634 MI.eraseFromParent();
635 return true;
636}
637
638bool RegBankLegalizeHelper::lowerSBufToBuf(MachineInstr &MI,
639 WaterfallInfo &WFI) {
640 Register Dst = MI.getOperand(i: 0).getReg();
641 LLT Ty = MRI.getType(Reg: Dst);
642 const RegisterBank *RSrcBank = MRI.getRegBank(Reg: MI.getOperand(i: 1).getReg());
643 unsigned LoadSize = Ty.getSizeInBits();
644 int NumLoads = 1;
645 SmallVector<Register, 4> LoadParts;
646 if (LoadSize == 256 || LoadSize == 512) {
647 NumLoads = LoadSize / 128;
648 Ty = Ty.divide(Factor: NumLoads);
649 }
650 for (int I = 0; I < NumLoads; ++I)
651 LoadParts.emplace_back(Args: MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty}));
652 MachineMemOperand *OrigMMO = *MI.memoperands_begin();
653 const Align Alignment = OrigMMO->getAlign();
654 MachineFunction &MF = B.getMF();
655 Register SOffset;
656 Register VOffset;
657 int64_t ImmOffset = 0;
658 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
659 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
660 const unsigned MemSize = divideCeil(Numerator: OrigMMO->getSize().getValue(), Denominator: NumLoads);
661 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(MMO: OrigMMO, Offset: 0, Size: MemSize);
662 if (MMOOffset != 0)
663 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
664 // If only the offset is divergent, emit a MUBUF buffer load
665 // instead. We can assume that the buffer is unswizzled.
666 Register RSrc = MI.getOperand(i: 1).getReg();
667 Register VIndex = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
668 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
669 unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
670 switch (MI.getOpcode()) {
671 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
672 Opc = G_AMDGPU_BUFFER_LOAD_SBYTE;
673 break;
674 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
675 Opc = G_AMDGPU_BUFFER_LOAD_UBYTE;
676 break;
677 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
678 Opc = G_AMDGPU_BUFFER_LOAD_SSHORT;
679 break;
680 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
681 Opc = G_AMDGPU_BUFFER_LOAD_USHORT;
682 break;
683 default:
684 break;
685 }
686 for (int I = 0; I < NumLoads; ++I) {
687 B.buildInstr(Opcode: Opc)
688 .addDef(RegNo: LoadParts[I]) // vdata
689 .addUse(RegNo: RSrc) // rsrc
690 .addUse(RegNo: VIndex) // vindex
691 .addUse(RegNo: VOffset) // voffset
692 .addUse(RegNo: SOffset) // soffset
693 .addImm(Val: ImmOffset + 16 * I) // offset(imm)
694 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
695 .addImm(Val: 0) // idxen(imm)
696 .addMemOperand(MMO: MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16 * I, Size: MemSize));
697 }
698 if (NumLoads == 1)
699 B.buildCopy(Res: Dst, Op: LoadParts[0]);
700 else
701 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
702 B.setInstr(*MRI.getVRegDef(Reg: LoadParts[0]));
703 if (RSrcBank != SgprRB) {
704 WFI.SgprWaterfallOperandRegs.insert(V: RSrc);
705 WFI.Start = MRI.getVRegDef(Reg: LoadParts.front());
706 WFI.End = std::next(x: MRI.getVRegDef(Reg: LoadParts.back())->getIterator());
707 }
708 MI.eraseFromParent();
709 return true;
710}
711
712static bool isSignedBFE(MachineInstr &MI) {
713 if (GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI))
714 return (GI->is(ID: Intrinsic::amdgcn_sbfe));
715
716 return MI.getOpcode() == AMDGPU::G_SBFX;
717}
718
719bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
720 Register Dst = MI.getOperand(i: 0).getReg();
721 assert(MRI.getType(Dst) == LLT::scalar(64));
722 bool Signed = isSignedBFE(MI);
723 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
724 // Extract bitfield from Src, LSBit is the least-significant bit for the
725 // extraction (field offset) and Width is size of bitfield.
726 Register Src = MI.getOperand(i: FirstOpnd).getReg();
727 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
728 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
729 // Comments are for signed bitfield extract, similar for unsigned. x is sign
730 // bit. s is sign, l is LSB and y are remaining bits of bitfield to extract.
731
732 // Src >> LSBit Hi|Lo: x?????syyyyyyl??? -> xxxx?????syyyyyyl
733 unsigned SHROpc = Signed ? AMDGPU::G_ASHR : AMDGPU::G_LSHR;
734 auto SHRSrc = B.buildInstr(Opc: SHROpc, DstOps: {VgprRB_I64}, SrcOps: {Src, LSBit});
735
736 auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: Width, MRI);
737
738 // Expand to Src >> LSBit << (64 - Width) >> (64 - Width)
739 // << (64 - Width): Hi|Lo: xxxx?????syyyyyyl -> syyyyyyl000000000
740 // >> (64 - Width): Hi|Lo: syyyyyyl000000000 -> ssssssssssyyyyyyl
741 if (!ConstWidth) {
742 auto Amt = B.buildSub(Dst: VgprRB_I32, Src0: B.buildConstant(Res: SgprRB_I32, Val: 64), Src1: Width);
743 auto SignBit = B.buildShl(Dst: VgprRB_I64, Src0: SHRSrc, Src1: Amt);
744 B.buildInstr(Opc: SHROpc, DstOps: {Dst}, SrcOps: {SignBit, Amt});
745 MI.eraseFromParent();
746 return true;
747 }
748
749 uint64_t WidthImm = ConstWidth->Value.getZExtValue();
750 auto UnmergeSHRSrc = B.buildUnmerge(Attrs: VgprRB_I32, Op: SHRSrc);
751 Register SHRSrcLo = UnmergeSHRSrc.getReg(Idx: 0);
752 Register SHRSrcHi = UnmergeSHRSrc.getReg(Idx: 1);
753 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
754 unsigned BFXOpc = Signed ? AMDGPU::G_SBFX : AMDGPU::G_UBFX;
755
756 if (WidthImm <= 32) {
757 // SHRSrc Hi|Lo: ????????|???syyyl -> ????????|ssssyyyl
758 Register Lo = SHRSrcLo;
759 // V_BFE masks its width to 5 bits, so 32 would extract zero bits.
760 if (WidthImm < 32) {
761 Lo =
762 B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcLo, Zero, Width}).getReg(Idx: 0);
763 }
764 MachineInstrBuilder Hi;
765 if (Signed) {
766 // SHRSrc Hi|Lo: ????????|ssssyyyl -> ssssssss|ssssyyyl
767 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: B.buildConstant(Res: VgprRB_I32, Val: 31));
768 } else {
769 // SHRSrc Hi|Lo: ????????|000syyyl -> 00000000|000syyyl
770 Hi = Zero;
771 }
772 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
773 } else {
774 // SHRSrc Hi|Lo: ??????sy|yyyyyyyl -> sssssssy|yyyyyyyl
775 Register Hi = SHRSrcHi;
776 // V_BFE masks its width to 5 bits, so 64 would extract zero bits.
777 if (WidthImm < 64) {
778 auto Amt = B.buildConstant(Res: VgprRB_I32, Val: WidthImm - 32);
779 Hi = B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcHi, Zero, Amt}).getReg(Idx: 0);
780 }
781 B.buildMergeLikeInstr(Res: Dst, Ops: {SHRSrcLo, Hi});
782 }
783
784 MI.eraseFromParent();
785 return true;
786}
787
788bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
789 Register DstReg = MI.getOperand(i: 0).getReg();
790 LLT Ty = MRI.getType(Reg: DstReg);
791 bool Signed = isSignedBFE(MI);
792 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
793 Register Src = MI.getOperand(i: FirstOpnd).getReg();
794 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
795 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
796 // For uniform bit field extract there are 4 available instructions, but
797 // LSBit(field offset) and Width(size of bitfield) need to be packed in S32,
798 // field offset in low and size in high 16 bits.
799
800 // Src1 Hi16|Lo16 = Size|FieldOffset
801 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: maskTrailingOnes<unsigned>(N: 6));
802 auto FieldOffset = B.buildAnd(Dst: SgprRB_I32, Src0: LSBit, Src1: Mask);
803 auto Size = B.buildShl(Dst: SgprRB_I32, Src0: Width, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
804 auto Src1 = B.buildOr(Dst: SgprRB_I32, Src0: FieldOffset, Src1: Size);
805 unsigned Opc32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
806 unsigned Opc64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
807 unsigned Opc = Ty == S32 ? Opc32 : Opc64;
808
809 // Select machine instruction, because of reg class constraining, insert
810 // copies from reg class to reg bank.
811 auto S_BFE = B.buildInstr(Opc, DstOps: {{SgprRB, Ty}},
812 SrcOps: {B.buildCopy(Res: Ty, Op: Src), B.buildCopy(Res: I32, Op: Src1)});
813 constrainSelectedInstRegOperands(I&: *S_BFE, TII: *ST.getInstrInfo(),
814 TRI: *ST.getRegisterInfo(), RBI);
815
816 B.buildCopy(Res: DstReg, Op: S_BFE->getOperand(i: 0).getReg());
817 MI.eraseFromParent();
818 return true;
819}
820
821bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
822 Register Dst = MI.getOperand(i: 0).getReg();
823 LLT DstTy = MRI.getType(Reg: Dst);
824 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
825 LLT Ty = DstTy.divide(Factor: 2);
826 auto Op1 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 1).getReg());
827 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
828 unsigned Opc = MI.getOpcode();
829 auto Flags = MI.getFlags();
830 auto Lo =
831 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 0), Op2.getReg(Idx: 0)}, Flags);
832 auto Hi =
833 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 1), Op2.getReg(Idx: 1)}, Flags);
834 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
835 MI.eraseFromParent();
836 return true;
837}
838
839bool RegBankLegalizeHelper::lowerSplitTo32Mul(MachineInstr &MI) {
840 Register Dst = MI.getOperand(i: 0).getReg();
841 assert(MRI.getType(Dst) == S64);
842 auto Op1 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 1).getReg());
843 auto Op2 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 2).getReg());
844
845 // TODO: G_AMDGPU_MAD_* optimizations for G_MUL divergent S64 operation to
846 // match GlobalISel with old regbankselect.
847 auto Lo = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
848 auto Carry = B.buildUMulH(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
849 auto MulLo0Hi1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 1));
850 auto MulHi0Lo1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 1), Src1: Op2.getReg(Idx: 0));
851 auto Sum = B.buildAdd(Dst: VgprRB_I32, Src0: MulLo0Hi1, Src1: MulHi0Lo1);
852 auto Hi = B.buildAdd(Dst: VgprRB_I32, Src0: Sum, Src1: Carry);
853
854 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
855 MI.eraseFromParent();
856 return true;
857}
858
859bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
860 Register Dst = MI.getOperand(i: 0).getReg();
861 assert(MRI.getType(Dst) == V2S16);
862 unsigned Opc = MI.getOpcode();
863 unsigned NumOps = MI.getNumOperands();
864 auto Flags = MI.getFlags();
865
866 auto [Op1Lo, Op1Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 1).getReg());
867 LLT EltTy = MRI.getType(Reg: Dst).getElementType();
868
869 if (NumOps == 2) {
870 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo}, Flags);
871 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi}, Flags);
872 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
873 MI.eraseFromParent();
874 return true;
875 }
876
877 auto [Op2Lo, Op2Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 2).getReg());
878
879 if (NumOps == 3) {
880 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo}, Flags);
881 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi}, Flags);
882 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
883 MI.eraseFromParent();
884 return true;
885 }
886
887 assert(NumOps == 4);
888 auto [Op3Lo, Op3Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 3).getReg());
889 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo, Op3Lo}, Flags);
890 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi, Op3Hi}, Flags);
891 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
892 MI.eraseFromParent();
893 return true;
894}
895
896bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
897 Register Dst0 = MI.getOperand(i: 0).getReg();
898 Register Dst1 = MI.getOperand(i: 1).getReg();
899 Register Src0 = MI.getOperand(i: 2).getReg();
900 Register Src1 = MI.getOperand(i: 3).getReg();
901 Register Src2 = MI.getOperand(i: 4).getReg();
902
903 const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
904
905 // Keep the multiplication on the SALU.
906 Register DstLo = B.buildMul(Dst: SgprRB_I32, Src0, Src1).getReg(Idx: 0);
907 Register DstHi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
908 if (ST.hasScalarMulHiInsts()) {
909 B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {{DstHi}}, SrcOps: {Src0, Src1});
910 } else {
911 auto VSrc0 = B.buildCopy(Res: VgprRB_I32, Op: Src0);
912 auto VSrc1 = B.buildCopy(Res: VgprRB_I32, Op: Src1);
913 auto MulHi = B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {VgprRB_I32}, SrcOps: {VSrc0, VSrc1});
914 buildReadAnyLane(B, SgprDst: DstHi, VgprSrc: MulHi.getReg(Idx: 0), RBI);
915 }
916
917 // Accumulate and produce the "carry-out" bit.
918
919 // The "carry-out" is defined as bit 64 of the result when computed as a
920 // big integer. For unsigned multiply-add, this matches the usual
921 // definition of carry-out.
922 if (mi_match(R: Src2, MRI, P: MIPatternMatch::m_ZeroInt())) {
923 // No accumulate: result is just the multiplication, carry is 0.
924 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
925 B.buildConstant(Res: Dst1, Val: 0);
926 } else {
927 // Accumulate: add Src2 to the multiplication result with carry chain.
928 Register Src2Lo = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
929 Register Src2Hi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
930 B.buildUnmerge(Res: {Src2Lo, Src2Hi}, Op: Src2);
931
932 auto AddLo = B.buildUAddo(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstLo, Op1: Src2Lo);
933 auto AddHi =
934 B.buildUAdde(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstHi, Op1: Src2Hi, CarryIn: AddLo.getReg(Idx: 1));
935 B.buildMergeLikeInstr(Res: Dst0, Ops: {AddLo.getReg(Idx: 0), AddHi.getReg(Idx: 0)});
936 B.buildCopy(Res: Dst1, Op: AddHi.getReg(Idx: 1));
937 }
938
939 MI.eraseFromParent();
940 return true;
941}
942
943bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
944 Register Dst = MI.getOperand(i: 0).getReg();
945 LLT DstTy = MRI.getType(Reg: Dst);
946 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
947 (DstTy.isPointer() && DstTy.getSizeInBits() == 64));
948 LLT Ty = DstTy.isFloat() ? LLT::float32() : DstTy.divide(Factor: 2);
949 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
950 auto Op3 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 3).getReg());
951 Register Cond = MI.getOperand(i: 1).getReg();
952 Cond = B.buildFreeze(Dst: VccRB_S1, Src: Cond).getReg(Idx: 0);
953 auto Flags = MI.getFlags();
954 auto Lo =
955 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 0), Op1: Op3.getReg(Idx: 0), Flags);
956 auto Hi =
957 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 1), Op1: Op3.getReg(Idx: 1), Flags);
958
959 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
960 MI.eraseFromParent();
961 return true;
962}
963
964bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
965 auto Op1 = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
966 int Amt = MI.getOperand(i: 2).getImm();
967 Register Lo, Hi;
968 // Hi|Lo: s sign bit, ?/x bits changed/not changed by sign-extend
969 if (Amt <= 32) {
970 auto Freeze = B.buildFreeze(Dst: VgprRB_I32, Src: Op1.getReg(Idx: 0));
971 if (Amt == 32) {
972 // Hi|Lo: ????????|sxxxxxxx -> ssssssss|sxxxxxxx
973 Lo = Freeze.getReg(Idx: 0);
974 } else {
975 // Hi|Lo: ????????|???sxxxx -> ssssssss|ssssxxxx
976 Lo = B.buildSExtInReg(Res: VgprRB_I32, Op: Freeze, ImmOp: Amt).getReg(Idx: 0);
977 }
978
979 auto SignExtCst = B.buildConstant(Res: SgprRB_I32, Val: 31);
980 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: SignExtCst).getReg(Idx: 0);
981 } else {
982 // Hi|Lo: ?????sxx|xxxxxxxx -> ssssssxx|xxxxxxxx
983 Lo = Op1.getReg(Idx: 0);
984 Hi = B.buildSExtInReg(Res: VgprRB_I32, Op: Op1.getReg(Idx: 1), ImmOp: Amt - 32).getReg(Idx: 0);
985 }
986
987 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
988 MI.eraseFromParent();
989 return true;
990}
991
992bool RegBankLegalizeHelper::lowerSplitBitCount64To32(MachineInstr &MI) {
993 // Split 64-bit find-first-bit operations into 32-bit halves:
994 // (ffbh hi:lo) -> umin(ffbh(hi), uaddsat(ffbh(lo), 32))
995 // (ffbl hi:lo) -> umin(ffbl(lo), uaddsat(ffbl(hi), 32))
996 // (ctlz_zero_poison hi:lo) -> umin(ffbh(hi), add(ffbh(lo), 32))
997 // (cttz_zero_poison hi:lo) -> umin(ffbl(lo), add(ffbl(hi), 32))
998 unsigned Opc = MI.getOpcode();
999
1000 // FFBH/FFBL return 0xFFFFFFFF on zero input, using uaddsat to avoid
1001 // wrapping. CTLZ/CTTZ guarantee non-zero input (zero_poison), so plain add
1002 // is fine.
1003 unsigned FFBOpc;
1004 unsigned AddOpc;
1005 bool SearchFromMSB;
1006 switch (Opc) {
1007 case AMDGPU::G_AMDGPU_FFBH_U32:
1008 FFBOpc = Opc;
1009 AddOpc = AMDGPU::G_UADDSAT;
1010 SearchFromMSB = true;
1011 break;
1012 case AMDGPU::G_AMDGPU_FFBL_B32:
1013 FFBOpc = Opc;
1014 AddOpc = AMDGPU::G_UADDSAT;
1015 SearchFromMSB = false;
1016 break;
1017 case AMDGPU::G_CTLZ_ZERO_POISON:
1018 FFBOpc = AMDGPU::G_AMDGPU_FFBH_U32;
1019 AddOpc = AMDGPU::G_ADD;
1020 SearchFromMSB = true;
1021 break;
1022 case AMDGPU::G_CTTZ_ZERO_POISON:
1023 FFBOpc = AMDGPU::G_AMDGPU_FFBL_B32;
1024 AddOpc = AMDGPU::G_ADD;
1025 SearchFromMSB = false;
1026 break;
1027 default:
1028 llvm_unreachable("unexpected opcode in lowerSplitBitCount64To32");
1029 }
1030
1031 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1032 Register Lo = Unmerge.getReg(Idx: 0);
1033 Register Hi = Unmerge.getReg(Idx: 1);
1034
1035 // MSB-first (FFBH/CTLZ) searches hi first; LSB-first (FFBL/CTTZ) searches
1036 // lo first. The secondary half adds 32 to account for the primary half's
1037 // width.
1038 auto Primary = B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Hi : Lo});
1039 auto Secondary =
1040 B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Lo : Hi});
1041
1042 auto Adjusted = B.buildInstr(Opc: AddOpc, DstOps: {VgprRB_I32},
1043 SrcOps: {Secondary, B.buildConstant(Res: VgprRB_I32, Val: 32)});
1044 B.buildUMin(Dst: MI.getOperand(i: 0).getReg(), Src0: Primary, Src1: Adjusted);
1045
1046 MI.eraseFromParent();
1047 return true;
1048}
1049
1050bool RegBankLegalizeHelper::lowerExtrVecEltToSel(MachineInstr &MI) {
1051 // Lower extract vector element to a compare-select chain:
1052 // result = elt[0]
1053 // for i in 1..N-1:
1054 // result = (idx == i) ? elt[i] : result
1055 //
1056 // When the index is divergent, each lane may want a different element, so
1057 // we must check every element per lane.
1058 Register Dst = MI.getOperand(i: 0).getReg();
1059 Register Src = MI.getOperand(i: 1).getReg();
1060 Register Idx = MI.getOperand(i: 2).getReg();
1061
1062 LLT VecTy = MRI.getType(Reg: Src);
1063 LLT ScalarTy = VecTy.getScalarType();
1064 unsigned NumElts = VecTy.getNumElements();
1065 MachineRegisterInfo::VRegAttrs VgprRB_EltTy = {.RCOrRB: VgprRB, .Ty: ScalarTy};
1066
1067 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_EltTy, Op: Src);
1068
1069 if (ScalarTy.getSizeInBits() == 32) {
1070 Register PrevSelect = Unmerge.getReg(Idx: 0);
1071 for (unsigned I = 1; I < NumElts; ++I) {
1072 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1073 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1074 PrevSelect =
1075 B.buildSelect(Res: VgprRB_EltTy, Tst: Cmp, Op0: Unmerge.getReg(Idx: I), Op1: PrevSelect)
1076 .getReg(Idx: 0);
1077 }
1078 B.buildCopy(Res: Dst, Op: PrevSelect);
1079 } else if (ScalarTy.getSizeInBits() == 64) {
1080 auto InitUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: 0));
1081 Register PrevLo = InitUnmerge.getReg(Idx: 0);
1082 Register PrevHi = InitUnmerge.getReg(Idx: 1);
1083 for (unsigned I = 1; I < NumElts; ++I) {
1084 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1085 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1086 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: I));
1087 PrevLo = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 0), Op1: PrevLo)
1088 .getReg(Idx: 0);
1089 PrevHi = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 1), Op1: PrevHi)
1090 .getReg(Idx: 0);
1091 }
1092 B.buildMergeLikeInstr(Res: Dst, Ops: {PrevLo, PrevHi});
1093 } else {
1094 reportGISelFailure(
1095 MF, MORE, DEBUG_TYPE,
1096 Msg: "AMDGPU RegBankLegalize: ExtrVecEltToSel unsupported element type", MI);
1097 return false;
1098 }
1099
1100 MI.eraseFromParent();
1101 return true;
1102}
1103
1104bool RegBankLegalizeHelper::lowerExtrVecEltTo32(MachineInstr &MI) {
1105 // Reduce a 64-bit element extract to two 32-bit extracts:
1106 // vec32 = bitcast <N x s64> to <2N x s32>
1107 // lo = vec32[idx * 2]
1108 // hi = vec32[idx * 2 + 1]
1109 // result = merge(lo, hi)
1110 //
1111 // When the index is uniform, all lanes extract the same element, so we can
1112 // just split the s64 extract into two s32 extracts which lower to MOVREL.
1113 Register Dst = MI.getOperand(i: 0).getReg();
1114 Register Src = MI.getOperand(i: 1).getReg();
1115 Register Idx = MI.getOperand(i: 2).getReg();
1116
1117 LLT SrcTy = MRI.getType(Reg: Src);
1118 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1119
1120 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1121 "expected VGPR src and SGPR idx");
1122
1123 auto CastSrc = B.buildBitcast(Dst: {VgprRB, Vec32Ty}, Src);
1124
1125 // Calculate new Lo and Hi indices
1126 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1127 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1128 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1129
1130 auto ExtLo = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxLo);
1131 auto ExtHi = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxHi);
1132
1133 B.buildMergeLikeInstr(Res: Dst, Ops: {ExtLo.getReg(Idx: 0), ExtHi.getReg(Idx: 0)});
1134
1135 MI.eraseFromParent();
1136 return true;
1137}
1138
1139bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
1140 // Lower insert vector element to a compare-select chain:
1141 // for i in 0..N-1:
1142 // result[i] = (idx == i) ? elt : srcVec[i]
1143 // dst = merge(result[0..N-1])
1144 //
1145 // VGPR B64 requires splitting to lo/hi s32 pairs since there is no
1146 // v_cndmask_b64. SGPR B64/B32 and VGPR B32 can be handled natively.
1147 Register Dst = MI.getOperand(i: 0).getReg();
1148 Register Src = MI.getOperand(i: 1).getReg();
1149 Register Elt = MI.getOperand(i: 2).getReg();
1150 Register Idx = MI.getOperand(i: 3).getReg();
1151
1152 LLT VecTy = MRI.getType(Reg: Src);
1153 LLT ScalarTy = VecTy.getScalarType();
1154 unsigned NumElts = VecTy.getNumElements();
1155 const RegisterBank *SrcRB = MRI.getRegBank(Reg: Src);
1156 bool IsSGPR = (SrcRB == SgprRB);
1157 SmallVector<Register, 16> Selects;
1158
1159 if (!IsSGPR && ScalarTy.getSizeInBits() == 64) {
1160 // VGPR B64: split to 32-bit lo/hi since there is no v_cndmask_b64.
1161 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1162 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1163 Register EltLo = EltUnmerge.getReg(Idx: 0);
1164 Register EltHi = EltUnmerge.getReg(Idx: 1);
1165 for (unsigned I = 0; I < NumElts; ++I) {
1166 auto IdxConst = B.buildConstant(Res: VgprRB_I32, Val: I);
1167 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1168 Selects.push_back(
1169 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltLo, Op1: Unmerge.getReg(Idx: 2 * I))
1170 .getReg(Idx: 0));
1171 Selects.push_back(
1172 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltHi, Op1: Unmerge.getReg(Idx: 2 * I + 1))
1173 .getReg(Idx: 0));
1174 }
1175 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * NumElts, ScalarSizeInBits: 32);
1176 auto Vec32 = B.buildBuildVector(Res: {VgprRB, Vec32Ty}, Ops: Selects);
1177 B.buildBitcast(Dst, Src: Vec32);
1178 } else if (ScalarTy.getSizeInBits() == 32 || ScalarTy.getSizeInBits() == 64) {
1179 // B32 (any bank) and SGPR B64: element-wise select at native width.
1180 MachineRegisterInfo::VRegAttrs SrcRB_EltTy = {.RCOrRB: SrcRB, .Ty: ScalarTy};
1181 MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_I32 : VccRB_S1;
1182 auto Unmerge = B.buildUnmerge(Attrs: SrcRB_EltTy, Op: Src);
1183 for (unsigned I = 0; I < NumElts; ++I) {
1184 auto IdxConst = B.buildConstant(Res: SgprRB_I32, Val: I);
1185 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpTy, Op0: Idx, Op1: IdxConst);
1186 Selects.push_back(
1187 Elt: B.buildSelect(Res: SrcRB_EltTy, Tst: Cmp, Op0: Elt, Op1: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
1188 }
1189 B.buildMergeLikeInstr(Res: Dst, Ops: Selects);
1190 } else {
1191 reportGISelFailure(
1192 MF, MORE, DEBUG_TYPE,
1193 Msg: "AMDGPU RegBankLegalize: InsVecEltToSel unsupported element type", MI);
1194 return false;
1195 }
1196
1197 MI.eraseFromParent();
1198 return true;
1199}
1200
1201bool RegBankLegalizeHelper::lowerInsVecEltTo32(MachineInstr &MI) {
1202 // Reduce a 64-bit element insert to two 32-bit inserts:
1203 // vec32 = bitcast <N x s64> to <2N x s32>
1204 // lo, hi = unmerge elt
1205 // vec32[idx * 2] = lo
1206 // vec32[idx * 2 + 1] = hi
1207 // dst = bitcast <2N x s32> to <N x s64>
1208 //
1209 // When the index is uniform, all lanes insert at the same position, so we
1210 // can split the s64 insert into two s32 inserts which lower to MOVREL/GPRIDX.
1211 Register Dst = MI.getOperand(i: 0).getReg();
1212 Register Src = MI.getOperand(i: 1).getReg();
1213 Register Elt = MI.getOperand(i: 2).getReg();
1214 Register Idx = MI.getOperand(i: 3).getReg();
1215
1216 LLT SrcTy = MRI.getType(Reg: Src);
1217 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1218
1219 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1220 "expected VGPR src and SGPR idx");
1221
1222 MachineRegisterInfo::VRegAttrs VgprRB_Vec32Ty = {.RCOrRB: VgprRB, .Ty: Vec32Ty};
1223
1224 auto CastSrc = B.buildBitcast(Dst: VgprRB_Vec32Ty, Src);
1225 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1226
1227 // Calculate new Lo and Hi indices
1228 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1229 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1230 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1231
1232 auto InsLo = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: CastSrc,
1233 Elt: EltUnmerge.getReg(Idx: 0), Idx: IdxLo);
1234 auto InsHi = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: InsLo,
1235 Elt: EltUnmerge.getReg(Idx: 1), Idx: IdxHi);
1236
1237 B.buildBitcast(Dst, Src: InsHi);
1238
1239 MI.eraseFromParent();
1240 return true;
1241}
1242
1243bool RegBankLegalizeHelper::lowerAbsToNegMax(MachineInstr &MI) {
1244 // Lower divergent G_ABS to smax(x, 0 - x) in the VGPR bank:
1245 // zero = 0
1246 // neg = G_SUB zero, x
1247 // dst = G_SMAX x, neg
1248 //
1249 // There is no integer v_abs instruction on AMDGPU, so divergent G_ABS is
1250 // expanded to this sub/smax pair.
1251 Register DstReg = MI.getOperand(i: 0).getReg();
1252 Register SrcReg = MI.getOperand(i: 1).getReg();
1253 LLT Ty = MRI.getType(Reg: DstReg);
1254
1255 Register Zero;
1256 if (Ty == V2S16) {
1257 // buildConstant cannot produce a V2S16 directly; pack two S16 zeros.
1258 Register Zero16 = B.buildConstant(Res: {VgprRB, I16}, Val: 0).getReg(Idx: 0);
1259 Zero = B.buildBuildVector(Res: {VgprRB, Ty}, Ops: {Zero16, Zero16}).getReg(Idx: 0);
1260 } else {
1261 assert((Ty == S32 || Ty == S16) && "unexpected type for AbsToNegMax");
1262 Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0).getReg(Idx: 0);
1263 }
1264
1265 auto Neg = B.buildSub(Dst: {VgprRB, Ty}, Src0: Zero, Src1: SrcReg);
1266 B.buildSMax(Dst: DstReg, Src0: SrcReg, Src1: Neg);
1267 MI.eraseFromParent();
1268 return true;
1269}
1270
1271bool RegBankLegalizeHelper::lowerAbsToS32(MachineInstr &MI) {
1272 // Lower uniform V2S16 abs by unpacking the values to two separate SGPR
1273 // registers and re-emitting G_ABS on each:
1274 // packed = bitcast <2 x s16> src to s32
1275 // lo = sext_inreg packed, 16
1276 // hi = ashr packed, 16
1277 // dst = build_vector_trunc G_ABS(lo), G_ABS(hi)
1278 //
1279 // SALU only has s_abs_i32, with no direct uniform V2S16 abs. The
1280 // re-emitted G_ABS(SgprRB, S32) selects to s_abs_i32 on each value.
1281 auto Bitcast = B.buildBitcast(Dst: {SgprRB_I32}, Src: MI.getOperand(i: 1).getReg());
1282 auto SextInReg = B.buildSExtInReg(Res: {SgprRB_I32}, Op: Bitcast, ImmOp: 16);
1283 auto ShiftHi =
1284 B.buildAShr(Dst: {SgprRB_I32}, Src0: Bitcast, Src1: B.buildConstant(Res: {SgprRB_I32}, Val: 16));
1285
1286 auto AbsLo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {SextInReg});
1287 auto AbsHi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {ShiftHi});
1288 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
1289 Ops: {AbsLo.getReg(Idx: 0), AbsHi.getReg(Idx: 0)});
1290
1291 MI.eraseFromParent();
1292 return true;
1293}
1294
1295// Ported from SITargetLowering::lowerSET_ROUNDING in SIISelLowering.cpp.
1296// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1297bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
1298 Register NewMode = MI.getOperand(i: 0).getReg();
1299
1300 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
1301 // hardware MODE.fp_round values.
1302 if (auto ConstMode = getIConstantVRegValWithLookThrough(VReg: NewMode, MRI)) {
1303 uint32_t ClampedVal = std::min(
1304 a: static_cast<uint32_t>(ConstMode->Value.getZExtValue()),
1305 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
1306 uint32_t DecodedVal = AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal);
1307 NewMode = B.buildConstant(Res: SgprRB_I32, Val: DecodedVal).getReg(Idx: 0);
1308 } else {
1309 // If we know the input can only be one of the supported standard modes in
1310 // the range 0-3, we can use a simplified mapping to hardware values.
1311 KnownBits Known = VT->getKnownBits(R: NewMode);
1312 const bool UseReducedTable = Known.countMinLeadingZeros() >= 30;
1313 // The supported standard values are 0-3. The extended values start at 8. We
1314 // need to offset by 4 if the value is in the extended range.
1315
1316 if (UseReducedTable) {
1317 // Truncate to the low 32-bits.
1318 auto BitTable = B.buildConstant(
1319 Res: SgprRB_I32, Val: AMDGPU::FltRoundToHWConversionTable & 0xffff);
1320
1321 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1322 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: NewMode, Src1: Two);
1323
1324 NewMode =
1325 B.buildLShr(Dst: SgprRB_I32, Src0: BitTable, Src1: RoundModeTimesNumBits).getReg(Idx: 0);
1326
1327 // TODO: A demanded-bits simplification on the setreg source here could
1328 // likely reduce the table extracted bits into inline immediates.
1329 } else {
1330 // table_index = umin(value, value - 4)
1331 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
1332 auto NegFour = B.buildConstant(Res: SgprRB_I32, Val: -4);
1333 auto OffsetEnum = B.buildAdd(Dst: SgprRB_I32, Src0: NewMode, Src1: NegFour);
1334 auto IndexVal = B.buildUMin(Dst: SgprRB_I32, Src0: NewMode, Src1: OffsetEnum);
1335
1336 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1337 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: IndexVal, Src1: Two);
1338
1339 auto BitTable =
1340 B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundToHWConversionTable);
1341 auto TableValue =
1342 B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1343 // No need to mask out the high bits since the setreg will ignore them
1344 // anyway.
1345 NewMode = B.buildTrunc(Res: SgprRB_I32, Op: TableValue).getReg(Idx: 0);
1346 }
1347 }
1348
1349 // N.B. The setreg will be later folded into s_round_mode on supported
1350 // targets.
1351 uint32_t BothRoundHwReg =
1352 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1353 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
1354 /*HasSideEffects=*/true, /*isConvergent=*/false)
1355 .addImm(Val: static_cast<int16_t>(BothRoundHwReg))
1356 .addReg(RegNo: NewMode);
1357
1358 MI.eraseFromParent();
1359 return true;
1360}
1361
1362// Ported from SITargetLowering::lowerGET_ROUNDING in SIISelLowering.cpp.
1363// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1364bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
1365 Register Dst = MI.getOperand(i: 0).getReg();
1366
1367 uint32_t BothRoundHwReg =
1368 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1369 auto GetReg =
1370 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {SgprRB_I32},
1371 /*HasSideEffects=*/true, /*isConvergent=*/false)
1372 .addImm(Val: BothRoundHwReg);
1373
1374 // There are two rounding modes, one for f32 and one for f64/f16. We only
1375 // report in the standard value range if both are the same.
1376 //
1377 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
1378 // ties away from zero is not supported, and the other values are rotated by
1379 // 1.
1380 //
1381 // If the two rounding modes are not the same, report a target defined value.
1382
1383 // Mode register rounding mode fields:
1384 //
1385 // [1:0] Single-precision round mode.
1386 // [3:2] Double/Half-precision round mode.
1387 //
1388 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
1389 //
1390 // Hardware Spec
1391 // Toward-0 3 0
1392 // Nearest Even 0 1
1393 // +Inf 1 2
1394 // -Inf 2 3
1395 // NearestAway0 N/A 4
1396 //
1397 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
1398 // table we can index by the raw hardware mode.
1399 //
1400 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
1401 auto BitTable = B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundConversionTable);
1402
1403 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1404 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: GetReg, Src1: Two);
1405
1406 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
1407 // knew only one mode was demanded.
1408 auto TableValue = B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1409 auto TruncTable = B.buildTrunc(Res: SgprRB_I32, Op: TableValue);
1410
1411 auto EntryMask = B.buildConstant(Res: SgprRB_I32, Val: 0xf);
1412 auto TableEntry = B.buildAnd(Dst: SgprRB_I32, Src0: TruncTable, Src1: EntryMask);
1413
1414 // There's a gap in the 4-bit encoded table and actual enum values, so offset
1415 // if it's an extended value.
1416 auto Four = B.buildConstant(Res: SgprRB_I32, Val: 4);
1417 auto EnumOffset = B.buildAdd(Dst: SgprRB_I32, Src0: TableEntry, Src1: Four);
1418 auto IsStandardMode =
1419 B.buildICmp(Pred: CmpInst::ICMP_ULT, Res: SgprRB_I32, Op0: TableEntry, Op1: Four);
1420 B.buildSelect(Res: Dst, Tst: IsStandardMode, Op0: TableEntry, Op1: EnumOffset);
1421
1422 MI.eraseFromParent();
1423 return true;
1424}
1425
1426bool RegBankLegalizeHelper::lower(MachineInstr &MI,
1427 const RegBankLLTMapping &Mapping,
1428 WaterfallInfo &WFI) {
1429
1430 switch (Mapping.LoweringMethod) {
1431 case DoNotLower:
1432 break;
1433 case VccExtToSel:
1434 return lowerVccExtToSel(MI);
1435 case UniExtToSel: {
1436 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1437 auto True = B.buildConstant(Res: {SgprRB, Ty},
1438 Val: MI.getOpcode() == AMDGPU::G_SEXT ? -1 : 1);
1439 auto False = B.buildConstant(Res: {SgprRB, Ty}, Val: 0);
1440 // Input to G_{Z|S}EXT is 'Legalizer legal' S1. Most common case is compare.
1441 // We are making select here. S1 cond was already 'any-extended to S32' +
1442 // 'AND with 1 to clean high bits' by Sgpr32AExtBoolInReg.
1443 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: MI.getOperand(i: 1).getReg(), Op0: True,
1444 Op1: False);
1445 MI.eraseFromParent();
1446 return true;
1447 }
1448 case UnpackBitShift:
1449 return lowerUnpackBitShift(MI);
1450 case UnpackMinMax:
1451 return lowerUnpackMinMax(MI);
1452 case ScalarizeToS16:
1453 return lowerSplitTo16(MI);
1454 case Ext32To64: {
1455 const RegisterBank *RB = MRI.getRegBank(Reg: MI.getOperand(i: 0).getReg());
1456 MachineInstrBuilder Hi;
1457 switch (MI.getOpcode()) {
1458 case AMDGPU::G_ZEXT: {
1459 Hi = B.buildConstant(Res: {RB, I32}, Val: 0);
1460 break;
1461 }
1462 case AMDGPU::G_SEXT: {
1463 // Replicate sign bit from 32-bit extended part.
1464 auto ShiftAmt = B.buildConstant(Res: {RB, I32}, Val: 31);
1465 Hi = B.buildAShr(Dst: {RB, MRI.getType(Reg: MI.getOperand(i: 1).getReg())},
1466 Src0: MI.getOperand(i: 1).getReg(), Src1: ShiftAmt);
1467 break;
1468 }
1469 case AMDGPU::G_ANYEXT: {
1470 Hi = B.buildUndef(Res: {RB, I32});
1471 break;
1472 }
1473 default:
1474 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1475 Msg: "AMDGPU RegBankLegalize: Ext32To64, unsuported opcode",
1476 MI);
1477 return false;
1478 }
1479
1480 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(),
1481 Ops: {MI.getOperand(i: 1).getReg(), Hi});
1482 MI.eraseFromParent();
1483 return true;
1484 }
1485 case UniCstExt: {
1486 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1487 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: ConstVal);
1488
1489 MI.eraseFromParent();
1490 return true;
1491 }
1492 case VgprToVccCopy: {
1493 Register Src = MI.getOperand(i: 1).getReg();
1494 LLT Ty = MRI.getType(Reg: Src);
1495 // Take lowest bit from each lane and put it in lane mask.
1496 // Lowering via compare, but we need to clean high bits first as compare
1497 // compares all bits in register.
1498 Register BoolSrc = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
1499 if (Ty == S64) {
1500 auto Src64 = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1501 auto One = B.buildConstant(Res: VgprRB_I32, Val: 1);
1502 auto AndLo = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 0), Src1: One);
1503 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
1504 auto AndHi = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 1), Src1: Zero);
1505 B.buildMergeLikeInstr(Res: BoolSrc, Ops: {AndLo, AndHi});
1506 } else {
1507 assert(Ty == S32 || Ty == S16);
1508 auto One = B.buildConstant(Res: {VgprRB, Ty}, Val: 1);
1509 B.buildAnd(Dst: BoolSrc, Src0: Src, Src1: One);
1510 }
1511 auto Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
1512 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 0).getReg(), Op0: BoolSrc, Op1: Zero);
1513 MI.eraseFromParent();
1514 return true;
1515 }
1516 case V_BFE:
1517 return lowerV_BFE(MI);
1518 case S_BFE:
1519 return lowerS_BFE(MI);
1520 case UniMAD64:
1521 return lowerUniMAD64(MI);
1522 case UniMul64: {
1523 B.buildMul(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2));
1524 MI.eraseFromParent();
1525 return true;
1526 }
1527 case DivSMulToMAD: {
1528 auto Op1 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 1));
1529 auto Op2 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 2));
1530 auto Zero = B.buildConstant(Res: VgprRB_I64, Val: 0);
1531
1532 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_AMDGPU_S_MUL_U64_U32
1533 ? AMDGPU::G_AMDGPU_MAD_U64_U32
1534 : AMDGPU::G_AMDGPU_MAD_I64_I32;
1535
1536 B.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0).getReg(), SgprRB_I32},
1537 SrcOps: {Op1, Op2, Zero});
1538 MI.eraseFromParent();
1539 return true;
1540 }
1541 case SplitTo32:
1542 return lowerSplitTo32(MI);
1543 case SplitTo32Mul:
1544 return lowerSplitTo32Mul(MI);
1545 case SplitTo32Select:
1546 return lowerSplitTo32Select(MI);
1547 case SplitTo32SExtInReg:
1548 return lowerSplitTo32SExtInReg(MI);
1549 case CtPop64To32: {
1550 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1551 auto LoPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 0));
1552 auto HiPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 1));
1553 // Max popcount of two 32-bit values is 64, so this add cannot overflow.
1554 B.buildAdd(Dst: MI.getOperand(i: 0).getReg(), Src0: LoPopCnt, Src1: HiPopCnt,
1555 Flags: MachineInstr::NoSWrap | MachineInstr::NoUWrap);
1556
1557 MI.eraseFromParent();
1558 break;
1559 }
1560 case S_BUF_to_BUF:
1561 return lowerSBufToBuf(MI, WFI);
1562 case SplitLoad: {
1563 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1564 unsigned Size = DstTy.getSizeInBits();
1565 // Even split to 128-bit loads
1566 if (Size > 128) {
1567 LLT B128;
1568 if (DstTy.isVector()) {
1569 LLT EltTy = DstTy.getElementType();
1570 B128 = LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1571 } else {
1572 B128 = LLT::integer(SizeInBits: 128);
1573 }
1574 if (Size / 128 == 2)
1575 splitLoad(MI, LLTBreakdown: {B128, B128});
1576 else if (Size / 128 == 4)
1577 splitLoad(MI, LLTBreakdown: {B128, B128, B128, B128});
1578 else {
1579 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1580 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1581 MI);
1582 return false;
1583 }
1584 }
1585 // 64 and 32 bit load
1586 else if (DstTy == S96)
1587 splitLoad(MI, LLTBreakdown: {S64, S32}, MergeTy: S32);
1588 else if (DstTy == V3S32)
1589 splitLoad(MI, LLTBreakdown: {V2S32, S32}, MergeTy: S32);
1590 else if (DstTy == V6S16)
1591 splitLoad(MI, LLTBreakdown: {V4S16, V2S16}, MergeTy: V2S16);
1592 else {
1593 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1594 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1595 MI);
1596 return false;
1597 }
1598 return true;
1599 }
1600 case DynStackAlloc: {
1601 const auto &TFI = *ST.getFrameLowering();
1602 // Guard in case the stack growth direction ever changes with scratch
1603 // instructions.
1604 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1605 "Stack grows upwards for AMDGPU");
1606
1607 Register Dst = MI.getOperand(i: 0).getReg();
1608 Register AllocSize = MI.getOperand(i: 1).getReg();
1609 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1610
1611 // Erase before building new instrs to avoid hitting multiple Dst assert
1612 // with CSE.
1613 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
1614 MI.eraseFromParent();
1615
1616 if (MRI.getRegBank(Reg: AllocSize) != SgprRB) {
1617 auto WaveReduction =
1618 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {SgprRB_I32})
1619 .addUse(RegNo: AllocSize)
1620 .addImm(Val: 0);
1621 AllocSize = WaveReduction.getReg(Idx: 0);
1622 }
1623
1624 LLT PtrTy = MRI.getType(Reg: Dst);
1625 assert(PtrTy.getSizeInBits() == 32 &&
1626 "Expected 32-bit pointer for stack allocation");
1627 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1628 Register SPReg = Info->getStackPtrOffsetReg();
1629
1630 // When using flat-scratch, the stack offset is unscaled.
1631 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1632 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1633
1634 Register AdjustedSize = AllocSize;
1635 if (!HasFlatScratch) {
1636 auto WaveSize = B.buildConstant(Res: SgprRB_I32, Val: WavefrontSizeLog2);
1637 AdjustedSize = B.buildShl(Dst: SgprRB_I32, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1638 }
1639 if (Alignment > TFI.getStackAlign()) {
1640 const uint64_t EffectiveAlignment =
1641 Alignment.value() << (HasFlatScratch ? 0 : WavefrontSizeLog2);
1642 auto OldSP = B.buildCopy(Res: {SgprRB, PtrTy}, Op: SPReg);
1643 auto Tmp1 =
1644 B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: OldSP,
1645 Op1: B.buildConstant(Res: SgprRB_I32, Val: EffectiveAlignment - 1));
1646 uint64_t Mask = maskTrailingZeros<uint64_t>(N: Log2_64(Value: EffectiveAlignment));
1647 B.buildPtrMask(Res: Dst, Op0: Tmp1, Op1: B.buildConstant(Res: SgprRB_I32, Val: Mask));
1648 } else {
1649 B.buildCopy(Res: Dst, Op: SPReg);
1650 }
1651 auto PtrAdd = B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: Dst, Op1: AdjustedSize);
1652 B.buildCopy(Res: SPReg, Op: PtrAdd);
1653 return true;
1654 }
1655 case WidenLoad: {
1656 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1657 if (DstTy == S96)
1658 widenLoad(MI, WideTy: S128);
1659 else if (DstTy == V3S32)
1660 widenLoad(MI, WideTy: V4S32, MergeTy: S32);
1661 else if (DstTy == V6S16)
1662 widenLoad(MI, WideTy: V8S16, MergeTy: V2S16);
1663 else {
1664 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1665 Msg: "AMDGPU RegBankLegalize: WidenLoad, unsuported type",
1666 MI);
1667 return false;
1668 }
1669 return true;
1670 }
1671 case UnpackAExt:
1672 return lowerUnpackAExt(MI);
1673 case WidenMMOToS32:
1674 return widenMMOToS32(MI&: cast<GAnyLoad>(Val&: MI));
1675 case VerifyAllSgpr: {
1676 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1677 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1678 }));
1679 return true;
1680 }
1681 case ApplyAllVgpr: {
1682 assert(llvm::all_of(MI.defs(), [&](const MachineOperand &Op) {
1683 return MRI.getRegBankOrNull(Op.getReg()) == VgprRB;
1684 }));
1685 B.setInstrAndDebugLoc(MI);
1686 for (unsigned i = MI.getNumDefs(); i < MI.getNumOperands(); ++i) {
1687 MachineOperand &Op = MI.getOperand(i);
1688 if (!Op.isReg())
1689 continue;
1690 Register Reg = Op.getReg();
1691 if (MRI.getRegBank(Reg) != VgprRB) {
1692 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
1693 Op.setReg(Copy.getReg(Idx: 0));
1694 }
1695 }
1696 return true;
1697 }
1698 case UnmergeToShiftTrunc: {
1699 GUnmerge *Unmerge = dyn_cast<GUnmerge>(Val: &MI);
1700 LLT Ty = MRI.getType(Reg: Unmerge->getSourceReg());
1701 if (Ty.getSizeInBits() % 32 != 0) {
1702 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1703 Msg: "AMDGPU RegBankLegalize: unmerge not multiple of 32",
1704 MI);
1705 return false;
1706 }
1707
1708 B.setInstrAndDebugLoc(MI);
1709 if (Ty.getSizeInBits() > 32) {
1710 auto UnmergeI32 = B.buildUnmerge(Attrs: SgprRB_I32, Op: Unmerge->getSourceReg());
1711 unsigned NumDefs = UnmergeI32->getNumDefs();
1712 for (unsigned i = 0; i < NumDefs; ++i) {
1713 auto [Dst0I32, Dst1I32] = unpackAExt(Reg: UnmergeI32.getReg(Idx: i));
1714 B.buildTrunc(Res: MI.getOperand(i: i * 2).getReg(), Op: Dst0I32);
1715 B.buildTrunc(Res: MI.getOperand(i: i * 2 + 1).getReg(), Op: Dst1I32);
1716 }
1717 } else {
1718 auto [Dst0I32, Dst1I32] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
1719 B.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Dst0I32);
1720 B.buildTrunc(Res: MI.getOperand(i: 1).getReg(), Op: Dst1I32);
1721 }
1722
1723 MI.eraseFromParent();
1724 return true;
1725 }
1726 case AextToS32InIncomingBlockGPHI: {
1727 Register Dst = MI.getOperand(i: 0).getReg();
1728 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
1729 B.setInsertPt(MBB&: *MI.getParent(), II: MI.getParent()->getFirstNonPHI());
1730 MI.getOperand(i: 0).setReg(NewDst);
1731 B.buildTrunc(Res: Dst, Op: NewDst);
1732
1733 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1734 Register UseReg = MI.getOperand(i).getReg();
1735
1736 auto DefMI = MRI.getVRegDef(Reg: UseReg)->getIterator();
1737 MachineBasicBlock *DefMBB = DefMI->getParent();
1738
1739 B.setInsertPt(MBB&: *DefMBB, II: DefMBB->SkipPHIsAndLabels(I: std::next(x: DefMI)));
1740
1741 auto NewUse = B.buildAnyExt(Res: SgprRB_I32, Op: UseReg);
1742 MI.getOperand(i).setReg(NewUse.getReg(Idx: 0));
1743 }
1744 break;
1745 }
1746 case VerifyAllSgprGPHI: {
1747 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1748 if (Op.isMBB())
1749 return true;
1750 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1751 }));
1752 return true;
1753 }
1754 case VerifyAllSgprOrVgprGPHI: {
1755 assert(MRI.getRegBankOrNull(MI.getOperand(0).getReg()) == VgprRB);
1756 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1757 if (Op.isMBB())
1758 return true;
1759 const RegisterBank *RB = MRI.getRegBankOrNull(Op.getReg());
1760 return RB == VgprRB || RB == SgprRB;
1761 }));
1762 return true;
1763 }
1764 case ApplyINTRIN_IMAGE: {
1765 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
1766 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
1767 assert(RSrcIntrin && RSrcIntrin->IsImage);
1768 // The reported argument index is relative to the IR intrinsic call
1769 // arguments, so shift by the number of defs and the intrinsic ID.
1770 unsigned RsrcIdx = RSrcIntrin->RsrcArg + MI.getNumExplicitDefs() + 1;
1771 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1772 }
1773 case ApplyBVH_INTERSECT_RAY: {
1774 // Rsrc is the last register operand. Base BVH trails an A16 immediate
1775 // after rsrc; dual/BVH8 do not. Scan backwards for the last virtual
1776 // register.
1777 unsigned RsrcIdx = MI.getNumOperands();
1778 while (RsrcIdx-- > MI.getNumExplicitDefs()) {
1779 const MachineOperand &Op = MI.getOperand(i: RsrcIdx);
1780 if (Op.isReg() && Op.getReg().isVirtual())
1781 break;
1782 }
1783 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1784 }
1785 case SplitBitCount64To32:
1786 return lowerSplitBitCount64To32(MI);
1787 case ExtrVecEltToSel:
1788 return lowerExtrVecEltToSel(MI);
1789 case ExtrVecEltTo32:
1790 return lowerExtrVecEltTo32(MI);
1791 case InsVecEltToSel:
1792 return lowerInsVecEltToSel(MI);
1793 case InsVecEltTo32:
1794 return lowerInsVecEltTo32(MI);
1795 case AbsToNegMax:
1796 return lowerAbsToNegMax(MI);
1797 case AbsToS32:
1798 return lowerAbsToS32(MI);
1799 case DeletePrefetch:
1800 MI.eraseFromParent();
1801 return true;
1802 case LowerSetRounding:
1803 return lowerSetRounding(MI);
1804 case LowerGetRounding:
1805 return lowerGetRounding(MI);
1806 }
1807
1808 return true;
1809}
1810
1811LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
1812 switch (ID) {
1813 case Vcc:
1814 case UniInVcc:
1815 return LLT::scalar(SizeInBits: 1);
1816 case Sgpr16:
1817 case Vgpr16:
1818 case UniInVgprS16:
1819 return LLT::scalar(SizeInBits: 16);
1820 case Sgpr32:
1821 case Sgpr32_WF:
1822 case Sgpr32Trunc:
1823 case Sgpr32AExt:
1824 case Sgpr32AExtBoolInReg:
1825 case Sgpr32SExt:
1826 case Sgpr32ZExt:
1827 case UniInVgprS32:
1828 case Sgpr32ToVgprDst:
1829 case Vgpr32:
1830 case Vgpr32AExt:
1831 case Vgpr32SExt:
1832 case Vgpr32ZExt:
1833 return LLT::scalar(SizeInBits: 32);
1834 case Sgpr64:
1835 case Vgpr64:
1836 case UniInVgprS64:
1837 case Sgpr64ToVgprDst:
1838 return LLT::scalar(SizeInBits: 64);
1839 case Sgpr128:
1840 case Vgpr128:
1841 return LLT::scalar(SizeInBits: 128);
1842 case SgprP0:
1843 case SgprP0Call_WF:
1844 case VgprP0:
1845 return LLT::pointer(AddressSpace: 0, SizeInBits: 64);
1846 case SgprP1:
1847 case VgprP1:
1848 return LLT::pointer(AddressSpace: 1, SizeInBits: 64);
1849 case SgprP2:
1850 case VgprP2:
1851 return LLT::pointer(AddressSpace: 2, SizeInBits: 32);
1852 case SgprP3:
1853 case VgprP3:
1854 return LLT::pointer(AddressSpace: 3, SizeInBits: 32);
1855 case SgprP4:
1856 case SgprP4Call_WF:
1857 case VgprP4:
1858 return LLT::pointer(AddressSpace: 4, SizeInBits: 64);
1859 case SgprP5:
1860 case VgprP5:
1861 return LLT::pointer(AddressSpace: 5, SizeInBits: 32);
1862 case SgprP6:
1863 return LLT::pointer(AddressSpace: 6, SizeInBits: 32);
1864 case SgprP8:
1865 return LLT::pointer(AddressSpace: 8, SizeInBits: 128);
1866 case SgprV2S16:
1867 case VgprV2S16:
1868 case UniInVgprV2S16:
1869 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
1870 case SgprV2S32:
1871 case VgprV2S32:
1872 case UniInVgprV2S32:
1873 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
1874 case VgprV3S32:
1875 case UniInVgprV3S32:
1876 return LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
1877 case VgprV4S16:
1878 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
1879 case VgprV8S16:
1880 case UniInVgprV8S16:
1881 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
1882 case VgprV16S16:
1883 case UniInVgprV16S16:
1884 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
1885 case SgprV4S32:
1886 case SgprV4S32_WF:
1887 case SgprV4S32_ReadFirstLane:
1888 case VgprV4S32:
1889 case UniInVgprV4S32:
1890 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
1891 case VgprV8S32:
1892 case UniInVgprV8S32:
1893 case SgprV8S32_ReadFirstLane:
1894 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
1895 case VgprV2S64:
1896 case UniInVgprV2S64:
1897 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1898 case VgprV6S32:
1899 case UniInVgprV6S32:
1900 return LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32);
1901 case VgprV16S32:
1902 case UniInVgprV16S32:
1903 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
1904 case VgprV32S16:
1905 case UniInVgprV32S16:
1906 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
1907 case VgprV32S32:
1908 case UniInVgprV32S32:
1909 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32);
1910 default:
1911 return LLT();
1912 }
1913}
1914
1915LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
1916 switch (ID) {
1917 case SgprB32:
1918 case VgprB32:
1919 case SgprB32_M0:
1920 case SgprB32_ReadFirstLane:
1921 case UniInVgprB32:
1922 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
1923 isAnyPtr(Ty, Width: 32))
1924 return Ty;
1925 return LLT();
1926 case SgprPtr32:
1927 case VgprPtr32:
1928 return isAnyPtr(Ty, Width: 32) ? Ty : LLT();
1929 case SgprPtr64:
1930 case VgprPtr64:
1931 return isAnyPtr(Ty, Width: 64) ? Ty : LLT();
1932 case SgprPtr128:
1933 case VgprPtr128:
1934 return isAnyPtr(Ty, Width: 128) ? Ty : LLT();
1935 case SgprB64:
1936 case VgprB64:
1937 case SgprB64_ReadFirstLane:
1938 case UniInVgprB64:
1939 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
1940 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
1941 return Ty;
1942 return LLT();
1943 case SgprB96:
1944 case VgprB96:
1945 case UniInVgprB96:
1946 if (Ty == LLT::scalar(SizeInBits: 96) || Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) ||
1947 Ty == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 16))
1948 return Ty;
1949 return LLT();
1950 case SgprB128:
1951 case VgprB128:
1952 case UniInVgprB128:
1953 if (Ty.getSizeInBits() == 128)
1954 return Ty;
1955 return LLT();
1956 case VgprB160:
1957 case UniInVgprB160:
1958 if (Ty.getSizeInBits() == 160)
1959 return Ty;
1960 return LLT();
1961 case SgprB256:
1962 case VgprB256:
1963 case UniInVgprB256:
1964 if (Ty.getSizeInBits() == 256)
1965 return Ty;
1966 return LLT();
1967 case SgprB512:
1968 case VgprB512:
1969 case UniInVgprB512:
1970 if (Ty.getSizeInBits() == 512)
1971 return Ty;
1972 return LLT();
1973 case SgprBRC: {
1974 const SIRegisterInfo *TRI =
1975 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1976 unsigned LLTSize = Ty.getSizeInBits();
1977 if (LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize))
1978 return Ty;
1979 return LLT();
1980 }
1981 case VgprBRC: {
1982 const SIRegisterInfo *TRI =
1983 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1984 if (TRI->getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()))
1985 return Ty;
1986 return LLT();
1987 }
1988 default:
1989 return LLT();
1990 }
1991}
1992
1993const RegisterBank *
1994RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
1995 switch (ID) {
1996 case Vcc:
1997 return VccRB;
1998 case Sgpr16:
1999 case Sgpr32:
2000 case Sgpr32_WF:
2001 case Sgpr64:
2002 case Sgpr128:
2003 case SgprP0:
2004 case SgprP0Call_WF:
2005 case SgprP1:
2006 case SgprP2:
2007 case SgprP3:
2008 case SgprP4:
2009 case SgprP4Call_WF:
2010 case SgprP5:
2011 case SgprP6:
2012 case SgprP8:
2013 case SgprPtr32:
2014 case SgprPtr64:
2015 case SgprPtr128:
2016 case SgprV2S16:
2017 case SgprV2S32:
2018 case SgprV4S32:
2019 case SgprV4S32_WF:
2020 case SgprV4S32_ReadFirstLane:
2021 case SgprV8S32_ReadFirstLane:
2022 case SgprB32:
2023 case SgprB64:
2024 case SgprB96:
2025 case SgprB128:
2026 case SgprB256:
2027 case SgprB512:
2028 case SgprBRC:
2029 case UniInVcc:
2030 case UniInVgprS16:
2031 case UniInVgprS32:
2032 case UniInVgprS64:
2033 case UniInVgprV2S16:
2034 case UniInVgprV2S32:
2035 case UniInVgprV3S32:
2036 case UniInVgprV4S32:
2037 case UniInVgprV2S64:
2038 case UniInVgprV6S32:
2039 case UniInVgprV8S16:
2040 case UniInVgprV8S32:
2041 case UniInVgprV16S16:
2042 case UniInVgprV16S32:
2043 case UniInVgprV32S16:
2044 case UniInVgprV32S32:
2045 case UniInVgprB32:
2046 case UniInVgprB64:
2047 case UniInVgprB96:
2048 case UniInVgprB128:
2049 case UniInVgprB160:
2050 case UniInVgprB256:
2051 case UniInVgprB512:
2052 case Sgpr32Trunc:
2053 case Sgpr32AExt:
2054 case Sgpr32AExtBoolInReg:
2055 case Sgpr32SExt:
2056 case Sgpr32ZExt:
2057 return SgprRB;
2058 case AgprAnyTy:
2059 return AgprRB;
2060 case Vgpr16:
2061 case Vgpr32:
2062 case Vgpr64:
2063 case Vgpr128:
2064 case VgprP0:
2065 case VgprP1:
2066 case VgprP2:
2067 case VgprP3:
2068 case VgprP4:
2069 case VgprP5:
2070 case VgprPtr32:
2071 case VgprPtr64:
2072 case VgprPtr128:
2073 case VgprV2S16:
2074 case VgprV2S32:
2075 case VgprV2S64:
2076 case VgprV3S32:
2077 case VgprV4S16:
2078 case VgprV8S16:
2079 case VgprV16S16:
2080 case VgprV4S32:
2081 case VgprV6S32:
2082 case VgprV8S32:
2083 case VgprV16S32:
2084 case VgprV32S16:
2085 case VgprV32S32:
2086 case VgprB32:
2087 case VgprB64:
2088 case VgprB96:
2089 case VgprB128:
2090 case VgprB160:
2091 case VgprB256:
2092 case VgprB512:
2093 case VgprBRC:
2094 case VgprAnyTy:
2095 case Vgpr32AExt:
2096 case Vgpr32SExt:
2097 case Vgpr32ZExt:
2098 case Sgpr32ToVgprDst:
2099 case Sgpr64ToVgprDst:
2100 return VgprRB;
2101 default:
2102 return nullptr;
2103 }
2104}
2105
2106bool RegBankLegalizeHelper::applyMappingDst(
2107 MachineInstr &MI, unsigned &OpIdx,
2108 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs) {
2109 // Defs start from operand 0
2110 for (; OpIdx < MethodIDs.size(); ++OpIdx) {
2111 if (MethodIDs[OpIdx] == None)
2112 continue;
2113 MachineOperand &Op = MI.getOperand(i: OpIdx);
2114 Register Reg = Op.getReg();
2115 LLT Ty = MRI.getType(Reg);
2116 [[maybe_unused]] const RegisterBank *RB = MRI.getRegBank(Reg);
2117
2118 switch (MethodIDs[OpIdx]) {
2119 // vcc, sgpr and vgpr scalars, pointers and vectors
2120 case Vcc:
2121 case Sgpr16:
2122 case Sgpr32:
2123 case Sgpr64:
2124 case Sgpr128:
2125 case SgprP0:
2126 case SgprP1:
2127 case SgprP3:
2128 case SgprP4:
2129 case SgprP5:
2130 case SgprP6:
2131 case SgprP8:
2132 case SgprV2S16:
2133 case SgprV2S32:
2134 case SgprV4S32:
2135 case Vgpr16:
2136 case Vgpr32:
2137 case Vgpr64:
2138 case Vgpr128:
2139 case VgprP0:
2140 case VgprP1:
2141 case VgprP2:
2142 case VgprP3:
2143 case VgprP4:
2144 case VgprP5:
2145 case VgprV2S16:
2146 case VgprV2S32:
2147 case VgprV2S64:
2148 case VgprV3S32:
2149 case VgprV4S16:
2150 case VgprV8S16:
2151 case VgprV16S16:
2152 case VgprV4S32:
2153 case VgprV6S32:
2154 case VgprV8S32:
2155 case VgprV16S32:
2156 case VgprV32S16:
2157 case VgprV32S32: {
2158 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2159 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2160 break;
2161 }
2162 // sgpr and vgpr B-types
2163 case SgprB32:
2164 case SgprB64:
2165 case SgprB96:
2166 case SgprB128:
2167 case SgprB256:
2168 case SgprB512:
2169 case SgprBRC:
2170 case SgprPtr32:
2171 case SgprPtr64:
2172 case SgprPtr128:
2173 case VgprB32:
2174 case VgprB64:
2175 case VgprB96:
2176 case VgprB128:
2177 case VgprB160:
2178 case VgprB256:
2179 case VgprB512:
2180 case VgprBRC:
2181 case VgprPtr32:
2182 case VgprPtr64:
2183 case VgprPtr128: {
2184 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2185 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2186 break;
2187 }
2188 case VgprAnyTy: {
2189 assert(RB == VgprRB);
2190 break;
2191 }
2192 case AgprAnyTy: {
2193 if (RB == AgprRB)
2194 break;
2195 Register NewAgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: AgprRB, .Ty: Ty});
2196 Op.setReg(NewAgprDst);
2197 if (!MRI.use_nodbg_empty(RegNo: Reg))
2198 B.buildCopy(Res: Reg, Op: NewAgprDst);
2199 break;
2200 }
2201 case VgprOrAgprAnyTy: {
2202 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2203 const RegisterBank *DstRB =
2204 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2205 if (RB == DstRB)
2206 break;
2207 Register NewDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: DstRB, .Ty: Ty});
2208 Op.setReg(NewDst);
2209 if (!MRI.use_nodbg_empty(RegNo: Reg))
2210 B.buildCopy(Res: Reg, Op: NewDst);
2211 break;
2212 }
2213 // uniform in vcc/vgpr: scalars, vectors and B-types
2214 case UniInVcc: {
2215 assert(Ty == S1);
2216 assert(RB == SgprRB);
2217 Register NewDst = MRI.createVirtualRegister(RegAttr: VccRB_S1);
2218 Op.setReg(NewDst);
2219 if (!MRI.use_empty(RegNo: Reg)) {
2220 auto CopyS32_Vcc =
2221 B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_SCC_VCC, DstOps: {SgprRB_I32}, SrcOps: {NewDst});
2222 B.buildTrunc(Res: Reg, Op: CopyS32_Vcc);
2223 }
2224 break;
2225 }
2226 case UniInVgprS16: {
2227 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2228 assert(RB == SgprRB);
2229 Register NewVgprDst16 = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2230 Register NewVgprDstI32 = MRI.createVirtualRegister(RegAttr: VgprRB_I32);
2231 Register NewSgprDstI32 = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2232 Op.setReg(NewVgprDst16);
2233 B.buildAnyExt(Res: NewVgprDstI32, Op: NewVgprDst16);
2234 buildReadAnyLane(B, SgprDst: NewSgprDstI32, VgprSrc: NewVgprDstI32, RBI);
2235 B.buildTrunc(Res: Reg, Op: NewSgprDstI32);
2236 break;
2237 }
2238 case UniInVgprS32:
2239 case UniInVgprS64:
2240 case UniInVgprV2S16:
2241 case UniInVgprV2S32:
2242 case UniInVgprV3S32:
2243 case UniInVgprV4S32:
2244 case UniInVgprV2S64:
2245 case UniInVgprV6S32:
2246 case UniInVgprV8S16:
2247 case UniInVgprV8S32:
2248 case UniInVgprV16S16:
2249 case UniInVgprV16S32:
2250 case UniInVgprV32S16:
2251 case UniInVgprV32S32: {
2252 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2253 assert(RB == SgprRB);
2254 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2255 Op.setReg(NewVgprDst);
2256 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2257 break;
2258 }
2259 case UniInVgprB32:
2260 case UniInVgprB64:
2261 case UniInVgprB96:
2262 case UniInVgprB128:
2263 case UniInVgprB160:
2264 case UniInVgprB256:
2265 case UniInVgprB512: {
2266 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2267 assert(RB == SgprRB);
2268 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2269 Op.setReg(NewVgprDst);
2270 AMDGPU::buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2271 break;
2272 }
2273 // sgpr trunc
2274 case Sgpr32Trunc: {
2275 assert(Ty.getSizeInBits() < 32);
2276 assert(RB == SgprRB);
2277 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2278 Op.setReg(NewDst);
2279 if (!MRI.use_empty(RegNo: Reg))
2280 B.buildTrunc(Res: Reg, Op: NewDst);
2281 break;
2282 }
2283 case Sgpr32ToVgprDst:
2284 case Sgpr64ToVgprDst: {
2285 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2286 assert(RB == VgprRB);
2287 Op.setReg(MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty}));
2288 B.buildCopy(Res: Reg, Op: Op.getReg());
2289 break;
2290 }
2291 case InvalidMapping: {
2292 reportGISelFailure(
2293 MF, MORE, DEBUG_TYPE,
2294 Msg: "AMDGPU RegBankLegalize: missing fast rule ('Div' or 'Uni') for", MI);
2295 return false;
2296 }
2297 default:
2298 reportGISelFailure(
2299 MF, MORE, DEBUG_TYPE,
2300 Msg: "AMDGPU RegBankLegalize: applyMappingDst, ID not supported", MI);
2301 return false;
2302 }
2303 }
2304
2305 return true;
2306}
2307
2308bool RegBankLegalizeHelper::applyMappingSrc(
2309 MachineInstr &MI, unsigned &OpIdx,
2310 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
2311 WaterfallInfo &WFI) {
2312 for (unsigned i = 0; i < MethodIDs.size(); ++OpIdx, ++i) {
2313 if (MethodIDs[i] == None || MethodIDs[i] == IntrId || MethodIDs[i] == Imm)
2314 continue;
2315
2316 MachineOperand &Op = MI.getOperand(i: OpIdx);
2317 Register Reg = Op.getReg();
2318 LLT Ty = MRI.getType(Reg);
2319 const RegisterBank *RB = MRI.getRegBank(Reg);
2320
2321 switch (MethodIDs[i]) {
2322 case Vcc: {
2323 assert(Ty == S1);
2324 assert(RB == VccRB || RB == SgprRB);
2325 if (RB == SgprRB) {
2326 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2327 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2328 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2329 auto CopyVcc_Scc = B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_VCC_SCC,
2330 DstOps: {VccRB_S1}, SrcOps: {BoolInReg});
2331 Op.setReg(CopyVcc_Scc.getReg(Idx: 0));
2332 }
2333 break;
2334 }
2335 // sgpr scalars, pointers and vectors
2336 case Sgpr16:
2337 case Sgpr32:
2338 case Sgpr64:
2339 case Sgpr128:
2340 case SgprP0:
2341 case SgprP1:
2342 case SgprP3:
2343 case SgprP4:
2344 case SgprP5:
2345 case SgprP6:
2346 case SgprP8:
2347 case SgprV2S16:
2348 case SgprV2S32:
2349 case SgprV4S32: {
2350 assert(Ty == getTyFromID(MethodIDs[i]));
2351 assert(RB == getRegBankFromID(MethodIDs[i]));
2352 break;
2353 }
2354 // sgpr B-types
2355 case SgprB32:
2356 case SgprB64:
2357 case SgprB96:
2358 case SgprB128:
2359 case SgprB256:
2360 case SgprB512:
2361 case SgprBRC:
2362 case SgprPtr32:
2363 case SgprPtr64:
2364 case SgprPtr128: {
2365 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2366 assert(RB == getRegBankFromID(MethodIDs[i]));
2367 break;
2368 }
2369 // vgpr scalars, pointers and vectors
2370 case Vgpr16:
2371 case Vgpr32:
2372 case Vgpr64:
2373 case Vgpr128:
2374 case VgprP0:
2375 case VgprP1:
2376 case VgprP2:
2377 case VgprP3:
2378 case VgprP4:
2379 case VgprP5:
2380 case VgprV2S16:
2381 case VgprV2S32:
2382 case VgprV2S64:
2383 case VgprV3S32:
2384 case VgprV4S16:
2385 case VgprV8S16:
2386 case VgprV16S16:
2387 case VgprV4S32:
2388 case VgprV6S32:
2389 case VgprV8S32:
2390 case VgprV16S32:
2391 case VgprV32S16:
2392 case VgprV32S32: {
2393 assert(Ty == getTyFromID(MethodIDs[i]));
2394 if (RB != VgprRB) {
2395 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2396 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2397 }
2398 break;
2399 }
2400 // vgpr B-types
2401 case VgprB32:
2402 case VgprB64:
2403 case VgprB96:
2404 case VgprB128:
2405 case VgprB160:
2406 case VgprB256:
2407 case VgprB512:
2408 case VgprBRC:
2409 case VgprPtr32:
2410 case VgprPtr64:
2411 case VgprPtr128: {
2412 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2413 if (RB != VgprRB) {
2414 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2415 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2416 }
2417 break;
2418 }
2419 case VgprAnyTy: {
2420 if (RB != VgprRB) {
2421 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2422 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2423 }
2424 break;
2425 }
2426 case AgprAnyTy: {
2427 if (RB != AgprRB) {
2428 auto CopyToAgpr = B.buildCopy(Res: {AgprRB, Ty}, Op: Reg);
2429 Op.setReg(CopyToAgpr.getReg(Idx: 0));
2430 }
2431 break;
2432 }
2433 case VgprOrAgprAnyTy: {
2434 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2435 const RegisterBank *SrcRB =
2436 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2437 if (RB != SrcRB)
2438 Op.setReg(B.buildCopy(Res: {SrcRB, Ty}, Op: Reg).getReg(Idx: 0));
2439 break;
2440 }
2441 // sgpr waterfall, scalars, and vectors
2442 case Sgpr32_WF:
2443 case SgprV4S32_WF: {
2444 assert(Ty == getTyFromID(MethodIDs[i]));
2445 if (RB != SgprRB) {
2446 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2447 if (!WFI.Start.isValid()) {
2448 WFI.Start = MI.getIterator();
2449 WFI.End = std::next(x: MI.getIterator());
2450 }
2451 }
2452 break;
2453 }
2454 case SgprP0Call_WF:
2455 case SgprP4Call_WF: {
2456 assert(Ty == getTyFromID(MethodIDs[i]));
2457 if (RB != SgprRB) {
2458 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2459
2460 // Find the ADJCALLSTACKUP before the call.
2461 MachineBasicBlock::iterator Start = MI.getIterator();
2462 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
2463 --Start;
2464
2465 // Find the ADJCALLSTACKDOWN after the call (include it in range).
2466 MachineBasicBlock::iterator End = MI.getIterator();
2467 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
2468 ++End;
2469 ++End;
2470
2471 WFI.Start = Start;
2472 WFI.End = End;
2473 }
2474 break;
2475 }
2476 case SgprB32_M0:
2477 case SgprB32_ReadFirstLane:
2478 case SgprB64_ReadFirstLane: {
2479 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2480 if (RB == SgprRB)
2481 break;
2482 assert(RB == VgprRB);
2483 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2484 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2485 Op.setReg(NewSGPR);
2486 break;
2487 }
2488 case SgprV4S32_ReadFirstLane:
2489 case SgprV8S32_ReadFirstLane: {
2490 assert(Ty == getTyFromID(MethodIDs[i]));
2491 if (RB == SgprRB)
2492 break;
2493 assert(RB == VgprRB);
2494 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2495 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2496 Op.setReg(NewSGPR);
2497 break;
2498 }
2499 // sgpr and vgpr scalars with extend
2500 case Sgpr32AExt: {
2501 // Note: this ext allows S1, and it is meant to be combined away.
2502 assert(Ty.getSizeInBits() < 32);
2503 assert(RB == SgprRB);
2504 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2505 Op.setReg(Aext.getReg(Idx: 0));
2506 break;
2507 }
2508 case Sgpr32AExtBoolInReg: {
2509 // Note: this ext allows S1, and it is meant to be combined away.
2510 assert(Ty.getSizeInBits() == 1);
2511 assert(RB == SgprRB);
2512 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2513 // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
2514 // most of times meant to be combined away in AMDGPURegBankCombiner.
2515 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2516 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2517 Op.setReg(BoolInReg.getReg(Idx: 0));
2518 break;
2519 }
2520 case Sgpr32SExt: {
2521 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2522 assert(RB == SgprRB);
2523 auto Sext = B.buildSExt(Res: SgprRB_I32, Op: Reg);
2524 Op.setReg(Sext.getReg(Idx: 0));
2525 break;
2526 }
2527 case Sgpr32ZExt: {
2528 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2529 assert(RB == SgprRB);
2530 auto Zext = B.buildZExt(Res: SgprRB_I32, Op: Reg);
2531 Op.setReg(Zext.getReg(Idx: 0));
2532 break;
2533 }
2534 case Vgpr32AExt: {
2535 assert(Ty.getSizeInBits() < 32);
2536 assert(RB == VgprRB);
2537 auto Aext = B.buildAnyExt(Res: VgprRB_I32, Op: Reg);
2538 Op.setReg(Aext.getReg(Idx: 0));
2539 break;
2540 }
2541 case Vgpr32SExt: {
2542 // Note this ext allows S1, and it is meant to be combined away.
2543 assert(Ty.getSizeInBits() < 32);
2544 assert(RB == VgprRB);
2545 auto Sext = B.buildSExt(Res: VgprRB_I32, Op: Reg);
2546 Op.setReg(Sext.getReg(Idx: 0));
2547 break;
2548 }
2549 case Vgpr32ZExt: {
2550 // Note this ext allows S1, and it is meant to be combined away.
2551 assert(Ty.getSizeInBits() < 32);
2552 assert(RB == VgprRB);
2553 auto Zext = B.buildZExt(Res: VgprRB_I32, Op: Reg);
2554 Op.setReg(Zext.getReg(Idx: 0));
2555 break;
2556 }
2557 default:
2558 reportGISelFailure(
2559 MF, MORE, DEBUG_TYPE,
2560 Msg: "AMDGPU RegBankLegalize: applyMappingSrc, ID not supported", MI);
2561 return false;
2562 }
2563 }
2564 return true;
2565}
2566
2567[[maybe_unused]] static bool verifyRegBankOnOperands(MachineInstr &MI,
2568 const RegisterBank *RB,
2569 MachineRegisterInfo &MRI,
2570 unsigned StartOpIdx,
2571 unsigned EndOpIdx) {
2572 for (unsigned i = StartOpIdx; i <= EndOpIdx; ++i) {
2573 if (MRI.getRegBankOrNull(Reg: MI.getOperand(i).getReg()) != RB)
2574 return false;
2575 }
2576 return true;
2577}
2578
2579bool RegBankLegalizeHelper::applyRegisterBanksVgprWithSgprRsrc(
2580 MachineInstr &MI, unsigned RsrcIdx) {
2581 const unsigned NumDefs = MI.getNumExplicitDefs();
2582
2583 MachineBasicBlock *MBB = MI.getParent();
2584 B.setInsertPt(MBB&: *MBB, II: MBB->SkipPHIsAndLabels(I: std::next(x: MI.getIterator())));
2585
2586 // Defs are vgpr.
2587 for (unsigned i = 0; i < NumDefs; ++i) {
2588 Register Reg = MI.getOperand(i).getReg();
2589 if (MRI.getRegBank(Reg) == VgprRB)
2590 continue;
2591
2592 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: MRI.getType(Reg)});
2593 MI.getOperand(i).setReg(NewVgprDst);
2594 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2595 }
2596
2597 B.setInstrAndDebugLoc(MI);
2598
2599 // Register uses before RsrcIdx are vgpr.
2600 for (unsigned i = NumDefs; i < RsrcIdx; ++i) {
2601 MachineOperand &Op = MI.getOperand(i);
2602 if (!Op.isReg())
2603 continue;
2604
2605 Register Reg = Op.getReg();
2606 if (!Reg.isVirtual())
2607 continue;
2608
2609 if (MRI.getRegBank(Reg) == VgprRB)
2610 continue;
2611
2612 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
2613 Op.setReg(Copy.getReg(Idx: 0));
2614 }
2615
2616 SmallSet<Register, 4> OpsToWaterfall;
2617
2618 // Register use RsrcIdx (and later register operands) is sgpr.
2619 for (unsigned i = RsrcIdx; i < MI.getNumOperands(); ++i) {
2620 MachineOperand &Op = MI.getOperand(i);
2621 if (!Op.isReg())
2622 continue;
2623
2624 Register Reg = Op.getReg();
2625 if (MRI.getRegBank(Reg) != SgprRB)
2626 OpsToWaterfall.insert(V: Reg);
2627 }
2628
2629 if (!OpsToWaterfall.empty()) {
2630 MachineBasicBlock::iterator MII = MI.getIterator();
2631 executeInWaterfallLoop(B, WFI: {.SgprWaterfallOperandRegs: OpsToWaterfall, .Start: MII, .End: std::next(x: MII)});
2632 }
2633
2634 return true;
2635}
2636