1//===-- AMDGPURegBankLegalizeHelper.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Implements actual lowering algorithms for each ID that can be used in
10/// Rule.OperandMapping. Similar to legalizer helper but with register banks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPURegBankLegalizeHelper.h"
15#include "AMDGPUGlobalISelUtils.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPURegBankLegalizeRules.h"
19#include "AMDGPURegisterBankInfo.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIMachineFunctionInfo.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
27#include "llvm/CodeGen/MachineInstr.h"
28#include "llvm/CodeGen/MachineUniformityAnalysis.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30
31#define DEBUG_TYPE "amdgpu-regbanklegalize"
32
33using namespace llvm;
34using namespace AMDGPU;
35
36RegBankLegalizeHelper::RegBankLegalizeHelper(
37 MachineIRBuilder &B, const MachineUniformityInfo &MUI,
38 GISelValueTracking *VT, const RegisterBankInfo &RBI,
39 const RegBankLegalizeRules &RBLRules)
40 : MF(B.getMF()), MFI(MF.getInfo<SIMachineFunctionInfo>()),
41 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()), B(B),
42 MRI(*B.getMRI()), MUI(MUI), VT(VT), RBI(RBI), MORE(MF, nullptr),
43 RBLRules(RBLRules), IsWave32(ST.isWave32()),
44 SgprRB(&RBI.getRegBank(ID: AMDGPU::SGPRRegBankID)),
45 VgprRB(&RBI.getRegBank(ID: AMDGPU::VGPRRegBankID)),
46 AgprRB(&RBI.getRegBank(ID: AMDGPU::AGPRRegBankID)),
47 VccRB(&RBI.getRegBank(ID: AMDGPU::VCCRegBankID)) {}
48
49bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
50 const SetOfRulesForOpcode *RuleSet = RBLRules.getRulesForOpc(MI);
51 if (!RuleSet) {
52 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
53 Msg: "No AMDGPU RegBankLegalize rules defined for opcode",
54 MI);
55 return false;
56 }
57
58 const RegBankLLTMapping *Mapping = RuleSet->findMappingForMI(MI, MRI, MUI);
59 if (!Mapping) {
60 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
61 Msg: "AMDGPU RegBankLegalize: none of the rules defined with "
62 "'Any' for MI's opcode matched MI",
63 MI);
64 return false;
65 }
66
67 WaterfallInfo WFI;
68 unsigned OpIdx = 0;
69 if (!Mapping->DstOpMapping.empty()) {
70 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
71 if (!applyMappingDst(MI, OpIdx, MethodIDs: Mapping->DstOpMapping))
72 return false;
73 }
74 if (!Mapping->SrcOpMapping.empty()) {
75 B.setInstr(MI);
76 if (!applyMappingSrc(MI, OpIdx, MethodIDs: Mapping->SrcOpMapping, WFI))
77 return false;
78 }
79
80 if (!lower(MI, Mapping: *Mapping, WFI))
81 return false;
82
83 if (!WFI.SgprWaterfallOperandRegs.empty()) {
84 if (!executeInWaterfallLoop(B, WFI))
85 return false;
86 }
87
88 return true;
89}
90
91bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
92 const WaterfallInfo &WFI) {
93 assert(WFI.Start.isValid() && WFI.End.isValid() &&
94 "Waterfall range not initialized");
95
96 // Track use registers which have already been expanded with a readfirstlane
97 // sequence. This may have multiple uses if moving a sequence.
98 DenseMap<Register, Register> WaterfalledRegMap;
99
100 MachineBasicBlock &MBB = B.getMBB();
101 MachineFunction &MF = B.getMF();
102
103 MachineBasicBlock::iterator BeginIt = WFI.Start;
104 MachineBasicBlock::iterator EndIt = WFI.End;
105
106 const SIRegisterInfo *TRI = ST.getRegisterInfo();
107 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
108 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
109
110#ifndef NDEBUG
111 const int OrigRangeSize = std::distance(BeginIt, EndIt);
112#endif
113
114 MachineRegisterInfo &MRI = *B.getMRI();
115 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
116 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
117
118 // Don't bother using generic instructions/registers for the exec mask.
119 B.setInstr(*WFI.Start);
120 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: InitSaveExecReg);
121
122 Register SavedExec = MRI.createVirtualRegister(RegClass: WaveRC);
123
124 // To insert the loop we need to split the block. Move everything before
125 // this point to a new block, and insert a new empty block before this
126 // instruction.
127 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
128 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
129 MachineBasicBlock *RestoreExecBB = MF.CreateMachineBasicBlock();
130 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
131 MachineFunction::iterator MBBI(MBB);
132 ++MBBI;
133 MF.insert(MBBI, MBB: LoopBB);
134 MF.insert(MBBI, MBB: BodyBB);
135 MF.insert(MBBI, MBB: RestoreExecBB);
136 MF.insert(MBBI, MBB: RemainderBB);
137
138 LoopBB->addSuccessor(Succ: BodyBB);
139 BodyBB->addSuccessor(Succ: RestoreExecBB);
140 BodyBB->addSuccessor(Succ: LoopBB);
141
142 // Move the rest of the block into a new block.
143 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
144 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: EndIt, To: MBB.end());
145
146 MBB.addSuccessor(Succ: LoopBB);
147 RestoreExecBB->addSuccessor(Succ: RemainderBB);
148
149 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
150
151 // +-MBB:------------+
152 // | ... |
153 // | %0 = G_INST_1 |
154 // | %Dst = MI %Vgpr |
155 // | %1 = G_INST_2 |
156 // | ... |
157 // +-----------------+
158 // ->
159 // +-MBB-------------------------------+
160 // | ... |
161 // | %0 = G_INST_1 |
162 // | %SaveExecReg = S_MOV_B32 $exec_lo |
163 // +----------------|------------------+
164 // | /------------------------------|
165 // V V |
166 // +-LoopBB---------------------------------------------------------------+ |
167 // | %CurrentLaneReg:sgpr(s32) = READFIRSTLANE %Vgpr | |
168 // | instead of executing for each lane, see if other lanes had | |
169 // | same value for %Vgpr and execute for them also. | |
170 // | %CondReg:vcc(s1) = G_ICMP eq %CurrentLaneReg, %Vgpr | |
171 // | %CondRegLM:sreg_32 = ballot %CondReg // copy vcc to sreg32 lane mask | |
172 // | %SavedExec = S_AND_SAVEEXEC_B32 %CondRegLM | |
173 // | exec is active for lanes with the same "CurrentLane value" in Vgpr | |
174 // +----------------|-----------------------------------------------------+ |
175 // V |
176 // +-BodyBB------------------------------------------------------------+ |
177 // | %Dst = MI %CurrentLaneReg:sgpr(s32) | |
178 // | executed only for active lanes and written to Dst | |
179 // | $exec = S_XOR_B32 $exec, %SavedExec | |
180 // | set active lanes to 0 in SavedExec, lanes that did not write to | |
181 // | Dst yet, and set this as new exec (for READFIRSTLANE and ICMP) | |
182 // | SI_WATERFALL_LOOP LoopBB |-----|
183 // +----------------|--------------------------------------------------+
184 // V
185 // +-RestoreExecBB--------------------------+
186 // | $exec_lo = S_MOV_B32_term %SaveExecReg |
187 // +----------------|-----------------------+
188 // V
189 // +-RemainderBB:----------------------+
190 // | %1 = G_INST_2 |
191 // | ... |
192 // +---------------------------------- +
193
194 // Move the instruction into the loop body. Note we moved everything after
195 // Range.end() already into a new block, so Range.end() is no longer valid.
196 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: BeginIt, To: MBB.end());
197
198 // Figure out the iterator range after splicing the instructions.
199 MachineBasicBlock::iterator NewBegin = BeginIt;
200 auto NewEnd = BodyBB->end();
201 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
202
203 B.setMBB(*LoopBB);
204 Register CondReg;
205
206 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
207 for (MachineOperand &Op : MI.all_uses()) {
208 Register OldReg = Op.getReg();
209 if (!WFI.SgprWaterfallOperandRegs.count(V: OldReg))
210 continue;
211
212 // See if we already processed this register in another instruction in
213 // the sequence.
214 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
215 if (OldVal != WaterfalledRegMap.end()) {
216 Op.setReg(OldVal->second);
217 continue;
218 }
219
220 Register OpReg = Op.getReg();
221 LLT OpTy = MRI.getType(Reg: OpReg);
222
223 // TODO: support for agpr
224 assert(MRI.getRegBank(OpReg) == VgprRB);
225 Register CurrentLaneReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: OpTy});
226 buildReadFirstLane(B, SgprDst: CurrentLaneReg, VgprSrc: OpReg, RBI);
227
228 // Build the comparison(s), CurrentLaneReg == OpReg.
229 unsigned OpSize = OpTy.getSizeInBits();
230 unsigned PartSize = (OpSize % 64 == 0) ? 64 : 32;
231 LLT PartTy = LLT::integer(SizeInBits: PartSize);
232 unsigned NumParts = OpSize / PartSize;
233 SmallVector<Register, 8> OpParts;
234 SmallVector<Register, 8> CurrentLaneParts;
235
236 if (NumParts == 1) {
237 OpParts.push_back(Elt: OpReg);
238 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
239 } else {
240 auto UnmergeOp = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: PartTy}, Op: OpReg);
241 auto UnmergeCurrLane = B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: PartTy}, Op: CurrentLaneReg);
242 for (unsigned i = 0; i < NumParts; ++i) {
243 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
244 CurrentLaneParts.push_back(Elt: UnmergeCurrLane.getReg(Idx: i));
245 }
246 }
247
248 for (unsigned i = 0; i < NumParts; ++i) {
249 Register CmpReg = MRI.createVirtualRegister(RegAttr: VccRB_S1);
250 B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpReg, Op0: CurrentLaneParts[i], Op1: OpParts[i]);
251
252 if (!CondReg)
253 CondReg = CmpReg;
254 else
255 CondReg = B.buildAnd(Dst: VccRB_S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
256 }
257
258 Op.setReg(CurrentLaneReg);
259
260 // Make sure we don't re-process this register again.
261 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
262 }
263 }
264
265 // Copy vcc to sgpr32/64, ballot becomes a no-op during instruction selection.
266 Register CondRegLM =
267 MRI.createVirtualRegister(RegAttr: {.RCOrRB: WaveRC, .Ty: LLT::integer(SizeInBits: IsWave32 ? 32 : 64)});
268 B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot, Res: CondRegLM).addReg(RegNo: CondReg);
269
270 // Update EXEC, save the original EXEC value to SavedExec.
271 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
272 .addDef(RegNo: SavedExec)
273 .addReg(RegNo: CondRegLM, Flags: RegState::Kill);
274 MRI.setSimpleHint(VReg: SavedExec, PrefReg: CondRegLM);
275
276 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
277
278 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
279 B.buildInstr(Opcode: LMC.XorTermOpc)
280 .addDef(RegNo: LMC.ExecReg)
281 .addReg(RegNo: LMC.ExecReg)
282 .addReg(RegNo: SavedExec);
283
284 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
285 // s_cbranch_scc0?
286
287 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
288 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
289
290 // Save the EXEC mask before the loop.
291 B.setInsertPt(MBB, II: MBB.end());
292 B.buildInstr(Opcode: LMC.MovOpc).addDef(RegNo: SaveExecReg).addReg(RegNo: LMC.ExecReg);
293
294 // Restore the EXEC mask after the loop.
295 B.setInsertPt(MBB&: *RestoreExecBB, II: RestoreExecBB->begin());
296 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
297
298 // Set the insert point after the original instruction, so any new
299 // instructions will be in the remainder.
300 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
301
302 return true;
303}
304
305// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
306// the three offsets (voffset, soffset and instoffset)
307unsigned RegBankLegalizeHelper::setBufferOffsets(
308 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
309 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) {
310 if (std::optional<int64_t> Imm =
311 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI)) {
312 uint32_t SOffset, ImmOffset;
313 if (TII.splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
314 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
315 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
316 InstOffsetVal = ImmOffset;
317 return SOffset + ImmOffset;
318 }
319 }
320 const bool CheckNUW = ST.hasGFX1250Insts();
321 auto [Base, Offset] = AMDGPU::getBaseWithConstantOffset(
322 MRI, Reg: CombinedOffset, /*KnownBits=*/ValueTracking: nullptr,
323 /*CheckNUW=*/CheckNUW);
324 uint32_t SOffset, ImmOffset;
325 if (static_cast<int32_t>(Offset) > 0 &&
326 TII.splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
327 if (Base.isValid() && MRI.getRegBank(Reg: Base) == VgprRB) {
328 VOffsetReg = Base;
329 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
330 InstOffsetVal = ImmOffset;
331 return 0;
332 }
333 // If we have SGPR base, we can use it for soffset.
334 if (SOffset == 0) {
335 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
336 SOffsetReg = Base;
337 InstOffsetVal = ImmOffset;
338 return 0;
339 }
340 }
341 // Handle the variable sgpr + vgpr case.
342 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI);
343 if (Add && static_cast<int32_t>(Offset) >= 0 &&
344 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
345 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI);
346 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI);
347 const RegisterBank *Src0Bank = MRI.getRegBank(Reg: Src0);
348 const RegisterBank *Src1Bank = MRI.getRegBank(Reg: Src1);
349 if (Src0Bank == VgprRB && Src1Bank == SgprRB) {
350 VOffsetReg = Src0;
351 SOffsetReg = Src1;
352 return 0;
353 }
354 if (Src0Bank == SgprRB && Src1Bank == VgprRB) {
355 VOffsetReg = Src1;
356 SOffsetReg = Src0;
357 return 0;
358 }
359 }
360 // Ensure we have a VGPR for the combined offset. This could be an issue if we
361 // have an SGPR offset and a VGPR resource.
362 if (MRI.getRegBank(Reg: CombinedOffset) == VgprRB) {
363 VOffsetReg = CombinedOffset;
364 } else {
365 VOffsetReg = B.buildCopy(Res: VgprRB_I32, Op: CombinedOffset).getReg(Idx: 0);
366 }
367 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: 0).getReg(Idx: 0);
368 return 0;
369}
370
371bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
372 ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
373 MachineFunction &MF = B.getMF();
374 assert(MI.getNumMemOperands() == 1);
375 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
376 Register Dst = MI.getOperand(i: 0).getReg();
377 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
378 Register Base = MI.getOperand(i: 1).getReg();
379 LLT PtrTy = MRI.getType(Reg: Base);
380 const RegisterBank *PtrRB = MRI.getRegBankOrNull(Reg: Base);
381 LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
382 SmallVector<Register, 4> LoadPartRegs;
383
384 unsigned ByteOffset = 0;
385 for (LLT PartTy : LLTBreakdown) {
386 Register BasePlusOffset;
387 if (ByteOffset == 0) {
388 BasePlusOffset = Base;
389 } else {
390 auto Offset = B.buildConstant(Res: {PtrRB, OffsetTy}, Val: ByteOffset);
391 BasePlusOffset =
392 B.buildObjectPtrOffset(Res: {PtrRB, PtrTy}, Op0: Base, Op1: Offset).getReg(Idx: 0);
393 }
394 auto *OffsetMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: ByteOffset, Ty: PartTy);
395 auto LoadPart = B.buildLoad(Res: {DstRB, PartTy}, Addr: BasePlusOffset, MMO&: *OffsetMMO);
396 LoadPartRegs.push_back(Elt: LoadPart.getReg(Idx: 0));
397 ByteOffset += PartTy.getSizeInBytes();
398 }
399
400 if (!MergeTy.isValid()) {
401 // Loads are of same size, concat or merge them together.
402 B.buildMergeLikeInstr(Res: Dst, Ops: LoadPartRegs);
403 } else {
404 // Loads are not all of same size, need to unmerge them to smaller pieces
405 // of MergeTy type, then merge pieces to Dst.
406 SmallVector<Register, 4> MergeTyParts;
407 for (Register Reg : LoadPartRegs) {
408 if (MRI.getType(Reg) == MergeTy) {
409 MergeTyParts.push_back(Elt: Reg);
410 } else {
411 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: Reg);
412 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i)
413 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
414 }
415 }
416 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
417 }
418 MI.eraseFromParent();
419 return true;
420}
421
422bool RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
423 LLT MergeTy) {
424 MachineFunction &MF = B.getMF();
425 assert(MI.getNumMemOperands() == 1);
426 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
427 Register Dst = MI.getOperand(i: 0).getReg();
428 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
429 Register Base = MI.getOperand(i: 1).getReg();
430
431 MachineMemOperand *WideMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: 0, Ty: WideTy);
432 auto WideLoad = B.buildLoad(Res: {DstRB, WideTy}, Addr: Base, MMO&: *WideMMO);
433
434 if (WideTy.isScalar()) {
435 B.buildTrunc(Res: Dst, Op: WideLoad);
436 } else {
437 SmallVector<Register, 4> MergeTyParts;
438 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: WideLoad);
439
440 LLT DstTy = MRI.getType(Reg: Dst);
441 unsigned NumElts = DstTy.getSizeInBits() / MergeTy.getSizeInBits();
442 for (unsigned i = 0; i < NumElts; ++i) {
443 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
444 }
445 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
446 }
447 MI.eraseFromParent();
448 return true;
449}
450
451bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
452 Register Dst = MI.getDstReg();
453 Register Ptr = MI.getPointerReg();
454 MachineMemOperand &MMO = MI.getMMO();
455 unsigned MemSize = 8 * MMO.getSize().getValue();
456
457 MachineMemOperand *WideMMO = B.getMF().getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: S32);
458
459 if (MI.getOpcode() == G_LOAD) {
460 B.buildLoad(Res: Dst, Addr: Ptr, MMO&: *WideMMO);
461 } else {
462 auto Load = B.buildLoad(Res: SgprRB_I32, Addr: Ptr, MMO&: *WideMMO);
463
464 if (MI.getOpcode() == G_ZEXTLOAD) {
465 APInt Mask = APInt::getLowBitsSet(numBits: S32.getSizeInBits(), loBitsSet: MemSize);
466 auto MaskCst = B.buildConstant(Res: SgprRB_I32, Val: Mask);
467 B.buildAnd(Dst, Src0: Load, Src1: MaskCst);
468 } else {
469 assert(MI.getOpcode() == G_SEXTLOAD);
470 B.buildSExtInReg(Res: Dst, Op: Load, ImmOp: MemSize);
471 }
472 }
473
474 MI.eraseFromParent();
475 return true;
476}
477
478bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
479 Register Dst = MI.getOperand(i: 0).getReg();
480 LLT Ty = MRI.getType(Reg: Dst);
481 Register Src = MI.getOperand(i: 1).getReg();
482 unsigned Opc = MI.getOpcode();
483 int TrueExtCst = Opc == G_SEXT ? -1 : 1;
484 if (Ty == S32 || Ty == S16) {
485 auto True = B.buildConstant(Res: {VgprRB, Ty}, Val: TrueExtCst);
486 auto False = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
487 B.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
488 } else if (Ty == S64) {
489 auto True = B.buildConstant(Res: {VgprRB_I32}, Val: TrueExtCst);
490 auto False = B.buildConstant(Res: {VgprRB_I32}, Val: 0);
491 auto Lo = B.buildSelect(Res: {VgprRB_I32}, Tst: Src, Op0: True, Op1: False);
492 MachineInstrBuilder Hi;
493 switch (Opc) {
494 case G_SEXT:
495 Hi = Lo;
496 break;
497 case G_ZEXT:
498 Hi = False;
499 break;
500 case G_ANYEXT:
501 Hi = B.buildUndef(Res: {VgprRB_I32});
502 break;
503 default:
504 reportGISelFailure(
505 MF, MORE, PassName: "amdgpu-regbanklegalize",
506 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Opcode not supported", MI);
507 return false;
508 }
509
510 B.buildMergeValues(Res: Dst, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
511 } else {
512 reportGISelFailure(
513 MF, MORE, PassName: "amdgpu-regbanklegalize",
514 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Type not supported", MI);
515 return false;
516 }
517
518 MI.eraseFromParent();
519 return true;
520}
521
522std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
523 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
524 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: 0x0000ffff);
525 auto Lo = B.buildAnd(Dst: SgprRB_I32, Src0: PackedI32, Src1: Mask);
526 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
527 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
528}
529
530std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
531 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
532 auto Lo = B.buildSExtInReg(Res: SgprRB_I32, Op: PackedI32, ImmOp: 16);
533 auto Hi = B.buildAShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
534 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
535}
536
537std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
538 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
539 auto Lo = PackedI32;
540 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
541 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
542}
543
544std::pair<Register, Register>
545RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
546 auto [Lo32, Hi32] = unpackAExt(Reg);
547 LLT EltTy = MRI.getType(Reg).getElementType();
548 return {B.buildTrunc(Res: {SgprRB, EltTy}, Op: Lo32).getReg(Idx: 0),
549 B.buildTrunc(Res: {SgprRB, EltTy}, Op: Hi32).getReg(Idx: 0)};
550}
551
552bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
553 Register Lo, Hi;
554 switch (MI.getOpcode()) {
555 case AMDGPU::G_SHL: {
556 auto [Val0, Val1] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
557 auto [Amt0, Amt1] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
558 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
559 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
560 break;
561 }
562 case AMDGPU::G_LSHR: {
563 auto [Val0, Val1] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
564 auto [Amt0, Amt1] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
565 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
566 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
567 break;
568 }
569 case AMDGPU::G_ASHR: {
570 auto [Val0, Val1] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
571 auto [Amt0, Amt1] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
572 Lo = B.buildAShr(Dst: SgprRB_I32, Src0: Val0, Src1: Amt0).getReg(Idx: 0);
573 Hi = B.buildAShr(Dst: SgprRB_I32, Src0: Val1, Src1: Amt1).getReg(Idx: 0);
574 break;
575 }
576 default:
577 reportGISelFailure(
578 MF, MORE, PassName: "amdgpu-regbanklegalize",
579 Msg: "AMDGPU RegBankLegalize: lowerUnpackBitShift, case not implemented",
580 MI);
581 return false;
582 }
583 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
584 MI.eraseFromParent();
585 return true;
586}
587
588bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
589 Register Lo, Hi;
590 switch (MI.getOpcode()) {
591 case AMDGPU::G_SMIN:
592 case AMDGPU::G_SMAX: {
593 // For signed operations, use sign extension
594 auto [Val0_Lo, Val0_Hi] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
595 auto [Val1_Lo, Val1_Hi] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
596 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
597 .getReg(Idx: 0);
598 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
599 .getReg(Idx: 0);
600 break;
601 }
602 case AMDGPU::G_UMIN:
603 case AMDGPU::G_UMAX: {
604 // For unsigned operations, use zero extension
605 auto [Val0_Lo, Val0_Hi] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
606 auto [Val1_Lo, Val1_Hi] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
607 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
608 .getReg(Idx: 0);
609 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
610 .getReg(Idx: 0);
611 break;
612 }
613 default:
614 reportGISelFailure(
615 MF, MORE, PassName: "amdgpu-regbanklegalize",
616 Msg: "AMDGPU RegBankLegalize: lowerUnpackMinMax, case not implemented", MI);
617 return false;
618 }
619 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
620 MI.eraseFromParent();
621 return true;
622}
623
624bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
625 auto [Op1Lo, Op1Hi] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
626 auto [Op2Lo, Op2Hi] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
627 auto ResLo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Lo, Op2Lo});
628 auto ResHi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Hi, Op2Hi});
629 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
630 Ops: {ResLo.getReg(Idx: 0), ResHi.getReg(Idx: 0)});
631 MI.eraseFromParent();
632 return true;
633}
634
635bool RegBankLegalizeHelper::lowerSBufToBuf(MachineInstr &MI,
636 WaterfallInfo &WFI) {
637 Register Dst = MI.getOperand(i: 0).getReg();
638 LLT Ty = MRI.getType(Reg: Dst);
639 const RegisterBank *RSrcBank = MRI.getRegBank(Reg: MI.getOperand(i: 1).getReg());
640 unsigned LoadSize = Ty.getSizeInBits();
641 int NumLoads = 1;
642 SmallVector<Register, 4> LoadParts;
643 if (LoadSize == 256 || LoadSize == 512) {
644 NumLoads = LoadSize / 128;
645 Ty = Ty.divide(Factor: NumLoads);
646 }
647 for (int i = 0; i < NumLoads; ++i)
648 LoadParts.emplace_back(Args: MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty}));
649 MachineMemOperand *OrigMMO = *MI.memoperands_begin();
650 const Align Alignment = OrigMMO->getAlign();
651 MachineFunction &MF = B.getMF();
652 Register SOffset;
653 Register VOffset;
654 int64_t ImmOffset = 0;
655 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
656 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
657 // Use the MMO size from the original instruction rather than the (possibly
658 // widened) register type. E.g. 96-bit loads are widened to 128-bit during
659 // legalization but the MMO still reflects the original 96-bit access size.
660 const unsigned MemSize = divideCeil(Numerator: OrigMMO->getSize().getValue(), Denominator: NumLoads);
661 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(MMO: OrigMMO, Offset: 0, Size: MemSize);
662 if (MMOOffset != 0)
663 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
664 // If only the offset is divergent, emit a MUBUF buffer load
665 // instead. We can assume that the buffer is unswizzled.
666 Register RSrc = MI.getOperand(i: 1).getReg();
667 Register VIndex = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
668 unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
669 switch (MI.getOpcode()) {
670 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
671 Opc = G_AMDGPU_BUFFER_LOAD_SBYTE;
672 break;
673 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
674 Opc = G_AMDGPU_BUFFER_LOAD_UBYTE;
675 break;
676 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
677 Opc = G_AMDGPU_BUFFER_LOAD_SSHORT;
678 break;
679 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
680 Opc = G_AMDGPU_BUFFER_LOAD_USHORT;
681 break;
682 default:
683 break;
684 }
685 for (int i = 0; i < NumLoads; ++i) {
686 B.buildInstr(Opcode: Opc)
687 .addDef(RegNo: LoadParts[i]) // vdata
688 .addUse(RegNo: RSrc) // rsrc
689 .addUse(RegNo: VIndex) // vindex
690 .addUse(RegNo: VOffset) // voffset
691 .addUse(RegNo: SOffset) // soffset
692 .addImm(Val: ImmOffset + 16 * i) // offset(imm)
693 .addImm(Val: 0) // cachepolicy, swizzled buffer(imm)
694 .addImm(Val: 0) // idxen(imm)
695 .addMemOperand(MMO: MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16 * i, Size: MemSize));
696 }
697 if (NumLoads == 1)
698 B.buildCopy(Res: Dst, Op: LoadParts[0]);
699 else
700 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
701 B.setInstr(*MRI.getVRegDef(Reg: LoadParts[0]));
702 if (RSrcBank != SgprRB) {
703 WFI.SgprWaterfallOperandRegs.insert(V: RSrc);
704 WFI.Start = MRI.getVRegDef(Reg: LoadParts.front());
705 WFI.End = std::next(x: MRI.getVRegDef(Reg: LoadParts.back())->getIterator());
706 }
707 MI.eraseFromParent();
708 return true;
709}
710
711static bool isSignedBFE(MachineInstr &MI) {
712 if (GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI))
713 return (GI->is(ID: Intrinsic::amdgcn_sbfe));
714
715 return MI.getOpcode() == AMDGPU::G_SBFX;
716}
717
718bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
719 Register Dst = MI.getOperand(i: 0).getReg();
720 assert(MRI.getType(Dst) == LLT::scalar(64));
721 bool Signed = isSignedBFE(MI);
722 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
723 // Extract bitfield from Src, LSBit is the least-significant bit for the
724 // extraction (field offset) and Width is size of bitfield.
725 Register Src = MI.getOperand(i: FirstOpnd).getReg();
726 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
727 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
728 // Comments are for signed bitfield extract, similar for unsigned. x is sign
729 // bit. s is sign, l is LSB and y are remaining bits of bitfield to extract.
730
731 // Src >> LSBit Hi|Lo: x?????syyyyyyl??? -> xxxx?????syyyyyyl
732 unsigned SHROpc = Signed ? AMDGPU::G_ASHR : AMDGPU::G_LSHR;
733 auto SHRSrc = B.buildInstr(Opc: SHROpc, DstOps: {VgprRB_I64}, SrcOps: {Src, LSBit});
734
735 auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: Width, MRI);
736
737 // Expand to Src >> LSBit << (64 - Width) >> (64 - Width)
738 // << (64 - Width): Hi|Lo: xxxx?????syyyyyyl -> syyyyyyl000000000
739 // >> (64 - Width): Hi|Lo: syyyyyyl000000000 -> ssssssssssyyyyyyl
740 if (!ConstWidth) {
741 auto Amt = B.buildSub(Dst: VgprRB_I32, Src0: B.buildConstant(Res: SgprRB_I32, Val: 64), Src1: Width);
742 auto SignBit = B.buildShl(Dst: VgprRB_I64, Src0: SHRSrc, Src1: Amt);
743 B.buildInstr(Opc: SHROpc, DstOps: {Dst}, SrcOps: {SignBit, Amt});
744 MI.eraseFromParent();
745 return true;
746 }
747
748 uint64_t WidthImm = ConstWidth->Value.getZExtValue();
749 auto UnmergeSHRSrc = B.buildUnmerge(Attrs: VgprRB_I32, Op: SHRSrc);
750 Register SHRSrcLo = UnmergeSHRSrc.getReg(Idx: 0);
751 Register SHRSrcHi = UnmergeSHRSrc.getReg(Idx: 1);
752 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
753 unsigned BFXOpc = Signed ? AMDGPU::G_SBFX : AMDGPU::G_UBFX;
754
755 if (WidthImm <= 32) {
756 // SHRSrc Hi|Lo: ????????|???syyyl -> ????????|ssssyyyl
757 auto Lo = B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcLo, Zero, Width});
758 MachineInstrBuilder Hi;
759 if (Signed) {
760 // SHRSrc Hi|Lo: ????????|ssssyyyl -> ssssssss|ssssyyyl
761 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: B.buildConstant(Res: VgprRB_I32, Val: 31));
762 } else {
763 // SHRSrc Hi|Lo: ????????|000syyyl -> 00000000|000syyyl
764 Hi = Zero;
765 }
766 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
767 } else {
768 auto Amt = B.buildConstant(Res: VgprRB_I32, Val: WidthImm - 32);
769 // SHRSrc Hi|Lo: ??????sy|yyyyyyyl -> sssssssy|yyyyyyyl
770 auto Hi = B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcHi, Zero, Amt});
771 B.buildMergeLikeInstr(Res: Dst, Ops: {SHRSrcLo, Hi});
772 }
773
774 MI.eraseFromParent();
775 return true;
776}
777
778bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
779 Register DstReg = MI.getOperand(i: 0).getReg();
780 LLT Ty = MRI.getType(Reg: DstReg);
781 bool Signed = isSignedBFE(MI);
782 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
783 Register Src = MI.getOperand(i: FirstOpnd).getReg();
784 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
785 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
786 // For uniform bit field extract there are 4 available instructions, but
787 // LSBit(field offset) and Width(size of bitfield) need to be packed in S32,
788 // field offset in low and size in high 16 bits.
789
790 // Src1 Hi16|Lo16 = Size|FieldOffset
791 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: maskTrailingOnes<unsigned>(N: 6));
792 auto FieldOffset = B.buildAnd(Dst: SgprRB_I32, Src0: LSBit, Src1: Mask);
793 auto Size = B.buildShl(Dst: SgprRB_I32, Src0: Width, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
794 auto Src1 = B.buildOr(Dst: SgprRB_I32, Src0: FieldOffset, Src1: Size);
795 unsigned Opc32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
796 unsigned Opc64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
797 unsigned Opc = Ty == S32 ? Opc32 : Opc64;
798
799 // Select machine instruction, because of reg class constraining, insert
800 // copies from reg class to reg bank.
801 auto S_BFE = B.buildInstr(Opc, DstOps: {{SgprRB, Ty}},
802 SrcOps: {B.buildCopy(Res: Ty, Op: Src), B.buildCopy(Res: I32, Op: Src1)});
803 constrainSelectedInstRegOperands(I&: *S_BFE, TII: *ST.getInstrInfo(),
804 TRI: *ST.getRegisterInfo(), RBI);
805
806 B.buildCopy(Res: DstReg, Op: S_BFE->getOperand(i: 0).getReg());
807 MI.eraseFromParent();
808 return true;
809}
810
811bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
812 Register Dst = MI.getOperand(i: 0).getReg();
813 LLT DstTy = MRI.getType(Reg: Dst);
814 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
815 LLT Ty = DstTy.divide(Factor: 2);
816 auto Op1 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 1).getReg());
817 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
818 unsigned Opc = MI.getOpcode();
819 auto Flags = MI.getFlags();
820 auto Lo =
821 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 0), Op2.getReg(Idx: 0)}, Flags);
822 auto Hi =
823 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 1), Op2.getReg(Idx: 1)}, Flags);
824 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
825 MI.eraseFromParent();
826 return true;
827}
828
829bool RegBankLegalizeHelper::lowerSplitTo32Mul(MachineInstr &MI) {
830 Register Dst = MI.getOperand(i: 0).getReg();
831 assert(MRI.getType(Dst) == S64);
832 auto Op1 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 1).getReg());
833 auto Op2 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 2).getReg());
834
835 // TODO: G_AMDGPU_MAD_* optimizations for G_MUL divergent S64 operation to
836 // match GlobalISel with old regbankselect.
837 auto Lo = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
838 auto Carry = B.buildUMulH(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
839 auto MulLo0Hi1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 1));
840 auto MulHi0Lo1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 1), Src1: Op2.getReg(Idx: 0));
841 auto Sum = B.buildAdd(Dst: VgprRB_I32, Src0: MulLo0Hi1, Src1: MulHi0Lo1);
842 auto Hi = B.buildAdd(Dst: VgprRB_I32, Src0: Sum, Src1: Carry);
843
844 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
845 MI.eraseFromParent();
846 return true;
847}
848
849bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
850 Register Dst = MI.getOperand(i: 0).getReg();
851 assert(MRI.getType(Dst) == V2S16);
852 unsigned Opc = MI.getOpcode();
853 unsigned NumOps = MI.getNumOperands();
854 auto Flags = MI.getFlags();
855
856 auto [Op1Lo, Op1Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 1).getReg());
857 LLT EltTy = MRI.getType(Reg: Dst).getElementType();
858
859 if (NumOps == 2) {
860 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo}, Flags);
861 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi}, Flags);
862 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
863 MI.eraseFromParent();
864 return true;
865 }
866
867 auto [Op2Lo, Op2Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 2).getReg());
868
869 if (NumOps == 3) {
870 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo}, Flags);
871 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi}, Flags);
872 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
873 MI.eraseFromParent();
874 return true;
875 }
876
877 assert(NumOps == 4);
878 auto [Op3Lo, Op3Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 3).getReg());
879 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo, Op3Lo}, Flags);
880 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi, Op3Hi}, Flags);
881 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
882 MI.eraseFromParent();
883 return true;
884}
885
886bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
887 Register Dst0 = MI.getOperand(i: 0).getReg();
888 Register Dst1 = MI.getOperand(i: 1).getReg();
889 Register Src0 = MI.getOperand(i: 2).getReg();
890 Register Src1 = MI.getOperand(i: 3).getReg();
891 Register Src2 = MI.getOperand(i: 4).getReg();
892
893 const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
894
895 // Keep the multiplication on the SALU.
896 Register DstLo = B.buildMul(Dst: SgprRB_I32, Src0, Src1).getReg(Idx: 0);
897 Register DstHi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
898 if (ST.hasScalarMulHiInsts()) {
899 B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {{DstHi}}, SrcOps: {Src0, Src1});
900 } else {
901 auto VSrc0 = B.buildCopy(Res: VgprRB_I32, Op: Src0);
902 auto VSrc1 = B.buildCopy(Res: VgprRB_I32, Op: Src1);
903 auto MulHi = B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {VgprRB_I32}, SrcOps: {VSrc0, VSrc1});
904 buildReadAnyLane(B, SgprDst: DstHi, VgprSrc: MulHi.getReg(Idx: 0), RBI);
905 }
906
907 // Accumulate and produce the "carry-out" bit.
908
909 // The "carry-out" is defined as bit 64 of the result when computed as a
910 // big integer. For unsigned multiply-add, this matches the usual
911 // definition of carry-out.
912 if (mi_match(R: Src2, MRI, P: MIPatternMatch::m_ZeroInt())) {
913 // No accumulate: result is just the multiplication, carry is 0.
914 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
915 B.buildConstant(Res: Dst1, Val: 0);
916 } else {
917 // Accumulate: add Src2 to the multiplication result with carry chain.
918 Register Src2Lo = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
919 Register Src2Hi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
920 B.buildUnmerge(Res: {Src2Lo, Src2Hi}, Op: Src2);
921
922 auto AddLo = B.buildUAddo(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstLo, Op1: Src2Lo);
923 auto AddHi =
924 B.buildUAdde(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstHi, Op1: Src2Hi, CarryIn: AddLo.getReg(Idx: 1));
925 B.buildMergeLikeInstr(Res: Dst0, Ops: {AddLo.getReg(Idx: 0), AddHi.getReg(Idx: 0)});
926 B.buildCopy(Res: Dst1, Op: AddHi.getReg(Idx: 1));
927 }
928
929 MI.eraseFromParent();
930 return true;
931}
932
933bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
934 Register Dst = MI.getOperand(i: 0).getReg();
935 LLT DstTy = MRI.getType(Reg: Dst);
936 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
937 (DstTy.isPointer() && DstTy.getSizeInBits() == 64));
938 LLT Ty = DstTy.isFloat() ? LLT::float32() : DstTy.divide(Factor: 2);
939 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
940 auto Op3 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 3).getReg());
941 Register Cond = MI.getOperand(i: 1).getReg();
942 auto Flags = MI.getFlags();
943 auto Lo =
944 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 0), Op1: Op3.getReg(Idx: 0), Flags);
945 auto Hi =
946 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 1), Op1: Op3.getReg(Idx: 1), Flags);
947
948 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
949 MI.eraseFromParent();
950 return true;
951}
952
953bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
954 auto Op1 = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
955 int Amt = MI.getOperand(i: 2).getImm();
956 Register Lo, Hi;
957 // Hi|Lo: s sign bit, ?/x bits changed/not changed by sign-extend
958 if (Amt <= 32) {
959 auto Freeze = B.buildFreeze(Dst: VgprRB_I32, Src: Op1.getReg(Idx: 0));
960 if (Amt == 32) {
961 // Hi|Lo: ????????|sxxxxxxx -> ssssssss|sxxxxxxx
962 Lo = Freeze.getReg(Idx: 0);
963 } else {
964 // Hi|Lo: ????????|???sxxxx -> ssssssss|ssssxxxx
965 Lo = B.buildSExtInReg(Res: VgprRB_I32, Op: Freeze, ImmOp: Amt).getReg(Idx: 0);
966 }
967
968 auto SignExtCst = B.buildConstant(Res: SgprRB_I32, Val: 31);
969 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: SignExtCst).getReg(Idx: 0);
970 } else {
971 // Hi|Lo: ?????sxx|xxxxxxxx -> ssssssxx|xxxxxxxx
972 Lo = Op1.getReg(Idx: 0);
973 Hi = B.buildSExtInReg(Res: VgprRB_I32, Op: Op1.getReg(Idx: 1), ImmOp: Amt - 32).getReg(Idx: 0);
974 }
975
976 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
977 MI.eraseFromParent();
978 return true;
979}
980
981bool RegBankLegalizeHelper::lowerSplitBitCount64To32(MachineInstr &MI) {
982 // Split 64-bit find-first-bit operations into 32-bit halves:
983 // (ffbh hi:lo) -> umin(ffbh(hi), uaddsat(ffbh(lo), 32))
984 // (ffbl hi:lo) -> umin(ffbl(lo), uaddsat(ffbl(hi), 32))
985 // (ctlz_zero_poison hi:lo) -> umin(ffbh(hi), add(ffbh(lo), 32))
986 // (cttz_zero_poison hi:lo) -> umin(ffbl(lo), add(ffbl(hi), 32))
987 unsigned Opc = MI.getOpcode();
988
989 // FFBH/FFBL return 0xFFFFFFFF on zero input, using uaddsat to avoid
990 // wrapping. CTLZ/CTTZ guarantee non-zero input (zero_poison), so plain add
991 // is fine.
992 unsigned FFBOpc;
993 unsigned AddOpc;
994 bool SearchFromMSB;
995 switch (Opc) {
996 case AMDGPU::G_AMDGPU_FFBH_U32:
997 FFBOpc = Opc;
998 AddOpc = AMDGPU::G_UADDSAT;
999 SearchFromMSB = true;
1000 break;
1001 case AMDGPU::G_AMDGPU_FFBL_B32:
1002 FFBOpc = Opc;
1003 AddOpc = AMDGPU::G_UADDSAT;
1004 SearchFromMSB = false;
1005 break;
1006 case AMDGPU::G_CTLZ_ZERO_POISON:
1007 FFBOpc = AMDGPU::G_AMDGPU_FFBH_U32;
1008 AddOpc = AMDGPU::G_ADD;
1009 SearchFromMSB = true;
1010 break;
1011 case AMDGPU::G_CTTZ_ZERO_POISON:
1012 FFBOpc = AMDGPU::G_AMDGPU_FFBL_B32;
1013 AddOpc = AMDGPU::G_ADD;
1014 SearchFromMSB = false;
1015 break;
1016 default:
1017 llvm_unreachable("unexpected opcode in lowerSplitBitCount64To32");
1018 }
1019
1020 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1021 Register Lo = Unmerge.getReg(Idx: 0);
1022 Register Hi = Unmerge.getReg(Idx: 1);
1023
1024 // MSB-first (FFBH/CTLZ) searches hi first; LSB-first (FFBL/CTTZ) searches
1025 // lo first. The secondary half adds 32 to account for the primary half's
1026 // width.
1027 auto Primary = B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Hi : Lo});
1028 auto Secondary =
1029 B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Lo : Hi});
1030
1031 auto Adjusted = B.buildInstr(Opc: AddOpc, DstOps: {VgprRB_I32},
1032 SrcOps: {Secondary, B.buildConstant(Res: VgprRB_I32, Val: 32)});
1033 B.buildUMin(Dst: MI.getOperand(i: 0).getReg(), Src0: Primary, Src1: Adjusted);
1034
1035 MI.eraseFromParent();
1036 return true;
1037}
1038
1039bool RegBankLegalizeHelper::lowerExtrVecEltToSel(MachineInstr &MI) {
1040 // Lower extract vector element to a compare-select chain:
1041 // result = elt[0]
1042 // for i in 1..N-1:
1043 // result = (idx == i) ? elt[i] : result
1044 //
1045 // When the index is divergent, each lane may want a different element, so
1046 // we must check every element per lane.
1047 Register Dst = MI.getOperand(i: 0).getReg();
1048 Register Src = MI.getOperand(i: 1).getReg();
1049 Register Idx = MI.getOperand(i: 2).getReg();
1050
1051 LLT VecTy = MRI.getType(Reg: Src);
1052 LLT ScalarTy = VecTy.getScalarType();
1053 unsigned NumElts = VecTy.getNumElements();
1054 MachineRegisterInfo::VRegAttrs VgprRB_EltTy = {.RCOrRB: VgprRB, .Ty: ScalarTy};
1055
1056 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_EltTy, Op: Src);
1057
1058 if (ScalarTy.getSizeInBits() == 32) {
1059 Register PrevSelect = Unmerge.getReg(Idx: 0);
1060 for (unsigned I = 1; I < NumElts; ++I) {
1061 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1062 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1063 PrevSelect =
1064 B.buildSelect(Res: VgprRB_EltTy, Tst: Cmp, Op0: Unmerge.getReg(Idx: I), Op1: PrevSelect)
1065 .getReg(Idx: 0);
1066 }
1067 B.buildCopy(Res: Dst, Op: PrevSelect);
1068 } else if (ScalarTy.getSizeInBits() == 64) {
1069 auto InitUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: 0));
1070 Register PrevLo = InitUnmerge.getReg(Idx: 0);
1071 Register PrevHi = InitUnmerge.getReg(Idx: 1);
1072 for (unsigned I = 1; I < NumElts; ++I) {
1073 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1074 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1075 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: I));
1076 PrevLo = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 0), Op1: PrevLo)
1077 .getReg(Idx: 0);
1078 PrevHi = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 1), Op1: PrevHi)
1079 .getReg(Idx: 0);
1080 }
1081 B.buildMergeLikeInstr(Res: Dst, Ops: {PrevLo, PrevHi});
1082 } else {
1083 reportGISelFailure(
1084 MF, MORE, PassName: "amdgpu-regbanklegalize",
1085 Msg: "AMDGPU RegBankLegalize: ExtrVecEltToSel unsupported element type", MI);
1086 return false;
1087 }
1088
1089 MI.eraseFromParent();
1090 return true;
1091}
1092
1093bool RegBankLegalizeHelper::lowerExtrVecEltTo32(MachineInstr &MI) {
1094 // Reduce a 64-bit element extract to two 32-bit extracts:
1095 // vec32 = bitcast <N x s64> to <2N x s32>
1096 // lo = vec32[idx * 2]
1097 // hi = vec32[idx * 2 + 1]
1098 // result = merge(lo, hi)
1099 //
1100 // When the index is uniform, all lanes extract the same element, so we can
1101 // just split the s64 extract into two s32 extracts which lower to MOVREL.
1102 Register Dst = MI.getOperand(i: 0).getReg();
1103 Register Src = MI.getOperand(i: 1).getReg();
1104 Register Idx = MI.getOperand(i: 2).getReg();
1105
1106 LLT SrcTy = MRI.getType(Reg: Src);
1107 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1108
1109 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1110 "expected VGPR src and SGPR idx");
1111
1112 auto CastSrc = B.buildBitcast(Dst: {VgprRB, Vec32Ty}, Src);
1113
1114 // Calculate new Lo and Hi indices
1115 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1116 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1117 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1118
1119 auto ExtLo = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxLo);
1120 auto ExtHi = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxHi);
1121
1122 B.buildMergeLikeInstr(Res: Dst, Ops: {ExtLo.getReg(Idx: 0), ExtHi.getReg(Idx: 0)});
1123
1124 MI.eraseFromParent();
1125 return true;
1126}
1127
1128bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
1129 // Lower insert vector element to a compare-select chain:
1130 // for i in 0..N-1:
1131 // result[i] = (idx == i) ? elt : srcVec[i]
1132 // dst = merge(result[0..N-1])
1133 //
1134 // VGPR B64 requires splitting to lo/hi s32 pairs since there is no
1135 // v_cndmask_b64. SGPR B64/B32 and VGPR B32 can be handled natively.
1136 Register Dst = MI.getOperand(i: 0).getReg();
1137 Register Src = MI.getOperand(i: 1).getReg();
1138 Register Elt = MI.getOperand(i: 2).getReg();
1139 Register Idx = MI.getOperand(i: 3).getReg();
1140
1141 LLT VecTy = MRI.getType(Reg: Src);
1142 LLT ScalarTy = VecTy.getScalarType();
1143 unsigned NumElts = VecTy.getNumElements();
1144 const RegisterBank *SrcRB = MRI.getRegBank(Reg: Src);
1145 bool IsSGPR = (SrcRB == SgprRB);
1146 SmallVector<Register, 16> Selects;
1147
1148 if (!IsSGPR && ScalarTy.getSizeInBits() == 64) {
1149 // VGPR B64: split to 32-bit lo/hi since there is no v_cndmask_b64.
1150 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1151 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1152 Register EltLo = EltUnmerge.getReg(Idx: 0);
1153 Register EltHi = EltUnmerge.getReg(Idx: 1);
1154 for (unsigned I = 0; I < NumElts; ++I) {
1155 auto IdxConst = B.buildConstant(Res: VgprRB_I32, Val: I);
1156 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1157 Selects.push_back(
1158 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltLo, Op1: Unmerge.getReg(Idx: 2 * I))
1159 .getReg(Idx: 0));
1160 Selects.push_back(
1161 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltHi, Op1: Unmerge.getReg(Idx: 2 * I + 1))
1162 .getReg(Idx: 0));
1163 }
1164 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * NumElts, ScalarSizeInBits: 32);
1165 auto Vec32 = B.buildBuildVector(Res: {VgprRB, Vec32Ty}, Ops: Selects);
1166 B.buildBitcast(Dst, Src: Vec32);
1167 } else if (ScalarTy.getSizeInBits() == 32 || ScalarTy.getSizeInBits() == 64) {
1168 // B32 (any bank) and SGPR B64: element-wise select at native width.
1169 MachineRegisterInfo::VRegAttrs SrcRB_EltTy = {.RCOrRB: SrcRB, .Ty: ScalarTy};
1170 MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_I32 : VccRB_S1;
1171 auto Unmerge = B.buildUnmerge(Attrs: SrcRB_EltTy, Op: Src);
1172 for (unsigned I = 0; I < NumElts; ++I) {
1173 auto IdxConst = B.buildConstant(Res: SgprRB_I32, Val: I);
1174 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpTy, Op0: Idx, Op1: IdxConst);
1175 Selects.push_back(
1176 Elt: B.buildSelect(Res: SrcRB_EltTy, Tst: Cmp, Op0: Elt, Op1: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
1177 }
1178 B.buildMergeLikeInstr(Res: Dst, Ops: Selects);
1179 } else {
1180 reportGISelFailure(
1181 MF, MORE, PassName: "amdgpu-regbanklegalize",
1182 Msg: "AMDGPU RegBankLegalize: InsVecEltToSel unsupported element type", MI);
1183 return false;
1184 }
1185
1186 MI.eraseFromParent();
1187 return true;
1188}
1189
1190bool RegBankLegalizeHelper::lowerInsVecEltTo32(MachineInstr &MI) {
1191 // Reduce a 64-bit element insert to two 32-bit inserts:
1192 // vec32 = bitcast <N x s64> to <2N x s32>
1193 // lo, hi = unmerge elt
1194 // vec32[idx * 2] = lo
1195 // vec32[idx * 2 + 1] = hi
1196 // dst = bitcast <2N x s32> to <N x s64>
1197 //
1198 // When the index is uniform, all lanes insert at the same position, so we
1199 // can split the s64 insert into two s32 inserts which lower to MOVREL/GPRIDX.
1200 Register Dst = MI.getOperand(i: 0).getReg();
1201 Register Src = MI.getOperand(i: 1).getReg();
1202 Register Elt = MI.getOperand(i: 2).getReg();
1203 Register Idx = MI.getOperand(i: 3).getReg();
1204
1205 LLT SrcTy = MRI.getType(Reg: Src);
1206 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1207
1208 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1209 "expected VGPR src and SGPR idx");
1210
1211 MachineRegisterInfo::VRegAttrs VgprRB_Vec32Ty = {.RCOrRB: VgprRB, .Ty: Vec32Ty};
1212
1213 auto CastSrc = B.buildBitcast(Dst: VgprRB_Vec32Ty, Src);
1214 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1215
1216 // Calculate new Lo and Hi indices
1217 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1218 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1219 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1220
1221 auto InsLo = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: CastSrc,
1222 Elt: EltUnmerge.getReg(Idx: 0), Idx: IdxLo);
1223 auto InsHi = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: InsLo,
1224 Elt: EltUnmerge.getReg(Idx: 1), Idx: IdxHi);
1225
1226 B.buildBitcast(Dst, Src: InsHi);
1227
1228 MI.eraseFromParent();
1229 return true;
1230}
1231
1232bool RegBankLegalizeHelper::lowerAbsToNegMax(MachineInstr &MI) {
1233 // Lower divergent G_ABS to smax(x, 0 - x) in the VGPR bank:
1234 // zero = 0
1235 // neg = G_SUB zero, x
1236 // dst = G_SMAX x, neg
1237 //
1238 // There is no integer v_abs instruction on AMDGPU, so divergent G_ABS is
1239 // expanded to this sub/smax pair.
1240 Register DstReg = MI.getOperand(i: 0).getReg();
1241 Register SrcReg = MI.getOperand(i: 1).getReg();
1242 LLT Ty = MRI.getType(Reg: DstReg);
1243
1244 Register Zero;
1245 if (Ty == V2S16) {
1246 // buildConstant cannot produce a V2S16 directly; pack two S16 zeros.
1247 Register Zero16 = B.buildConstant(Res: {VgprRB, I16}, Val: 0).getReg(Idx: 0);
1248 Zero = B.buildBuildVector(Res: {VgprRB, Ty}, Ops: {Zero16, Zero16}).getReg(Idx: 0);
1249 } else {
1250 assert((Ty == S32 || Ty == S16) && "unexpected type for AbsToNegMax");
1251 Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0).getReg(Idx: 0);
1252 }
1253
1254 auto Neg = B.buildSub(Dst: {VgprRB, Ty}, Src0: Zero, Src1: SrcReg);
1255 B.buildSMax(Dst: DstReg, Src0: SrcReg, Src1: Neg);
1256 MI.eraseFromParent();
1257 return true;
1258}
1259
1260bool RegBankLegalizeHelper::lowerAbsToS32(MachineInstr &MI) {
1261 // Lower uniform V2S16 abs by unpacking the values to two separate SGPR
1262 // registers and re-emitting G_ABS on each:
1263 // packed = bitcast <2 x s16> src to s32
1264 // lo = sext_inreg packed, 16
1265 // hi = ashr packed, 16
1266 // dst = build_vector_trunc G_ABS(lo), G_ABS(hi)
1267 //
1268 // SALU only has s_abs_i32, with no direct uniform V2S16 abs. The
1269 // re-emitted G_ABS(SgprRB, S32) selects to s_abs_i32 on each value.
1270 auto Bitcast = B.buildBitcast(Dst: {SgprRB_I32}, Src: MI.getOperand(i: 1).getReg());
1271 auto SextInReg = B.buildSExtInReg(Res: {SgprRB_I32}, Op: Bitcast, ImmOp: 16);
1272 auto ShiftHi =
1273 B.buildAShr(Dst: {SgprRB_I32}, Src0: Bitcast, Src1: B.buildConstant(Res: {SgprRB_I32}, Val: 16));
1274
1275 auto AbsLo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {SextInReg});
1276 auto AbsHi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {ShiftHi});
1277 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
1278 Ops: {AbsLo.getReg(Idx: 0), AbsHi.getReg(Idx: 0)});
1279
1280 MI.eraseFromParent();
1281 return true;
1282}
1283
1284// Ported from SITargetLowering::lowerSET_ROUNDING in SIISelLowering.cpp.
1285// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1286bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
1287 Register NewMode = MI.getOperand(i: 0).getReg();
1288
1289 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
1290 // hardware MODE.fp_round values.
1291 if (auto ConstMode = getIConstantVRegValWithLookThrough(VReg: NewMode, MRI)) {
1292 uint32_t ClampedVal = std::min(
1293 a: static_cast<uint32_t>(ConstMode->Value.getZExtValue()),
1294 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
1295 uint32_t DecodedVal = AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal);
1296 NewMode = B.buildConstant(Res: SgprRB_I32, Val: DecodedVal).getReg(Idx: 0);
1297 } else {
1298 // If we know the input can only be one of the supported standard modes in
1299 // the range 0-3, we can use a simplified mapping to hardware values.
1300 KnownBits Known = VT->getKnownBits(R: NewMode);
1301 const bool UseReducedTable = Known.countMinLeadingZeros() >= 30;
1302 // The supported standard values are 0-3. The extended values start at 8. We
1303 // need to offset by 4 if the value is in the extended range.
1304
1305 if (UseReducedTable) {
1306 // Truncate to the low 32-bits.
1307 auto BitTable = B.buildConstant(
1308 Res: SgprRB_I32, Val: AMDGPU::FltRoundToHWConversionTable & 0xffff);
1309
1310 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1311 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: NewMode, Src1: Two);
1312
1313 NewMode =
1314 B.buildLShr(Dst: SgprRB_I32, Src0: BitTable, Src1: RoundModeTimesNumBits).getReg(Idx: 0);
1315
1316 // TODO: A demanded-bits simplification on the setreg source here could
1317 // likely reduce the table extracted bits into inline immediates.
1318 } else {
1319 // table_index = umin(value, value - 4)
1320 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
1321 auto NegFour = B.buildConstant(Res: SgprRB_I32, Val: -4);
1322 auto OffsetEnum = B.buildAdd(Dst: SgprRB_I32, Src0: NewMode, Src1: NegFour);
1323 auto IndexVal = B.buildUMin(Dst: SgprRB_I32, Src0: NewMode, Src1: OffsetEnum);
1324
1325 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1326 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: IndexVal, Src1: Two);
1327
1328 auto BitTable =
1329 B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundToHWConversionTable);
1330 auto TableValue =
1331 B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1332 // No need to mask out the high bits since the setreg will ignore them
1333 // anyway.
1334 NewMode = B.buildTrunc(Res: SgprRB_I32, Op: TableValue).getReg(Idx: 0);
1335 }
1336 }
1337
1338 // N.B. The setreg will be later folded into s_round_mode on supported
1339 // targets.
1340 uint32_t BothRoundHwReg =
1341 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1342 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
1343 /*HasSideEffects=*/true, /*isConvergent=*/false)
1344 .addImm(Val: static_cast<int16_t>(BothRoundHwReg))
1345 .addReg(RegNo: NewMode);
1346
1347 MI.eraseFromParent();
1348 return true;
1349}
1350
1351// Ported from SITargetLowering::lowerGET_ROUNDING in SIISelLowering.cpp.
1352// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1353bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
1354 Register Dst = MI.getOperand(i: 0).getReg();
1355
1356 uint32_t BothRoundHwReg =
1357 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1358 auto GetReg =
1359 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {SgprRB_I32},
1360 /*HasSideEffects=*/true, /*isConvergent=*/false)
1361 .addImm(Val: BothRoundHwReg);
1362
1363 // There are two rounding modes, one for f32 and one for f64/f16. We only
1364 // report in the standard value range if both are the same.
1365 //
1366 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
1367 // ties away from zero is not supported, and the other values are rotated by
1368 // 1.
1369 //
1370 // If the two rounding modes are not the same, report a target defined value.
1371
1372 // Mode register rounding mode fields:
1373 //
1374 // [1:0] Single-precision round mode.
1375 // [3:2] Double/Half-precision round mode.
1376 //
1377 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
1378 //
1379 // Hardware Spec
1380 // Toward-0 3 0
1381 // Nearest Even 0 1
1382 // +Inf 1 2
1383 // -Inf 2 3
1384 // NearestAway0 N/A 4
1385 //
1386 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
1387 // table we can index by the raw hardware mode.
1388 //
1389 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
1390 auto BitTable = B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundConversionTable);
1391
1392 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1393 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: GetReg, Src1: Two);
1394
1395 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
1396 // knew only one mode was demanded.
1397 auto TableValue = B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1398 auto TruncTable = B.buildTrunc(Res: SgprRB_I32, Op: TableValue);
1399
1400 auto EntryMask = B.buildConstant(Res: SgprRB_I32, Val: 0xf);
1401 auto TableEntry = B.buildAnd(Dst: SgprRB_I32, Src0: TruncTable, Src1: EntryMask);
1402
1403 // There's a gap in the 4-bit encoded table and actual enum values, so offset
1404 // if it's an extended value.
1405 auto Four = B.buildConstant(Res: SgprRB_I32, Val: 4);
1406 auto EnumOffset = B.buildAdd(Dst: SgprRB_I32, Src0: TableEntry, Src1: Four);
1407 auto IsStandardMode =
1408 B.buildICmp(Pred: CmpInst::ICMP_ULT, Res: SgprRB_I32, Op0: TableEntry, Op1: Four);
1409 B.buildSelect(Res: Dst, Tst: IsStandardMode, Op0: TableEntry, Op1: EnumOffset);
1410
1411 MI.eraseFromParent();
1412 return true;
1413}
1414
1415bool RegBankLegalizeHelper::lower(MachineInstr &MI,
1416 const RegBankLLTMapping &Mapping,
1417 WaterfallInfo &WFI) {
1418
1419 switch (Mapping.LoweringMethod) {
1420 case DoNotLower:
1421 break;
1422 case VccExtToSel:
1423 return lowerVccExtToSel(MI);
1424 case UniExtToSel: {
1425 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1426 auto True = B.buildConstant(Res: {SgprRB, Ty},
1427 Val: MI.getOpcode() == AMDGPU::G_SEXT ? -1 : 1);
1428 auto False = B.buildConstant(Res: {SgprRB, Ty}, Val: 0);
1429 // Input to G_{Z|S}EXT is 'Legalizer legal' S1. Most common case is compare.
1430 // We are making select here. S1 cond was already 'any-extended to S32' +
1431 // 'AND with 1 to clean high bits' by Sgpr32AExtBoolInReg.
1432 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: MI.getOperand(i: 1).getReg(), Op0: True,
1433 Op1: False);
1434 MI.eraseFromParent();
1435 return true;
1436 }
1437 case UnpackBitShift:
1438 return lowerUnpackBitShift(MI);
1439 case UnpackMinMax:
1440 return lowerUnpackMinMax(MI);
1441 case ScalarizeToS16:
1442 return lowerSplitTo16(MI);
1443 case Ext32To64: {
1444 const RegisterBank *RB = MRI.getRegBank(Reg: MI.getOperand(i: 0).getReg());
1445 MachineInstrBuilder Hi;
1446 switch (MI.getOpcode()) {
1447 case AMDGPU::G_ZEXT: {
1448 Hi = B.buildConstant(Res: {RB, I32}, Val: 0);
1449 break;
1450 }
1451 case AMDGPU::G_SEXT: {
1452 // Replicate sign bit from 32-bit extended part.
1453 auto ShiftAmt = B.buildConstant(Res: {RB, I32}, Val: 31);
1454 Hi = B.buildAShr(Dst: {RB, MRI.getType(Reg: MI.getOperand(i: 1).getReg())},
1455 Src0: MI.getOperand(i: 1).getReg(), Src1: ShiftAmt);
1456 break;
1457 }
1458 case AMDGPU::G_ANYEXT: {
1459 Hi = B.buildUndef(Res: {RB, I32});
1460 break;
1461 }
1462 default:
1463 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
1464 Msg: "AMDGPU RegBankLegalize: Ext32To64, unsuported opcode",
1465 MI);
1466 return false;
1467 }
1468
1469 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(),
1470 Ops: {MI.getOperand(i: 1).getReg(), Hi});
1471 MI.eraseFromParent();
1472 return true;
1473 }
1474 case UniCstExt: {
1475 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1476 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: ConstVal);
1477
1478 MI.eraseFromParent();
1479 return true;
1480 }
1481 case VgprToVccCopy: {
1482 Register Src = MI.getOperand(i: 1).getReg();
1483 LLT Ty = MRI.getType(Reg: Src);
1484 // Take lowest bit from each lane and put it in lane mask.
1485 // Lowering via compare, but we need to clean high bits first as compare
1486 // compares all bits in register.
1487 Register BoolSrc = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
1488 if (Ty == S64) {
1489 auto Src64 = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1490 auto One = B.buildConstant(Res: VgprRB_I32, Val: 1);
1491 auto AndLo = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 0), Src1: One);
1492 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
1493 auto AndHi = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 1), Src1: Zero);
1494 B.buildMergeLikeInstr(Res: BoolSrc, Ops: {AndLo, AndHi});
1495 } else {
1496 assert(Ty == S32 || Ty == S16);
1497 auto One = B.buildConstant(Res: {VgprRB, Ty}, Val: 1);
1498 B.buildAnd(Dst: BoolSrc, Src0: Src, Src1: One);
1499 }
1500 auto Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
1501 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 0).getReg(), Op0: BoolSrc, Op1: Zero);
1502 MI.eraseFromParent();
1503 return true;
1504 }
1505 case V_BFE:
1506 return lowerV_BFE(MI);
1507 case S_BFE:
1508 return lowerS_BFE(MI);
1509 case UniMAD64:
1510 return lowerUniMAD64(MI);
1511 case UniMul64: {
1512 B.buildMul(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2));
1513 MI.eraseFromParent();
1514 return true;
1515 }
1516 case DivSMulToMAD: {
1517 auto Op1 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 1));
1518 auto Op2 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 2));
1519 auto Zero = B.buildConstant(Res: VgprRB_I64, Val: 0);
1520
1521 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_AMDGPU_S_MUL_U64_U32
1522 ? AMDGPU::G_AMDGPU_MAD_U64_U32
1523 : AMDGPU::G_AMDGPU_MAD_I64_I32;
1524
1525 B.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0).getReg(), SgprRB_I32},
1526 SrcOps: {Op1, Op2, Zero});
1527 MI.eraseFromParent();
1528 return true;
1529 }
1530 case SplitTo32:
1531 return lowerSplitTo32(MI);
1532 case SplitTo32Mul:
1533 return lowerSplitTo32Mul(MI);
1534 case SplitTo32Select:
1535 return lowerSplitTo32Select(MI);
1536 case SplitTo32SExtInReg:
1537 return lowerSplitTo32SExtInReg(MI);
1538 case CtPop64To32: {
1539 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1540 auto LoPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 0));
1541 auto HiPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 1));
1542 // Max popcount of two 32-bit values is 64, so this add cannot overflow.
1543 B.buildAdd(Dst: MI.getOperand(i: 0).getReg(), Src0: LoPopCnt, Src1: HiPopCnt,
1544 Flags: MachineInstr::NoSWrap | MachineInstr::NoUWrap);
1545
1546 MI.eraseFromParent();
1547 break;
1548 }
1549 case S_BUF_to_BUF:
1550 return lowerSBufToBuf(MI, WFI);
1551 case SplitLoad: {
1552 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1553 unsigned Size = DstTy.getSizeInBits();
1554 // Even split to 128-bit loads
1555 if (Size > 128) {
1556 LLT B128;
1557 if (DstTy.isVector()) {
1558 LLT EltTy = DstTy.getElementType();
1559 B128 = LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1560 } else {
1561 B128 = LLT::integer(SizeInBits: 128);
1562 }
1563 if (Size / 128 == 2)
1564 splitLoad(MI, LLTBreakdown: {B128, B128});
1565 else if (Size / 128 == 4)
1566 splitLoad(MI, LLTBreakdown: {B128, B128, B128, B128});
1567 else {
1568 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
1569 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1570 MI);
1571 return false;
1572 }
1573 }
1574 // 64 and 32 bit load
1575 else if (DstTy == S96)
1576 splitLoad(MI, LLTBreakdown: {S64, S32}, MergeTy: S32);
1577 else if (DstTy == V3S32)
1578 splitLoad(MI, LLTBreakdown: {V2S32, S32}, MergeTy: S32);
1579 else if (DstTy == V6S16)
1580 splitLoad(MI, LLTBreakdown: {V4S16, V2S16}, MergeTy: V2S16);
1581 else {
1582 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
1583 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1584 MI);
1585 return false;
1586 }
1587 return true;
1588 }
1589 case DynStackAlloc: {
1590 const auto &TFI = *ST.getFrameLowering();
1591 // Guard in case the stack growth direction ever changes with scratch
1592 // instructions.
1593 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1594 "Stack grows upwards for AMDGPU");
1595
1596 Register Dst = MI.getOperand(i: 0).getReg();
1597 Register AllocSize = MI.getOperand(i: 1).getReg();
1598 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1599
1600 // Erase before building new instrs to avoid hitting multiple Dst assert
1601 // with CSE.
1602 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
1603 MI.eraseFromParent();
1604
1605 if (MRI.getRegBank(Reg: AllocSize) != SgprRB) {
1606 auto WaveReduction =
1607 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {SgprRB_I32})
1608 .addUse(RegNo: AllocSize)
1609 .addImm(Val: 0);
1610 AllocSize = WaveReduction.getReg(Idx: 0);
1611 }
1612
1613 LLT PtrTy = MRI.getType(Reg: Dst);
1614 assert(PtrTy.getSizeInBits() == 32 &&
1615 "Expected 32-bit pointer for stack allocation");
1616 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1617 Register SPReg = Info->getStackPtrOffsetReg();
1618
1619 // When using flat-scratch, the stack offset is unscaled.
1620 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1621 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1622
1623 Register AdjustedSize = AllocSize;
1624 if (!HasFlatScratch) {
1625 auto WaveSize = B.buildConstant(Res: SgprRB_I32, Val: WavefrontSizeLog2);
1626 AdjustedSize = B.buildShl(Dst: SgprRB_I32, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1627 }
1628 if (Alignment > TFI.getStackAlign()) {
1629 const uint64_t EffectiveAlignment =
1630 Alignment.value() << (HasFlatScratch ? 0 : WavefrontSizeLog2);
1631 auto OldSP = B.buildCopy(Res: {SgprRB, PtrTy}, Op: SPReg);
1632 auto Tmp1 =
1633 B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: OldSP,
1634 Op1: B.buildConstant(Res: SgprRB_I32, Val: EffectiveAlignment - 1));
1635 uint64_t Mask = maskTrailingZeros<uint64_t>(N: Log2_64(Value: EffectiveAlignment));
1636 B.buildPtrMask(Res: Dst, Op0: Tmp1, Op1: B.buildConstant(Res: SgprRB_I32, Val: Mask));
1637 } else {
1638 B.buildCopy(Res: Dst, Op: SPReg);
1639 }
1640 auto PtrAdd = B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: Dst, Op1: AdjustedSize);
1641 B.buildCopy(Res: SPReg, Op: PtrAdd);
1642 return true;
1643 }
1644 case WidenLoad: {
1645 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1646 if (DstTy == S96)
1647 widenLoad(MI, WideTy: S128);
1648 else if (DstTy == V3S32)
1649 widenLoad(MI, WideTy: V4S32, MergeTy: S32);
1650 else if (DstTy == V6S16)
1651 widenLoad(MI, WideTy: V8S16, MergeTy: V2S16);
1652 else {
1653 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
1654 Msg: "AMDGPU RegBankLegalize: WidenLoad, unsuported type",
1655 MI);
1656 return false;
1657 }
1658 return true;
1659 }
1660 case UnpackAExt:
1661 return lowerUnpackAExt(MI);
1662 case WidenMMOToS32:
1663 return widenMMOToS32(MI&: cast<GAnyLoad>(Val&: MI));
1664 case VerifyAllSgpr: {
1665 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1666 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1667 }));
1668 return true;
1669 }
1670 case ApplyAllVgpr: {
1671 assert(llvm::all_of(MI.defs(), [&](const MachineOperand &Op) {
1672 return MRI.getRegBankOrNull(Op.getReg()) == VgprRB;
1673 }));
1674 B.setInstrAndDebugLoc(MI);
1675 for (unsigned i = MI.getNumDefs(); i < MI.getNumOperands(); ++i) {
1676 MachineOperand &Op = MI.getOperand(i);
1677 if (!Op.isReg())
1678 continue;
1679 Register Reg = Op.getReg();
1680 if (MRI.getRegBank(Reg) != VgprRB) {
1681 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
1682 Op.setReg(Copy.getReg(Idx: 0));
1683 }
1684 }
1685 return true;
1686 }
1687 case UnmergeToShiftTrunc: {
1688 GUnmerge *Unmerge = dyn_cast<GUnmerge>(Val: &MI);
1689 LLT Ty = MRI.getType(Reg: Unmerge->getSourceReg());
1690 if (Ty.getSizeInBits() % 32 != 0) {
1691 reportGISelFailure(MF, MORE, PassName: "amdgpu-regbanklegalize",
1692 Msg: "AMDGPU RegBankLegalize: unmerge not multiple of 32",
1693 MI);
1694 return false;
1695 }
1696
1697 B.setInstrAndDebugLoc(MI);
1698 if (Ty.getSizeInBits() > 32) {
1699 auto UnmergeV2S16 =
1700 B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: V2S16}, Op: Unmerge->getSourceReg());
1701 for (unsigned i = 0; i < UnmergeV2S16->getNumDefs(); ++i) {
1702 auto [Dst0I32, Dst1I32] =
1703 unpackAExt(Reg: UnmergeV2S16->getOperand(i).getReg());
1704 B.buildTrunc(Res: MI.getOperand(i: i * 2).getReg(), Op: Dst0I32);
1705 B.buildTrunc(Res: MI.getOperand(i: i * 2 + 1).getReg(), Op: Dst1I32);
1706 }
1707 } else {
1708 auto [Dst0I32, Dst1I32] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
1709 B.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Dst0I32);
1710 B.buildTrunc(Res: MI.getOperand(i: 1).getReg(), Op: Dst1I32);
1711 }
1712
1713 MI.eraseFromParent();
1714 return true;
1715 }
1716 case AextToS32InIncomingBlockGPHI: {
1717 Register Dst = MI.getOperand(i: 0).getReg();
1718 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
1719 B.setInsertPt(MBB&: *MI.getParent(), II: MI.getParent()->getFirstNonPHI());
1720 MI.getOperand(i: 0).setReg(NewDst);
1721 B.buildTrunc(Res: Dst, Op: NewDst);
1722
1723 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1724 Register UseReg = MI.getOperand(i).getReg();
1725
1726 auto DefMI = MRI.getVRegDef(Reg: UseReg)->getIterator();
1727 MachineBasicBlock *DefMBB = DefMI->getParent();
1728
1729 B.setInsertPt(MBB&: *DefMBB, II: DefMBB->SkipPHIsAndLabels(I: std::next(x: DefMI)));
1730
1731 auto NewUse = B.buildAnyExt(Res: SgprRB_I32, Op: UseReg);
1732 MI.getOperand(i).setReg(NewUse.getReg(Idx: 0));
1733 }
1734 break;
1735 }
1736 case VerifyAllSgprGPHI: {
1737 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1738 if (Op.isMBB())
1739 return true;
1740 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1741 }));
1742 return true;
1743 }
1744 case VerifyAllSgprOrVgprGPHI: {
1745 assert(MRI.getRegBankOrNull(MI.getOperand(0).getReg()) == VgprRB);
1746 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1747 if (Op.isMBB())
1748 return true;
1749 const RegisterBank *RB = MRI.getRegBankOrNull(Op.getReg());
1750 return RB == VgprRB || RB == SgprRB;
1751 }));
1752 return true;
1753 }
1754 case ApplyINTRIN_IMAGE: {
1755 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
1756 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
1757 assert(RSrcIntrin && RSrcIntrin->IsImage);
1758 // The reported argument index is relative to the IR intrinsic call
1759 // arguments, so shift by the number of defs and the intrinsic ID.
1760 unsigned RsrcIdx = RSrcIntrin->RsrcArg + MI.getNumExplicitDefs() + 1;
1761 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1762 }
1763 case ApplyBVH_INTERSECT_RAY: {
1764 // Rsrc is the last register operand. Base BVH trails an A16 immediate
1765 // after rsrc; dual/BVH8 do not. Scan backwards for the last virtual
1766 // register.
1767 unsigned RsrcIdx = MI.getNumOperands();
1768 while (RsrcIdx-- > MI.getNumExplicitDefs()) {
1769 const MachineOperand &Op = MI.getOperand(i: RsrcIdx);
1770 if (Op.isReg() && Op.getReg().isVirtual())
1771 break;
1772 }
1773 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1774 }
1775 case SplitBitCount64To32:
1776 return lowerSplitBitCount64To32(MI);
1777 case ExtrVecEltToSel:
1778 return lowerExtrVecEltToSel(MI);
1779 case ExtrVecEltTo32:
1780 return lowerExtrVecEltTo32(MI);
1781 case InsVecEltToSel:
1782 return lowerInsVecEltToSel(MI);
1783 case InsVecEltTo32:
1784 return lowerInsVecEltTo32(MI);
1785 case AbsToNegMax:
1786 return lowerAbsToNegMax(MI);
1787 case AbsToS32:
1788 return lowerAbsToS32(MI);
1789 case DeletePrefetch:
1790 MI.eraseFromParent();
1791 return true;
1792 case LowerSetRounding:
1793 return lowerSetRounding(MI);
1794 case LowerGetRounding:
1795 return lowerGetRounding(MI);
1796 }
1797
1798 return true;
1799}
1800
1801LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
1802 switch (ID) {
1803 case Vcc:
1804 case UniInVcc:
1805 return LLT::scalar(SizeInBits: 1);
1806 case Sgpr16:
1807 case Vgpr16:
1808 case UniInVgprS16:
1809 return LLT::scalar(SizeInBits: 16);
1810 case Sgpr32:
1811 case Sgpr32_WF:
1812 case Sgpr32Trunc:
1813 case Sgpr32AExt:
1814 case Sgpr32AExtBoolInReg:
1815 case Sgpr32SExt:
1816 case Sgpr32ZExt:
1817 case UniInVgprS32:
1818 case Sgpr32ToVgprDst:
1819 case Vgpr32:
1820 case Vgpr32AExt:
1821 case Vgpr32SExt:
1822 case Vgpr32ZExt:
1823 return LLT::scalar(SizeInBits: 32);
1824 case Sgpr64:
1825 case Vgpr64:
1826 case UniInVgprS64:
1827 case Sgpr64ToVgprDst:
1828 return LLT::scalar(SizeInBits: 64);
1829 case Sgpr128:
1830 case Vgpr128:
1831 return LLT::scalar(SizeInBits: 128);
1832 case SgprP0:
1833 case SgprP0Call_WF:
1834 case VgprP0:
1835 return LLT::pointer(AddressSpace: 0, SizeInBits: 64);
1836 case SgprP1:
1837 case VgprP1:
1838 return LLT::pointer(AddressSpace: 1, SizeInBits: 64);
1839 case SgprP2:
1840 case VgprP2:
1841 return LLT::pointer(AddressSpace: 2, SizeInBits: 32);
1842 case SgprP3:
1843 case VgprP3:
1844 return LLT::pointer(AddressSpace: 3, SizeInBits: 32);
1845 case SgprP4:
1846 case SgprP4Call_WF:
1847 case VgprP4:
1848 return LLT::pointer(AddressSpace: 4, SizeInBits: 64);
1849 case SgprP5:
1850 case VgprP5:
1851 return LLT::pointer(AddressSpace: 5, SizeInBits: 32);
1852 case SgprP6:
1853 return LLT::pointer(AddressSpace: 6, SizeInBits: 32);
1854 case SgprP8:
1855 return LLT::pointer(AddressSpace: 8, SizeInBits: 128);
1856 case SgprV2S16:
1857 case VgprV2S16:
1858 case UniInVgprV2S16:
1859 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
1860 case SgprV2S32:
1861 case VgprV2S32:
1862 case UniInVgprV2S32:
1863 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
1864 case VgprV3S32:
1865 case UniInVgprV3S32:
1866 return LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
1867 case VgprV4S16:
1868 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
1869 case VgprV8S16:
1870 case UniInVgprV8S16:
1871 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
1872 case VgprV16S16:
1873 case UniInVgprV16S16:
1874 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
1875 case SgprV4S32:
1876 case SgprV4S32_WF:
1877 case SgprV4S32_ReadFirstLane:
1878 case VgprV4S32:
1879 case UniInVgprV4S32:
1880 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
1881 case VgprV8S32:
1882 case UniInVgprV8S32:
1883 case SgprV8S32_ReadFirstLane:
1884 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
1885 case VgprV2S64:
1886 case UniInVgprV2S64:
1887 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1888 case VgprV6S32:
1889 case UniInVgprV6S32:
1890 return LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32);
1891 case VgprV16S32:
1892 case UniInVgprV16S32:
1893 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
1894 case VgprV32S16:
1895 case UniInVgprV32S16:
1896 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
1897 case VgprV32S32:
1898 case UniInVgprV32S32:
1899 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32);
1900 default:
1901 return LLT();
1902 }
1903}
1904
1905LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
1906 switch (ID) {
1907 case SgprB32:
1908 case VgprB32:
1909 case SgprB32_M0:
1910 case SgprB32_ReadFirstLane:
1911 case UniInVgprB32:
1912 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
1913 isAnyPtr(Ty, Width: 32))
1914 return Ty;
1915 return LLT();
1916 case SgprPtr32:
1917 case VgprPtr32:
1918 return isAnyPtr(Ty, Width: 32) ? Ty : LLT();
1919 case SgprPtr64:
1920 case VgprPtr64:
1921 return isAnyPtr(Ty, Width: 64) ? Ty : LLT();
1922 case SgprPtr128:
1923 case VgprPtr128:
1924 return isAnyPtr(Ty, Width: 128) ? Ty : LLT();
1925 case SgprB64:
1926 case VgprB64:
1927 case SgprB64_ReadFirstLane:
1928 case UniInVgprB64:
1929 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
1930 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
1931 return Ty;
1932 return LLT();
1933 case SgprB96:
1934 case VgprB96:
1935 case UniInVgprB96:
1936 if (Ty == LLT::scalar(SizeInBits: 96) || Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) ||
1937 Ty == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 16))
1938 return Ty;
1939 return LLT();
1940 case SgprB128:
1941 case VgprB128:
1942 case UniInVgprB128:
1943 if (Ty.getSizeInBits() == 128)
1944 return Ty;
1945 return LLT();
1946 case VgprB160:
1947 case UniInVgprB160:
1948 if (Ty.getSizeInBits() == 160)
1949 return Ty;
1950 return LLT();
1951 case SgprB256:
1952 case VgprB256:
1953 case UniInVgprB256:
1954 if (Ty.getSizeInBits() == 256)
1955 return Ty;
1956 return LLT();
1957 case SgprB512:
1958 case VgprB512:
1959 case UniInVgprB512:
1960 if (Ty.getSizeInBits() == 512)
1961 return Ty;
1962 return LLT();
1963 case SgprBRC: {
1964 const SIRegisterInfo *TRI =
1965 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1966 unsigned LLTSize = Ty.getSizeInBits();
1967 if (LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize))
1968 return Ty;
1969 return LLT();
1970 }
1971 case VgprBRC: {
1972 const SIRegisterInfo *TRI =
1973 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1974 if (TRI->getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()))
1975 return Ty;
1976 return LLT();
1977 }
1978 default:
1979 return LLT();
1980 }
1981}
1982
1983const RegisterBank *
1984RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
1985 switch (ID) {
1986 case Vcc:
1987 return VccRB;
1988 case Sgpr16:
1989 case Sgpr32:
1990 case Sgpr32_WF:
1991 case Sgpr64:
1992 case Sgpr128:
1993 case SgprP0:
1994 case SgprP0Call_WF:
1995 case SgprP1:
1996 case SgprP2:
1997 case SgprP3:
1998 case SgprP4:
1999 case SgprP4Call_WF:
2000 case SgprP5:
2001 case SgprP6:
2002 case SgprP8:
2003 case SgprPtr32:
2004 case SgprPtr64:
2005 case SgprPtr128:
2006 case SgprV2S16:
2007 case SgprV2S32:
2008 case SgprV4S32:
2009 case SgprV4S32_WF:
2010 case SgprV4S32_ReadFirstLane:
2011 case SgprV8S32_ReadFirstLane:
2012 case SgprB32:
2013 case SgprB64:
2014 case SgprB96:
2015 case SgprB128:
2016 case SgprB256:
2017 case SgprB512:
2018 case SgprBRC:
2019 case UniInVcc:
2020 case UniInVgprS16:
2021 case UniInVgprS32:
2022 case UniInVgprS64:
2023 case UniInVgprV2S16:
2024 case UniInVgprV2S32:
2025 case UniInVgprV3S32:
2026 case UniInVgprV4S32:
2027 case UniInVgprV2S64:
2028 case UniInVgprV6S32:
2029 case UniInVgprV8S16:
2030 case UniInVgprV8S32:
2031 case UniInVgprV16S16:
2032 case UniInVgprV16S32:
2033 case UniInVgprV32S16:
2034 case UniInVgprV32S32:
2035 case UniInVgprB32:
2036 case UniInVgprB64:
2037 case UniInVgprB96:
2038 case UniInVgprB128:
2039 case UniInVgprB160:
2040 case UniInVgprB256:
2041 case UniInVgprB512:
2042 case Sgpr32Trunc:
2043 case Sgpr32AExt:
2044 case Sgpr32AExtBoolInReg:
2045 case Sgpr32SExt:
2046 case Sgpr32ZExt:
2047 return SgprRB;
2048 case AgprAnyTy:
2049 return AgprRB;
2050 case Vgpr16:
2051 case Vgpr32:
2052 case Vgpr64:
2053 case Vgpr128:
2054 case VgprP0:
2055 case VgprP1:
2056 case VgprP2:
2057 case VgprP3:
2058 case VgprP4:
2059 case VgprP5:
2060 case VgprPtr32:
2061 case VgprPtr64:
2062 case VgprPtr128:
2063 case VgprV2S16:
2064 case VgprV2S32:
2065 case VgprV2S64:
2066 case VgprV3S32:
2067 case VgprV4S16:
2068 case VgprV8S16:
2069 case VgprV16S16:
2070 case VgprV4S32:
2071 case VgprV6S32:
2072 case VgprV8S32:
2073 case VgprV16S32:
2074 case VgprV32S16:
2075 case VgprV32S32:
2076 case VgprB32:
2077 case VgprB64:
2078 case VgprB96:
2079 case VgprB128:
2080 case VgprB160:
2081 case VgprB256:
2082 case VgprB512:
2083 case VgprBRC:
2084 case VgprAnyTy:
2085 case Vgpr32AExt:
2086 case Vgpr32SExt:
2087 case Vgpr32ZExt:
2088 case Sgpr32ToVgprDst:
2089 case Sgpr64ToVgprDst:
2090 return VgprRB;
2091 default:
2092 return nullptr;
2093 }
2094}
2095
2096bool RegBankLegalizeHelper::applyMappingDst(
2097 MachineInstr &MI, unsigned &OpIdx,
2098 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs) {
2099 // Defs start from operand 0
2100 for (; OpIdx < MethodIDs.size(); ++OpIdx) {
2101 if (MethodIDs[OpIdx] == None)
2102 continue;
2103 MachineOperand &Op = MI.getOperand(i: OpIdx);
2104 Register Reg = Op.getReg();
2105 LLT Ty = MRI.getType(Reg);
2106 [[maybe_unused]] const RegisterBank *RB = MRI.getRegBank(Reg);
2107
2108 switch (MethodIDs[OpIdx]) {
2109 // vcc, sgpr and vgpr scalars, pointers and vectors
2110 case Vcc:
2111 case Sgpr16:
2112 case Sgpr32:
2113 case Sgpr64:
2114 case Sgpr128:
2115 case SgprP0:
2116 case SgprP1:
2117 case SgprP3:
2118 case SgprP4:
2119 case SgprP5:
2120 case SgprP6:
2121 case SgprP8:
2122 case SgprV2S16:
2123 case SgprV2S32:
2124 case SgprV4S32:
2125 case Vgpr16:
2126 case Vgpr32:
2127 case Vgpr64:
2128 case Vgpr128:
2129 case VgprP0:
2130 case VgprP1:
2131 case VgprP2:
2132 case VgprP3:
2133 case VgprP4:
2134 case VgprP5:
2135 case VgprV2S16:
2136 case VgprV2S32:
2137 case VgprV2S64:
2138 case VgprV3S32:
2139 case VgprV4S16:
2140 case VgprV8S16:
2141 case VgprV16S16:
2142 case VgprV4S32:
2143 case VgprV6S32:
2144 case VgprV8S32:
2145 case VgprV16S32:
2146 case VgprV32S16:
2147 case VgprV32S32: {
2148 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2149 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2150 break;
2151 }
2152 // sgpr and vgpr B-types
2153 case SgprB32:
2154 case SgprB64:
2155 case SgprB96:
2156 case SgprB128:
2157 case SgprB256:
2158 case SgprB512:
2159 case SgprBRC:
2160 case SgprPtr32:
2161 case SgprPtr64:
2162 case SgprPtr128:
2163 case VgprB32:
2164 case VgprB64:
2165 case VgprB96:
2166 case VgprB128:
2167 case VgprB160:
2168 case VgprB256:
2169 case VgprB512:
2170 case VgprBRC:
2171 case VgprPtr32:
2172 case VgprPtr64:
2173 case VgprPtr128: {
2174 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2175 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2176 break;
2177 }
2178 case VgprAnyTy: {
2179 assert(RB == VgprRB);
2180 break;
2181 }
2182 case AgprAnyTy: {
2183 if (RB == AgprRB)
2184 break;
2185 Register NewAgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: AgprRB, .Ty: Ty});
2186 Op.setReg(NewAgprDst);
2187 if (!MRI.use_nodbg_empty(RegNo: Reg))
2188 B.buildCopy(Res: Reg, Op: NewAgprDst);
2189 break;
2190 }
2191 case VgprOrAgprAnyTy: {
2192 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2193 const RegisterBank *DstRB =
2194 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2195 if (RB == DstRB)
2196 break;
2197 Register NewDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: DstRB, .Ty: Ty});
2198 Op.setReg(NewDst);
2199 if (!MRI.use_nodbg_empty(RegNo: Reg))
2200 B.buildCopy(Res: Reg, Op: NewDst);
2201 break;
2202 }
2203 // uniform in vcc/vgpr: scalars, vectors and B-types
2204 case UniInVcc: {
2205 assert(Ty == S1);
2206 assert(RB == SgprRB);
2207 Register NewDst = MRI.createVirtualRegister(RegAttr: VccRB_S1);
2208 Op.setReg(NewDst);
2209 if (!MRI.use_empty(RegNo: Reg)) {
2210 auto CopyS32_Vcc =
2211 B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_SCC_VCC, DstOps: {SgprRB_I32}, SrcOps: {NewDst});
2212 B.buildTrunc(Res: Reg, Op: CopyS32_Vcc);
2213 }
2214 break;
2215 }
2216 case UniInVgprS16: {
2217 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2218 assert(RB == SgprRB);
2219 Register NewVgprDst16 = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2220 Register NewVgprDstI32 = MRI.createVirtualRegister(RegAttr: VgprRB_I32);
2221 Register NewSgprDstI32 = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2222 Op.setReg(NewVgprDst16);
2223 B.buildAnyExt(Res: NewVgprDstI32, Op: NewVgprDst16);
2224 buildReadAnyLane(B, SgprDst: NewSgprDstI32, VgprSrc: NewVgprDstI32, RBI);
2225 B.buildTrunc(Res: Reg, Op: NewSgprDstI32);
2226 break;
2227 }
2228 case UniInVgprS32:
2229 case UniInVgprS64:
2230 case UniInVgprV2S16:
2231 case UniInVgprV2S32:
2232 case UniInVgprV3S32:
2233 case UniInVgprV4S32:
2234 case UniInVgprV2S64:
2235 case UniInVgprV6S32:
2236 case UniInVgprV8S16:
2237 case UniInVgprV8S32:
2238 case UniInVgprV16S16:
2239 case UniInVgprV16S32:
2240 case UniInVgprV32S16:
2241 case UniInVgprV32S32: {
2242 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2243 assert(RB == SgprRB);
2244 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2245 Op.setReg(NewVgprDst);
2246 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2247 break;
2248 }
2249 case UniInVgprB32:
2250 case UniInVgprB64:
2251 case UniInVgprB96:
2252 case UniInVgprB128:
2253 case UniInVgprB160:
2254 case UniInVgprB256:
2255 case UniInVgprB512: {
2256 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2257 assert(RB == SgprRB);
2258 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2259 Op.setReg(NewVgprDst);
2260 AMDGPU::buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2261 break;
2262 }
2263 // sgpr trunc
2264 case Sgpr32Trunc: {
2265 assert(Ty.getSizeInBits() < 32);
2266 assert(RB == SgprRB);
2267 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2268 Op.setReg(NewDst);
2269 if (!MRI.use_empty(RegNo: Reg))
2270 B.buildTrunc(Res: Reg, Op: NewDst);
2271 break;
2272 }
2273 case Sgpr32ToVgprDst:
2274 case Sgpr64ToVgprDst: {
2275 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2276 assert(RB == VgprRB);
2277 Op.setReg(MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty}));
2278 B.buildCopy(Res: Reg, Op: Op.getReg());
2279 break;
2280 }
2281 case InvalidMapping: {
2282 reportGISelFailure(
2283 MF, MORE, PassName: "amdgpu-regbanklegalize",
2284 Msg: "AMDGPU RegBankLegalize: missing fast rule ('Div' or 'Uni') for", MI);
2285 return false;
2286 }
2287 default:
2288 reportGISelFailure(
2289 MF, MORE, PassName: "amdgpu-regbanklegalize",
2290 Msg: "AMDGPU RegBankLegalize: applyMappingDst, ID not supported", MI);
2291 return false;
2292 }
2293 }
2294
2295 return true;
2296}
2297
2298bool RegBankLegalizeHelper::applyMappingSrc(
2299 MachineInstr &MI, unsigned &OpIdx,
2300 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
2301 WaterfallInfo &WFI) {
2302 for (unsigned i = 0; i < MethodIDs.size(); ++OpIdx, ++i) {
2303 if (MethodIDs[i] == None || MethodIDs[i] == IntrId || MethodIDs[i] == Imm)
2304 continue;
2305
2306 MachineOperand &Op = MI.getOperand(i: OpIdx);
2307 Register Reg = Op.getReg();
2308 LLT Ty = MRI.getType(Reg);
2309 const RegisterBank *RB = MRI.getRegBank(Reg);
2310
2311 switch (MethodIDs[i]) {
2312 case Vcc: {
2313 assert(Ty == S1);
2314 assert(RB == VccRB || RB == SgprRB);
2315 if (RB == SgprRB) {
2316 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2317 auto CopyVcc_Scc =
2318 B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_VCC_SCC, DstOps: {VccRB_S1}, SrcOps: {Aext});
2319 Op.setReg(CopyVcc_Scc.getReg(Idx: 0));
2320 }
2321 break;
2322 }
2323 // sgpr scalars, pointers and vectors
2324 case Sgpr16:
2325 case Sgpr32:
2326 case Sgpr64:
2327 case Sgpr128:
2328 case SgprP0:
2329 case SgprP1:
2330 case SgprP3:
2331 case SgprP4:
2332 case SgprP5:
2333 case SgprP6:
2334 case SgprP8:
2335 case SgprV2S16:
2336 case SgprV2S32:
2337 case SgprV4S32: {
2338 assert(Ty == getTyFromID(MethodIDs[i]));
2339 assert(RB == getRegBankFromID(MethodIDs[i]));
2340 break;
2341 }
2342 // sgpr B-types
2343 case SgprB32:
2344 case SgprB64:
2345 case SgprB96:
2346 case SgprB128:
2347 case SgprB256:
2348 case SgprB512:
2349 case SgprBRC:
2350 case SgprPtr32:
2351 case SgprPtr64:
2352 case SgprPtr128: {
2353 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2354 assert(RB == getRegBankFromID(MethodIDs[i]));
2355 break;
2356 }
2357 // vgpr scalars, pointers and vectors
2358 case Vgpr16:
2359 case Vgpr32:
2360 case Vgpr64:
2361 case Vgpr128:
2362 case VgprP0:
2363 case VgprP1:
2364 case VgprP2:
2365 case VgprP3:
2366 case VgprP4:
2367 case VgprP5:
2368 case VgprV2S16:
2369 case VgprV2S32:
2370 case VgprV2S64:
2371 case VgprV3S32:
2372 case VgprV4S16:
2373 case VgprV8S16:
2374 case VgprV16S16:
2375 case VgprV4S32:
2376 case VgprV6S32:
2377 case VgprV8S32:
2378 case VgprV16S32:
2379 case VgprV32S16:
2380 case VgprV32S32: {
2381 assert(Ty == getTyFromID(MethodIDs[i]));
2382 if (RB != VgprRB) {
2383 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2384 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2385 }
2386 break;
2387 }
2388 // vgpr B-types
2389 case VgprB32:
2390 case VgprB64:
2391 case VgprB96:
2392 case VgprB128:
2393 case VgprB160:
2394 case VgprB256:
2395 case VgprB512:
2396 case VgprBRC:
2397 case VgprPtr32:
2398 case VgprPtr64:
2399 case VgprPtr128: {
2400 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2401 if (RB != VgprRB) {
2402 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2403 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2404 }
2405 break;
2406 }
2407 case VgprAnyTy: {
2408 if (RB != VgprRB) {
2409 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2410 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2411 }
2412 break;
2413 }
2414 case AgprAnyTy: {
2415 if (RB != AgprRB) {
2416 auto CopyToAgpr = B.buildCopy(Res: {AgprRB, Ty}, Op: Reg);
2417 Op.setReg(CopyToAgpr.getReg(Idx: 0));
2418 }
2419 break;
2420 }
2421 case VgprOrAgprAnyTy: {
2422 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2423 const RegisterBank *SrcRB =
2424 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2425 if (RB != SrcRB)
2426 Op.setReg(B.buildCopy(Res: {SrcRB, Ty}, Op: Reg).getReg(Idx: 0));
2427 break;
2428 }
2429 // sgpr waterfall, scalars, and vectors
2430 case Sgpr32_WF:
2431 case SgprV4S32_WF: {
2432 assert(Ty == getTyFromID(MethodIDs[i]));
2433 if (RB != SgprRB) {
2434 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2435 if (!WFI.Start.isValid()) {
2436 WFI.Start = MI.getIterator();
2437 WFI.End = std::next(x: MI.getIterator());
2438 }
2439 }
2440 break;
2441 }
2442 case SgprP0Call_WF:
2443 case SgprP4Call_WF: {
2444 assert(Ty == getTyFromID(MethodIDs[i]));
2445 if (RB != SgprRB) {
2446 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2447
2448 // Find the ADJCALLSTACKUP before the call.
2449 MachineBasicBlock::iterator Start = MI.getIterator();
2450 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
2451 --Start;
2452
2453 // Find the ADJCALLSTACKDOWN after the call (include it in range).
2454 MachineBasicBlock::iterator End = MI.getIterator();
2455 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
2456 ++End;
2457 ++End;
2458
2459 WFI.Start = Start;
2460 WFI.End = End;
2461 }
2462 break;
2463 }
2464 case SgprB32_M0:
2465 case SgprB32_ReadFirstLane:
2466 case SgprB64_ReadFirstLane: {
2467 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2468 if (RB == SgprRB)
2469 break;
2470 assert(RB == VgprRB);
2471 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2472 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2473 Op.setReg(NewSGPR);
2474 break;
2475 }
2476 case SgprV4S32_ReadFirstLane:
2477 case SgprV8S32_ReadFirstLane: {
2478 assert(Ty == getTyFromID(MethodIDs[i]));
2479 if (RB == SgprRB)
2480 break;
2481 assert(RB == VgprRB);
2482 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2483 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2484 Op.setReg(NewSGPR);
2485 break;
2486 }
2487 // sgpr and vgpr scalars with extend
2488 case Sgpr32AExt: {
2489 // Note: this ext allows S1, and it is meant to be combined away.
2490 assert(Ty.getSizeInBits() < 32);
2491 assert(RB == SgprRB);
2492 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2493 Op.setReg(Aext.getReg(Idx: 0));
2494 break;
2495 }
2496 case Sgpr32AExtBoolInReg: {
2497 // Note: this ext allows S1, and it is meant to be combined away.
2498 assert(Ty.getSizeInBits() == 1);
2499 assert(RB == SgprRB);
2500 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2501 // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
2502 // most of times meant to be combined away in AMDGPURegBankCombiner.
2503 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2504 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2505 Op.setReg(BoolInReg.getReg(Idx: 0));
2506 break;
2507 }
2508 case Sgpr32SExt: {
2509 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2510 assert(RB == SgprRB);
2511 auto Sext = B.buildSExt(Res: SgprRB_I32, Op: Reg);
2512 Op.setReg(Sext.getReg(Idx: 0));
2513 break;
2514 }
2515 case Sgpr32ZExt: {
2516 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2517 assert(RB == SgprRB);
2518 auto Zext = B.buildZExt(Res: SgprRB_I32, Op: Reg);
2519 Op.setReg(Zext.getReg(Idx: 0));
2520 break;
2521 }
2522 case Vgpr32AExt: {
2523 assert(Ty.getSizeInBits() < 32);
2524 assert(RB == VgprRB);
2525 auto Aext = B.buildAnyExt(Res: VgprRB_I32, Op: Reg);
2526 Op.setReg(Aext.getReg(Idx: 0));
2527 break;
2528 }
2529 case Vgpr32SExt: {
2530 // Note this ext allows S1, and it is meant to be combined away.
2531 assert(Ty.getSizeInBits() < 32);
2532 assert(RB == VgprRB);
2533 auto Sext = B.buildSExt(Res: VgprRB_I32, Op: Reg);
2534 Op.setReg(Sext.getReg(Idx: 0));
2535 break;
2536 }
2537 case Vgpr32ZExt: {
2538 // Note this ext allows S1, and it is meant to be combined away.
2539 assert(Ty.getSizeInBits() < 32);
2540 assert(RB == VgprRB);
2541 auto Zext = B.buildZExt(Res: VgprRB_I32, Op: Reg);
2542 Op.setReg(Zext.getReg(Idx: 0));
2543 break;
2544 }
2545 default:
2546 reportGISelFailure(
2547 MF, MORE, PassName: "amdgpu-regbanklegalize",
2548 Msg: "AMDGPU RegBankLegalize: applyMappingSrc, ID not supported", MI);
2549 return false;
2550 }
2551 }
2552 return true;
2553}
2554
2555[[maybe_unused]] static bool verifyRegBankOnOperands(MachineInstr &MI,
2556 const RegisterBank *RB,
2557 MachineRegisterInfo &MRI,
2558 unsigned StartOpIdx,
2559 unsigned EndOpIdx) {
2560 for (unsigned i = StartOpIdx; i <= EndOpIdx; ++i) {
2561 if (MRI.getRegBankOrNull(Reg: MI.getOperand(i).getReg()) != RB)
2562 return false;
2563 }
2564 return true;
2565}
2566
2567bool RegBankLegalizeHelper::applyRegisterBanksVgprWithSgprRsrc(
2568 MachineInstr &MI, unsigned RsrcIdx) {
2569 const unsigned NumDefs = MI.getNumExplicitDefs();
2570
2571 MachineBasicBlock *MBB = MI.getParent();
2572 B.setInsertPt(MBB&: *MBB, II: MBB->SkipPHIsAndLabels(I: std::next(x: MI.getIterator())));
2573
2574 // Defs are vgpr.
2575 for (unsigned i = 0; i < NumDefs; ++i) {
2576 Register Reg = MI.getOperand(i).getReg();
2577 if (MRI.getRegBank(Reg) == VgprRB)
2578 continue;
2579
2580 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: MRI.getType(Reg)});
2581 MI.getOperand(i).setReg(NewVgprDst);
2582 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2583 }
2584
2585 B.setInstrAndDebugLoc(MI);
2586
2587 // Register uses before RsrcIdx are vgpr.
2588 for (unsigned i = NumDefs; i < RsrcIdx; ++i) {
2589 MachineOperand &Op = MI.getOperand(i);
2590 if (!Op.isReg())
2591 continue;
2592
2593 Register Reg = Op.getReg();
2594 if (!Reg.isVirtual())
2595 continue;
2596
2597 if (MRI.getRegBank(Reg) == VgprRB)
2598 continue;
2599
2600 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
2601 Op.setReg(Copy.getReg(Idx: 0));
2602 }
2603
2604 SmallSet<Register, 4> OpsToWaterfall;
2605
2606 // Register use RsrcIdx (and later register operands) is sgpr.
2607 for (unsigned i = RsrcIdx; i < MI.getNumOperands(); ++i) {
2608 MachineOperand &Op = MI.getOperand(i);
2609 if (!Op.isReg())
2610 continue;
2611
2612 Register Reg = Op.getReg();
2613 if (MRI.getRegBank(Reg) != SgprRB)
2614 OpsToWaterfall.insert(V: Reg);
2615 }
2616
2617 if (!OpsToWaterfall.empty()) {
2618 MachineBasicBlock::iterator MII = MI.getIterator();
2619 executeInWaterfallLoop(B, WFI: {.SgprWaterfallOperandRegs: OpsToWaterfall, .Start: MII, .End: std::next(x: MII)});
2620 }
2621
2622 return true;
2623}
2624