1//===-- AMDGPURegBankLegalizeHelper.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Implements actual lowering algorithms for each ID that can be used in
10/// Rule.OperandMapping. Similar to legalizer helper but with register banks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPURegBankLegalizeHelper.h"
15#include "AMDGPUGlobalISelUtils.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPURegBankLegalizeRules.h"
19#include "AMDGPURegisterBankInfo.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineInstr.h"
27#include "llvm/CodeGen/MachineUniformityAnalysis.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29
30#define DEBUG_TYPE "amdgpu-reg-bank-legalize"
31
32using namespace llvm;
33using namespace AMDGPU;
34
35RegBankLegalizeHelper::RegBankLegalizeHelper(
36 MachineIRBuilder &B, const MachineUniformityInfo &MUI,
37 GISelValueTracking *VT, const RegisterBankInfo &RBI,
38 const RegBankLegalizeRules &RBLRules)
39 : MF(B.getMF()), MFI(MF.getInfo<SIMachineFunctionInfo>()),
40 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()), B(B),
41 MRI(*B.getMRI()), MUI(MUI), VT(VT), RBI(RBI), MORE(MF, nullptr),
42 RBLRules(RBLRules), IsWave32(ST.isWave32()),
43 SgprRB(&RBI.getRegBank(ID: AMDGPU::SGPRRegBankID)),
44 VgprRB(&RBI.getRegBank(ID: AMDGPU::VGPRRegBankID)),
45 AgprRB(&RBI.getRegBank(ID: AMDGPU::AGPRRegBankID)),
46 VccRB(&RBI.getRegBank(ID: AMDGPU::VCCRegBankID)) {}
47
48bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
49 const SetOfRulesForOpcode *RuleSet = RBLRules.getRulesForOpc(MI);
50 if (!RuleSet) {
51 reportGISelFailure(MF, MORE, DEBUG_TYPE,
52 Msg: "No AMDGPU RegBankLegalize rules defined for opcode",
53 MI);
54 return false;
55 }
56
57 const RegBankLLTMapping *Mapping = RuleSet->findMappingForMI(MI, MRI, MUI);
58 if (!Mapping) {
59 reportGISelFailure(MF, MORE, DEBUG_TYPE,
60 Msg: "AMDGPU RegBankLegalize: none of the rules defined with "
61 "'Any' for MI's opcode matched MI",
62 MI);
63 return false;
64 }
65
66 WaterfallInfo WFI;
67 unsigned OpIdx = 0;
68 if (!Mapping->DstOpMapping.empty()) {
69 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
70 if (!applyMappingDst(MI, OpIdx, MethodIDs: Mapping->DstOpMapping))
71 return false;
72 }
73 if (!Mapping->SrcOpMapping.empty()) {
74 B.setInstr(MI);
75 if (!applyMappingSrc(MI, OpIdx, MethodIDs: Mapping->SrcOpMapping, WFI))
76 return false;
77 }
78
79 if (!lower(MI, Mapping: *Mapping, WFI))
80 return false;
81
82 if (!WFI.SgprWaterfallOperandRegs.empty()) {
83 if (!executeInWaterfallLoop(B, WFI))
84 return false;
85 }
86
87 return true;
88}
89
90bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
91 const WaterfallInfo &WFI) {
92 assert(WFI.Start.isValid() && WFI.End.isValid() &&
93 "Waterfall range not initialized");
94
95 // Track use registers which have already been expanded with a readfirstlane
96 // sequence. This may have multiple uses if moving a sequence.
97 DenseMap<Register, Register> WaterfalledRegMap;
98
99 MachineBasicBlock &MBB = B.getMBB();
100 MachineFunction &MF = B.getMF();
101
102 MachineBasicBlock::iterator BeginIt = WFI.Start;
103 MachineBasicBlock::iterator EndIt = WFI.End;
104
105 const SIRegisterInfo *TRI = ST.getRegisterInfo();
106 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
107 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
108
109#ifndef NDEBUG
110 const int OrigRangeSize = std::distance(BeginIt, EndIt);
111#endif
112
113 MachineRegisterInfo &MRI = *B.getMRI();
114 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
115 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
116
117 // Don't bother using generic instructions/registers for the exec mask.
118 B.setInstr(*WFI.Start);
119 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: InitSaveExecReg);
120
121 Register SavedExec = MRI.createVirtualRegister(RegClass: WaveRC);
122
123 // To insert the loop we need to split the block. Move everything before
124 // this point to a new block, and insert a new empty block before this
125 // instruction.
126 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
127 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
128 MachineBasicBlock *RestoreExecBB = MF.CreateMachineBasicBlock();
129 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
130 MachineFunction::iterator MBBI(MBB);
131 ++MBBI;
132 MF.insert(MBBI, MBB: LoopBB);
133 MF.insert(MBBI, MBB: BodyBB);
134 MF.insert(MBBI, MBB: RestoreExecBB);
135 MF.insert(MBBI, MBB: RemainderBB);
136
137 LoopBB->addSuccessor(Succ: BodyBB);
138 BodyBB->addSuccessor(Succ: RestoreExecBB);
139 BodyBB->addSuccessor(Succ: LoopBB);
140
141 // Move the rest of the block into a new block.
142 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
143 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: EndIt, To: MBB.end());
144
145 MBB.addSuccessor(Succ: LoopBB);
146 RestoreExecBB->addSuccessor(Succ: RemainderBB);
147
148 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
149
150 // +-MBB:------------+
151 // | ... |
152 // | %0 = G_INST_1 |
153 // | %Dst = MI %Vgpr |
154 // | %1 = G_INST_2 |
155 // | ... |
156 // +-----------------+
157 // ->
158 // +-MBB-------------------------------+
159 // | ... |
160 // | %0 = G_INST_1 |
161 // | %SaveExecReg = S_MOV_B32 $exec_lo |
162 // +----------------|------------------+
163 // | /------------------------------|
164 // V V |
165 // +-LoopBB---------------------------------------------------------------+ |
166 // | %CurrentLaneReg:sgpr(s32) = READFIRSTLANE %Vgpr | |
167 // | instead of executing for each lane, see if other lanes had | |
168 // | same value for %Vgpr and execute for them also. | |
169 // | %CondReg:vcc(s1) = G_ICMP eq %CurrentLaneReg, %Vgpr | |
170 // | %CondRegLM:sreg_32 = ballot %CondReg // copy vcc to sreg32 lane mask | |
171 // | %SavedExec = S_AND_SAVEEXEC_B32 %CondRegLM | |
172 // | exec is active for lanes with the same "CurrentLane value" in Vgpr | |
173 // +----------------|-----------------------------------------------------+ |
174 // V |
175 // +-BodyBB------------------------------------------------------------+ |
176 // | %Dst = MI %CurrentLaneReg:sgpr(s32) | |
177 // | executed only for active lanes and written to Dst | |
178 // | $exec = S_XOR_B32 $exec, %SavedExec | |
179 // | set active lanes to 0 in SavedExec, lanes that did not write to | |
180 // | Dst yet, and set this as new exec (for READFIRSTLANE and ICMP) | |
181 // | SI_WATERFALL_LOOP LoopBB |-----|
182 // +----------------|--------------------------------------------------+
183 // V
184 // +-RestoreExecBB--------------------------+
185 // | $exec_lo = S_MOV_B32_term %SaveExecReg |
186 // +----------------|-----------------------+
187 // V
188 // +-RemainderBB:----------------------+
189 // | %1 = G_INST_2 |
190 // | ... |
191 // +---------------------------------- +
192
193 // Move the instruction into the loop body. Note we moved everything after
194 // Range.end() already into a new block, so Range.end() is no longer valid.
195 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: BeginIt, To: MBB.end());
196
197 // Figure out the iterator range after splicing the instructions.
198 MachineBasicBlock::iterator NewBegin = BeginIt;
199 auto NewEnd = BodyBB->end();
200 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
201
202 B.setMBB(*LoopBB);
203 Register CondReg;
204
205 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
206 for (MachineOperand &Op : MI.all_uses()) {
207 Register OldReg = Op.getReg();
208 if (!WFI.SgprWaterfallOperandRegs.count(V: OldReg))
209 continue;
210
211 // See if we already processed this register in another instruction in
212 // the sequence.
213 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
214 if (OldVal != WaterfalledRegMap.end()) {
215 Op.setReg(OldVal->second);
216 continue;
217 }
218
219 Register OpReg = Op.getReg();
220 LLT OpTy = MRI.getType(Reg: OpReg);
221
222 // TODO: support for agpr
223 assert(MRI.getRegBank(OpReg) == VgprRB);
224 Register CurrentLaneReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: OpTy});
225 buildReadFirstLane(B, SgprDst: CurrentLaneReg, VgprSrc: OpReg, RBI);
226
227 // Build the comparison(s), CurrentLaneReg == OpReg.
228 unsigned OpSize = OpTy.getSizeInBits();
229 unsigned PartSize = (OpSize % 64 == 0) ? 64 : 32;
230 LLT PartTy = LLT::integer(SizeInBits: PartSize);
231 unsigned NumParts = OpSize / PartSize;
232 SmallVector<Register, 8> OpParts;
233 SmallVector<Register, 8> CurrentLaneParts;
234
235 if (NumParts == 1) {
236 OpParts.push_back(Elt: OpReg);
237 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
238 } else {
239 auto UnmergeOp = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: PartTy}, Op: OpReg);
240 auto UnmergeCurrLane = B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: PartTy}, Op: CurrentLaneReg);
241 for (unsigned i = 0; i < NumParts; ++i) {
242 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
243 CurrentLaneParts.push_back(Elt: UnmergeCurrLane.getReg(Idx: i));
244 }
245 }
246
247 for (unsigned i = 0; i < NumParts; ++i) {
248 Register CmpReg = MRI.createVirtualRegister(RegAttr: VccRB_S1);
249 B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpReg, Op0: CurrentLaneParts[i], Op1: OpParts[i]);
250
251 if (!CondReg)
252 CondReg = CmpReg;
253 else
254 CondReg = B.buildAnd(Dst: VccRB_S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
255 }
256
257 Op.setReg(CurrentLaneReg);
258
259 // Make sure we don't re-process this register again.
260 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
261 }
262 }
263
264 // Copy vcc to sgpr32/64, ballot becomes a no-op during instruction selection.
265 Register CondRegLM =
266 MRI.createVirtualRegister(RegAttr: {.RCOrRB: WaveRC, .Ty: LLT::integer(SizeInBits: IsWave32 ? 32 : 64)});
267 B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot, Res: CondRegLM).addReg(RegNo: CondReg);
268
269 // Update EXEC, save the original EXEC value to SavedExec.
270 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
271 .addDef(RegNo: SavedExec)
272 .addReg(RegNo: CondRegLM, Flags: RegState::Kill)
273 .setOperandDead(3);
274 MRI.setSimpleHint(VReg: SavedExec, PrefReg: CondRegLM);
275
276 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
277
278 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
279 B.buildInstr(Opcode: LMC.XorTermOpc)
280 .addDef(RegNo: LMC.ExecReg)
281 .addReg(RegNo: LMC.ExecReg)
282 .addReg(RegNo: SavedExec)
283 .setOperandDead(3);
284
285 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
286 // s_cbranch_scc0?
287
288 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
289 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
290
291 // Save the EXEC mask before the loop.
292 B.setInsertPt(MBB, II: MBB.end());
293 B.buildInstr(Opcode: LMC.MovOpc).addDef(RegNo: SaveExecReg).addReg(RegNo: LMC.ExecReg);
294
295 // Restore the EXEC mask after the loop.
296 B.setInsertPt(MBB&: *RestoreExecBB, II: RestoreExecBB->begin());
297 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
298
299 // Set the insert point after the original instruction, so any new
300 // instructions will be in the remainder.
301 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
302
303 return true;
304}
305
306// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
307// the three offsets (voffset, soffset and instoffset)
308unsigned RegBankLegalizeHelper::setBufferOffsets(
309 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
310 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) {
311 if (std::optional<int64_t> Imm =
312 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI)) {
313 uint32_t SOffset, ImmOffset;
314 if (TII.splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
315 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
316 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
317 InstOffsetVal = ImmOffset;
318 return SOffset + ImmOffset;
319 }
320 }
321 const bool CheckNUW = ST.hasGFX1250Insts();
322 auto [Base, Offset] = AMDGPU::getBaseWithConstantOffset(
323 MRI, Reg: CombinedOffset, /*KnownBits=*/ValueTracking: nullptr,
324 /*CheckNUW=*/CheckNUW);
325 uint32_t SOffset, ImmOffset;
326 if (static_cast<int32_t>(Offset) > 0 &&
327 TII.splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
328 if (Base.isValid() && MRI.getRegBank(Reg: Base) == VgprRB) {
329 VOffsetReg = Base;
330 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
331 InstOffsetVal = ImmOffset;
332 return 0;
333 }
334 // If we have SGPR base, we can use it for soffset.
335 if (SOffset == 0) {
336 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
337 SOffsetReg = Base;
338 InstOffsetVal = ImmOffset;
339 return 0;
340 }
341 }
342 // Handle the variable sgpr + vgpr case.
343 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI);
344 if (Add && static_cast<int32_t>(Offset) >= 0 &&
345 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
346 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI);
347 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI);
348 const RegisterBank *Src0Bank = MRI.getRegBank(Reg: Src0);
349 const RegisterBank *Src1Bank = MRI.getRegBank(Reg: Src1);
350 if (Src0Bank == VgprRB && Src1Bank == SgprRB) {
351 VOffsetReg = Src0;
352 SOffsetReg = Src1;
353 return 0;
354 }
355 if (Src0Bank == SgprRB && Src1Bank == VgprRB) {
356 VOffsetReg = Src1;
357 SOffsetReg = Src0;
358 return 0;
359 }
360 }
361 // Ensure we have a VGPR for the combined offset. This could be an issue if we
362 // have an SGPR offset and a VGPR resource.
363 if (MRI.getRegBank(Reg: CombinedOffset) == VgprRB) {
364 VOffsetReg = CombinedOffset;
365 } else {
366 VOffsetReg = B.buildCopy(Res: VgprRB_I32, Op: CombinedOffset).getReg(Idx: 0);
367 }
368 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: 0).getReg(Idx: 0);
369 return 0;
370}
371
372bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
373 ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
374 MachineFunction &MF = B.getMF();
375 assert(MI.getNumMemOperands() == 1);
376 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
377 Register Dst = MI.getOperand(i: 0).getReg();
378 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
379 Register Base = MI.getOperand(i: 1).getReg();
380 LLT PtrTy = MRI.getType(Reg: Base);
381 const RegisterBank *PtrRB = MRI.getRegBankOrNull(Reg: Base);
382 LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
383 SmallVector<Register, 4> LoadPartRegs;
384
385 unsigned ByteOffset = 0;
386 for (LLT PartTy : LLTBreakdown) {
387 Register BasePlusOffset;
388 if (ByteOffset == 0) {
389 BasePlusOffset = Base;
390 } else {
391 auto Offset = B.buildConstant(Res: {PtrRB, OffsetTy}, Val: ByteOffset);
392 BasePlusOffset =
393 B.buildObjectPtrOffset(Res: {PtrRB, PtrTy}, Op0: Base, Op1: Offset).getReg(Idx: 0);
394 }
395 auto *OffsetMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: ByteOffset, Ty: PartTy);
396 auto LoadPart = B.buildLoad(Res: {DstRB, PartTy}, Addr: BasePlusOffset, MMO&: *OffsetMMO);
397 LoadPartRegs.push_back(Elt: LoadPart.getReg(Idx: 0));
398 ByteOffset += PartTy.getSizeInBytes();
399 }
400
401 if (!MergeTy.isValid()) {
402 // Loads are of same size, concat or merge them together.
403 B.buildMergeLikeInstr(Res: Dst, Ops: LoadPartRegs);
404 } else {
405 // Loads are not all of same size, need to unmerge them to smaller pieces
406 // of MergeTy type, then merge pieces to Dst.
407 SmallVector<Register, 4> MergeTyParts;
408 for (Register Reg : LoadPartRegs) {
409 if (MRI.getType(Reg) == MergeTy) {
410 MergeTyParts.push_back(Elt: Reg);
411 } else {
412 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: Reg);
413 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i)
414 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
415 }
416 }
417 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
418 }
419 MI.eraseFromParent();
420 return true;
421}
422
423bool RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
424 LLT MergeTy) {
425 MachineFunction &MF = B.getMF();
426 assert(MI.getNumMemOperands() == 1);
427 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
428 Register Dst = MI.getOperand(i: 0).getReg();
429 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
430 Register Base = MI.getOperand(i: 1).getReg();
431
432 MachineMemOperand *WideMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: 0, Ty: WideTy);
433 auto WideLoad = B.buildLoad(Res: {DstRB, WideTy}, Addr: Base, MMO&: *WideMMO);
434
435 if (WideTy.isScalar()) {
436 B.buildTrunc(Res: Dst, Op: WideLoad);
437 } else {
438 SmallVector<Register, 4> MergeTyParts;
439 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: WideLoad);
440
441 LLT DstTy = MRI.getType(Reg: Dst);
442 unsigned NumElts = DstTy.getSizeInBits() / MergeTy.getSizeInBits();
443 for (unsigned i = 0; i < NumElts; ++i) {
444 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
445 }
446 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
447 }
448 MI.eraseFromParent();
449 return true;
450}
451
452bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
453 Register Dst = MI.getDstReg();
454 Register Ptr = MI.getPointerReg();
455 MachineMemOperand &MMO = MI.getMMO();
456 unsigned MemSize = 8 * MMO.getSize().getValue();
457
458 MachineMemOperand *WideMMO = B.getMF().getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: S32);
459
460 if (MI.getOpcode() == G_LOAD) {
461 B.buildLoad(Res: Dst, Addr: Ptr, MMO&: *WideMMO);
462 } else {
463 auto Load = B.buildLoad(Res: SgprRB_I32, Addr: Ptr, MMO&: *WideMMO);
464
465 if (MI.getOpcode() == G_ZEXTLOAD) {
466 APInt Mask = APInt::getLowBitsSet(numBits: S32.getSizeInBits(), loBitsSet: MemSize);
467 auto MaskCst = B.buildConstant(Res: SgprRB_I32, Val: Mask);
468 B.buildAnd(Dst, Src0: Load, Src1: MaskCst);
469 } else {
470 assert(MI.getOpcode() == G_SEXTLOAD);
471 B.buildSExtInReg(Res: Dst, Op: Load, ImmOp: MemSize);
472 }
473 }
474
475 MI.eraseFromParent();
476 return true;
477}
478
479bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
480 Register Dst = MI.getOperand(i: 0).getReg();
481 LLT Ty = MRI.getType(Reg: Dst);
482 Register Src = MI.getOperand(i: 1).getReg();
483 unsigned Opc = MI.getOpcode();
484 int TrueExtCst = Opc == G_SEXT ? -1 : 1;
485 if (Ty == S32 || Ty == S16) {
486 auto True = B.buildConstant(Res: {VgprRB, Ty}, Val: TrueExtCst);
487 auto False = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
488 B.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
489 } else if (Ty == S64) {
490 auto True = B.buildConstant(Res: {VgprRB_I32}, Val: TrueExtCst);
491 auto False = B.buildConstant(Res: {VgprRB_I32}, Val: 0);
492 auto Lo = B.buildSelect(Res: {VgprRB_I32}, Tst: Src, Op0: True, Op1: False);
493 MachineInstrBuilder Hi;
494 switch (Opc) {
495 case G_SEXT:
496 Hi = Lo;
497 break;
498 case G_ZEXT:
499 Hi = False;
500 break;
501 case G_ANYEXT:
502 Hi = B.buildUndef(Res: {VgprRB_I32});
503 break;
504 default:
505 reportGISelFailure(
506 MF, MORE, DEBUG_TYPE,
507 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Opcode not supported", MI);
508 return false;
509 }
510
511 B.buildMergeValues(Res: Dst, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
512 } else {
513 reportGISelFailure(
514 MF, MORE, DEBUG_TYPE,
515 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Type not supported", MI);
516 return false;
517 }
518
519 MI.eraseFromParent();
520 return true;
521}
522
523std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
524 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
525 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: 0x0000ffff);
526 auto Lo = B.buildAnd(Dst: SgprRB_I32, Src0: PackedI32, Src1: Mask);
527 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
528 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
529}
530
531std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
532 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
533 auto Lo = B.buildSExtInReg(Res: SgprRB_I32, Op: PackedI32, ImmOp: 16);
534 auto Hi = B.buildAShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
535 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
536}
537
538std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
539 Register RegI32 = Reg;
540 if (MRI.getType(Reg) != I32)
541 RegI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg).getReg(Idx: 0);
542
543 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: RegI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
544 return {RegI32, Hi.getReg(Idx: 0)};
545}
546
547std::pair<Register, Register>
548RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
549 auto [Lo32, Hi32] = unpackAExt(Reg);
550 LLT EltTy = MRI.getType(Reg).getElementType();
551 return {B.buildTrunc(Res: {SgprRB, EltTy}, Op: Lo32).getReg(Idx: 0),
552 B.buildTrunc(Res: {SgprRB, EltTy}, Op: Hi32).getReg(Idx: 0)};
553}
554
555bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
556 Register Lo, Hi;
557 switch (MI.getOpcode()) {
558 case AMDGPU::G_SHL: {
559 auto [Val0, Val1] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
560 auto [Amt0, Amt1] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
561 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
562 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
563 break;
564 }
565 case AMDGPU::G_LSHR: {
566 auto [Val0, Val1] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
567 auto [Amt0, Amt1] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
568 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
569 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
570 break;
571 }
572 case AMDGPU::G_ASHR: {
573 auto [Val0, Val1] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
574 auto [Amt0, Amt1] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
575 Lo = B.buildAShr(Dst: SgprRB_I32, Src0: Val0, Src1: Amt0).getReg(Idx: 0);
576 Hi = B.buildAShr(Dst: SgprRB_I32, Src0: Val1, Src1: Amt1).getReg(Idx: 0);
577 break;
578 }
579 default:
580 reportGISelFailure(
581 MF, MORE, DEBUG_TYPE,
582 Msg: "AMDGPU RegBankLegalize: lowerUnpackBitShift, case not implemented",
583 MI);
584 return false;
585 }
586 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
587 MI.eraseFromParent();
588 return true;
589}
590
591bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
592 Register Lo, Hi;
593 switch (MI.getOpcode()) {
594 case AMDGPU::G_SMIN:
595 case AMDGPU::G_SMAX: {
596 // For signed operations, use sign extension
597 auto [Val0_Lo, Val0_Hi] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
598 auto [Val1_Lo, Val1_Hi] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
599 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
600 .getReg(Idx: 0);
601 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
602 .getReg(Idx: 0);
603 break;
604 }
605 case AMDGPU::G_UMIN:
606 case AMDGPU::G_UMAX: {
607 // For unsigned operations, use zero extension
608 auto [Val0_Lo, Val0_Hi] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
609 auto [Val1_Lo, Val1_Hi] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
610 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
611 .getReg(Idx: 0);
612 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
613 .getReg(Idx: 0);
614 break;
615 }
616 default:
617 reportGISelFailure(
618 MF, MORE, DEBUG_TYPE,
619 Msg: "AMDGPU RegBankLegalize: lowerUnpackMinMax, case not implemented", MI);
620 return false;
621 }
622 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
623 MI.eraseFromParent();
624 return true;
625}
626
627bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
628 auto [Op1Lo, Op1Hi] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
629 auto [Op2Lo, Op2Hi] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
630 auto ResLo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Lo, Op2Lo});
631 auto ResHi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Hi, Op2Hi});
632 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
633 Ops: {ResLo.getReg(Idx: 0), ResHi.getReg(Idx: 0)});
634 MI.eraseFromParent();
635 return true;
636}
637
638bool RegBankLegalizeHelper::lowerSBufToBuf(MachineInstr &MI,
639 WaterfallInfo &WFI) {
640 Register Dst = MI.getOperand(i: 0).getReg();
641 LLT Ty = MRI.getType(Reg: Dst);
642 const RegisterBank *RSrcBank = MRI.getRegBank(Reg: MI.getOperand(i: 1).getReg());
643 unsigned LoadSize = Ty.getSizeInBits();
644 int NumLoads = 1;
645 SmallVector<Register, 4> LoadParts;
646 if (LoadSize == 256 || LoadSize == 512) {
647 NumLoads = LoadSize / 128;
648 Ty = Ty.divide(Factor: NumLoads);
649 }
650 for (int I = 0; I < NumLoads; ++I)
651 LoadParts.emplace_back(Args: MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty}));
652 MachineMemOperand *OrigMMO = *MI.memoperands_begin();
653 const Align Alignment = OrigMMO->getAlign();
654 MachineFunction &MF = B.getMF();
655 Register SOffset;
656 Register VOffset;
657 int64_t ImmOffset = 0;
658 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
659 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
660 const unsigned MemSize = divideCeil(Numerator: OrigMMO->getSize().getValue(), Denominator: NumLoads);
661 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(MMO: OrigMMO, Offset: 0, Size: MemSize);
662 if (MMOOffset != 0)
663 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
664 // If only the offset is divergent, emit a MUBUF buffer load
665 // instead. We can assume that the buffer is unswizzled.
666 Register RSrc = MI.getOperand(i: 1).getReg();
667 Register VIndex = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
668 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
669 unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
670 switch (MI.getOpcode()) {
671 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
672 Opc = G_AMDGPU_BUFFER_LOAD_SBYTE;
673 break;
674 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
675 Opc = G_AMDGPU_BUFFER_LOAD_UBYTE;
676 break;
677 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
678 Opc = G_AMDGPU_BUFFER_LOAD_SSHORT;
679 break;
680 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
681 Opc = G_AMDGPU_BUFFER_LOAD_USHORT;
682 break;
683 default:
684 break;
685 }
686 for (int I = 0; I < NumLoads; ++I) {
687 B.buildInstr(Opcode: Opc)
688 .addDef(RegNo: LoadParts[I]) // vdata
689 .addUse(RegNo: RSrc) // rsrc
690 .addUse(RegNo: VIndex) // vindex
691 .addUse(RegNo: VOffset) // voffset
692 .addUse(RegNo: SOffset) // soffset
693 .addImm(Val: ImmOffset + 16 * I) // offset(imm)
694 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
695 .addImm(Val: 0) // idxen(imm)
696 .addMemOperand(MMO: MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16 * I, Size: MemSize));
697 }
698 if (NumLoads == 1)
699 B.buildCopy(Res: Dst, Op: LoadParts[0]);
700 else
701 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
702 B.setInstr(*MRI.getVRegDef(Reg: LoadParts[0]));
703 if (RSrcBank != SgprRB) {
704 WFI.SgprWaterfallOperandRegs.insert(V: RSrc);
705 WFI.Start = MRI.getVRegDef(Reg: LoadParts.front());
706 WFI.End = std::next(x: MRI.getVRegDef(Reg: LoadParts.back())->getIterator());
707 }
708 MI.eraseFromParent();
709 return true;
710}
711
712static bool isSignedBFE(MachineInstr &MI) {
713 if (GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI))
714 return (GI->is(ID: Intrinsic::amdgcn_sbfe));
715
716 return MI.getOpcode() == AMDGPU::G_SBFX;
717}
718
719bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
720 Register Dst = MI.getOperand(i: 0).getReg();
721 assert(MRI.getType(Dst) == LLT::scalar(64));
722 bool Signed = isSignedBFE(MI);
723 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
724 // Extract bitfield from Src, LSBit is the least-significant bit for the
725 // extraction (field offset) and Width is size of bitfield.
726 Register Src = MI.getOperand(i: FirstOpnd).getReg();
727 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
728 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
729 // Comments are for signed bitfield extract, similar for unsigned. x is sign
730 // bit. s is sign, l is LSB and y are remaining bits of bitfield to extract.
731
732 // Src >> LSBit Hi|Lo: x?????syyyyyyl??? -> xxxx?????syyyyyyl
733 unsigned SHROpc = Signed ? AMDGPU::G_ASHR : AMDGPU::G_LSHR;
734 auto SHRSrc = B.buildInstr(Opc: SHROpc, DstOps: {VgprRB_I64}, SrcOps: {Src, LSBit});
735
736 auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: Width, MRI);
737
738 // Expand to Src >> LSBit << (64 - Width) >> (64 - Width)
739 // << (64 - Width): Hi|Lo: xxxx?????syyyyyyl -> syyyyyyl000000000
740 // >> (64 - Width): Hi|Lo: syyyyyyl000000000 -> ssssssssssyyyyyyl
741 if (!ConstWidth) {
742 auto Amt = B.buildSub(Dst: VgprRB_I32, Src0: B.buildConstant(Res: SgprRB_I32, Val: 64), Src1: Width);
743 auto SignBit = B.buildShl(Dst: VgprRB_I64, Src0: SHRSrc, Src1: Amt);
744 B.buildInstr(Opc: SHROpc, DstOps: {Dst}, SrcOps: {SignBit, Amt});
745 MI.eraseFromParent();
746 return true;
747 }
748
749 uint64_t WidthImm = ConstWidth->Value.getZExtValue();
750 auto UnmergeSHRSrc = B.buildUnmerge(Attrs: VgprRB_I32, Op: SHRSrc);
751 Register SHRSrcLo = UnmergeSHRSrc.getReg(Idx: 0);
752 Register SHRSrcHi = UnmergeSHRSrc.getReg(Idx: 1);
753 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
754 unsigned BFXOpc = Signed ? AMDGPU::G_SBFX : AMDGPU::G_UBFX;
755
756 if (WidthImm <= 32) {
757 // SHRSrc Hi|Lo: ????????|???syyyl -> ????????|ssssyyyl
758 Register Lo = SHRSrcLo;
759 // V_BFE masks its width to 5 bits, so 32 would extract zero bits.
760 if (WidthImm < 32) {
761 Lo =
762 B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcLo, Zero, Width}).getReg(Idx: 0);
763 }
764 MachineInstrBuilder Hi;
765 if (Signed) {
766 // SHRSrc Hi|Lo: ????????|ssssyyyl -> ssssssss|ssssyyyl
767 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: B.buildConstant(Res: VgprRB_I32, Val: 31));
768 } else {
769 // SHRSrc Hi|Lo: ????????|000syyyl -> 00000000|000syyyl
770 Hi = Zero;
771 }
772 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
773 } else {
774 // SHRSrc Hi|Lo: ??????sy|yyyyyyyl -> sssssssy|yyyyyyyl
775 Register Hi = SHRSrcHi;
776 // V_BFE masks its width to 5 bits, so 64 would extract zero bits.
777 if (WidthImm < 64) {
778 auto Amt = B.buildConstant(Res: VgprRB_I32, Val: WidthImm - 32);
779 Hi = B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcHi, Zero, Amt}).getReg(Idx: 0);
780 }
781 B.buildMergeLikeInstr(Res: Dst, Ops: {SHRSrcLo, Hi});
782 }
783
784 MI.eraseFromParent();
785 return true;
786}
787
788bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
789 Register DstReg = MI.getOperand(i: 0).getReg();
790 LLT Ty = MRI.getType(Reg: DstReg);
791 bool Signed = isSignedBFE(MI);
792 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
793 Register Src = MI.getOperand(i: FirstOpnd).getReg();
794 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
795 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
796 // For uniform bit field extract there are 4 available instructions, but
797 // LSBit(field offset) and Width(size of bitfield) need to be packed in S32,
798 // field offset in low and size in high 16 bits.
799
800 // Src1 Hi16|Lo16 = Size|FieldOffset
801 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: maskTrailingOnes<unsigned>(N: 6));
802 auto FieldOffset = B.buildAnd(Dst: SgprRB_I32, Src0: LSBit, Src1: Mask);
803 auto Size = B.buildShl(Dst: SgprRB_I32, Src0: Width, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
804 auto Src1 = B.buildOr(Dst: SgprRB_I32, Src0: FieldOffset, Src1: Size);
805 unsigned Opc32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
806 unsigned Opc64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
807 unsigned Opc = Ty == S32 ? Opc32 : Opc64;
808
809 // Select machine instruction, because of reg class constraining, insert
810 // copies from reg class to reg bank.
811 auto S_BFE = B.buildInstr(Opc, DstOps: {{SgprRB, Ty}},
812 SrcOps: {B.buildCopy(Res: Ty, Op: Src), B.buildCopy(Res: I32, Op: Src1)});
813 constrainSelectedInstRegOperands(I&: *S_BFE, TII: *ST.getInstrInfo(),
814 TRI: *ST.getRegisterInfo(), RBI);
815
816 B.buildCopy(Res: DstReg, Op: S_BFE->getOperand(i: 0).getReg());
817 MI.eraseFromParent();
818 return true;
819}
820
821bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
822 Register Dst = MI.getOperand(i: 0).getReg();
823 LLT DstTy = MRI.getType(Reg: Dst);
824 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
825 LLT Ty = DstTy.divide(Factor: 2);
826 auto Op1 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 1).getReg());
827 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
828 unsigned Opc = MI.getOpcode();
829 auto Flags = MI.getFlags();
830 auto Lo =
831 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 0), Op2.getReg(Idx: 0)}, Flags);
832 auto Hi =
833 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 1), Op2.getReg(Idx: 1)}, Flags);
834 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
835 MI.eraseFromParent();
836 return true;
837}
838
839bool RegBankLegalizeHelper::lowerSplitTo32Mul(MachineInstr &MI) {
840 Register Dst = MI.getOperand(i: 0).getReg();
841 assert(MRI.getType(Dst) == S64);
842 auto Op1 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 1).getReg());
843 auto Op2 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 2).getReg());
844
845 // TODO: G_AMDGPU_MAD_* optimizations for G_MUL divergent S64 operation to
846 // match GlobalISel with old regbankselect.
847 auto Lo = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
848 auto Carry = B.buildUMulH(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
849 auto MulLo0Hi1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 1));
850 auto MulHi0Lo1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 1), Src1: Op2.getReg(Idx: 0));
851 auto Sum = B.buildAdd(Dst: VgprRB_I32, Src0: MulLo0Hi1, Src1: MulHi0Lo1);
852 auto Hi = B.buildAdd(Dst: VgprRB_I32, Src0: Sum, Src1: Carry);
853
854 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
855 MI.eraseFromParent();
856 return true;
857}
858
859bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
860 Register Dst = MI.getOperand(i: 0).getReg();
861 assert(MRI.getType(Dst) == V2S16);
862 unsigned Opc = MI.getOpcode();
863 unsigned NumOps = MI.getNumOperands();
864 auto Flags = MI.getFlags();
865
866 auto [Op1Lo, Op1Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 1).getReg());
867 LLT EltTy = MRI.getType(Reg: Dst).getElementType();
868
869 if (NumOps == 2) {
870 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo}, Flags);
871 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi}, Flags);
872 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
873 MI.eraseFromParent();
874 return true;
875 }
876
877 auto [Op2Lo, Op2Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 2).getReg());
878
879 if (NumOps == 3) {
880 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo}, Flags);
881 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi}, Flags);
882 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
883 MI.eraseFromParent();
884 return true;
885 }
886
887 assert(NumOps == 4);
888 auto [Op3Lo, Op3Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 3).getReg());
889 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo, Op3Lo}, Flags);
890 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi, Op3Hi}, Flags);
891 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
892 MI.eraseFromParent();
893 return true;
894}
895
896bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
897 Register Dst0 = MI.getOperand(i: 0).getReg();
898 Register Dst1 = MI.getOperand(i: 1).getReg();
899 Register Src0 = MI.getOperand(i: 2).getReg();
900 Register Src1 = MI.getOperand(i: 3).getReg();
901 Register Src2 = MI.getOperand(i: 4).getReg();
902
903 const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
904
905 // Keep the multiplication on the SALU.
906 Register DstLo = B.buildMul(Dst: SgprRB_I32, Src0, Src1).getReg(Idx: 0);
907 Register DstHi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
908 if (ST.hasScalarMulHiInsts()) {
909 B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {{DstHi}}, SrcOps: {Src0, Src1});
910 } else {
911 auto VSrc0 = B.buildCopy(Res: VgprRB_I32, Op: Src0);
912 auto VSrc1 = B.buildCopy(Res: VgprRB_I32, Op: Src1);
913 auto MulHi = B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {VgprRB_I32}, SrcOps: {VSrc0, VSrc1});
914 buildReadAnyLane(B, SgprDst: DstHi, VgprSrc: MulHi.getReg(Idx: 0), RBI);
915 }
916
917 // Accumulate and produce the "carry-out" bit.
918
919 // The "carry-out" is defined as bit 64 of the result when computed as a
920 // big integer. For unsigned multiply-add, this matches the usual
921 // definition of carry-out.
922 if (mi_match(R: Src2, MRI, P: MIPatternMatch::m_ZeroInt())) {
923 // No accumulate: result is just the multiplication, carry is 0.
924 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
925 B.buildConstant(Res: Dst1, Val: 0);
926 } else {
927 // Accumulate: add Src2 to the multiplication result with carry chain.
928 Register Src2Lo = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
929 Register Src2Hi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
930 B.buildUnmerge(Res: {Src2Lo, Src2Hi}, Op: Src2);
931
932 auto AddLo = B.buildUAddo(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstLo, Op1: Src2Lo);
933 auto AddHi =
934 B.buildUAdde(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstHi, Op1: Src2Hi, CarryIn: AddLo.getReg(Idx: 1));
935 B.buildMergeLikeInstr(Res: Dst0, Ops: {AddLo.getReg(Idx: 0), AddHi.getReg(Idx: 0)});
936 B.buildCopy(Res: Dst1, Op: AddHi.getReg(Idx: 1));
937 }
938
939 MI.eraseFromParent();
940 return true;
941}
942
943bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
944 Register Dst = MI.getOperand(i: 0).getReg();
945 LLT DstTy = MRI.getType(Reg: Dst);
946 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
947 (DstTy.isPointer() && DstTy.getSizeInBits() == 64));
948 LLT Ty = DstTy.isFloat() ? LLT::float32() : DstTy.divide(Factor: 2);
949 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
950 auto Op3 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 3).getReg());
951 Register Cond = MI.getOperand(i: 1).getReg();
952 Cond = B.buildFreeze(Dst: VccRB_S1, Src: Cond).getReg(Idx: 0);
953 auto Flags = MI.getFlags();
954 auto Lo =
955 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 0), Op1: Op3.getReg(Idx: 0), Flags);
956 auto Hi =
957 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 1), Op1: Op3.getReg(Idx: 1), Flags);
958
959 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
960 MI.eraseFromParent();
961 return true;
962}
963
964bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
965 auto Op1 = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
966 int Amt = MI.getOperand(i: 2).getImm();
967 Register Lo, Hi;
968 // Hi|Lo: s sign bit, ?/x bits changed/not changed by sign-extend
969 if (Amt <= 32) {
970 auto Freeze = B.buildFreeze(Dst: VgprRB_I32, Src: Op1.getReg(Idx: 0));
971 if (Amt == 32) {
972 // Hi|Lo: ????????|sxxxxxxx -> ssssssss|sxxxxxxx
973 Lo = Freeze.getReg(Idx: 0);
974 } else {
975 // Hi|Lo: ????????|???sxxxx -> ssssssss|ssssxxxx
976 Lo = B.buildSExtInReg(Res: VgprRB_I32, Op: Freeze, ImmOp: Amt).getReg(Idx: 0);
977 }
978
979 auto SignExtCst = B.buildConstant(Res: SgprRB_I32, Val: 31);
980 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: SignExtCst).getReg(Idx: 0);
981 } else {
982 // Hi|Lo: ?????sxx|xxxxxxxx -> ssssssxx|xxxxxxxx
983 Lo = Op1.getReg(Idx: 0);
984 Hi = B.buildSExtInReg(Res: VgprRB_I32, Op: Op1.getReg(Idx: 1), ImmOp: Amt - 32).getReg(Idx: 0);
985 }
986
987 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
988 MI.eraseFromParent();
989 return true;
990}
991
992bool RegBankLegalizeHelper::lowerSplitBitCount64To32(MachineInstr &MI) {
993 // Split 64-bit find-first-bit operations into 32-bit halves:
994 // (ffbh hi:lo) -> umin(ffbh(hi), uaddsat(ffbh(lo), 32))
995 // (ffbl hi:lo) -> umin(ffbl(lo), uaddsat(ffbl(hi), 32))
996 // (ctlz_zero_poison hi:lo) -> umin(ffbh(hi), add(ffbh(lo), 32))
997 // (cttz_zero_poison hi:lo) -> umin(ffbl(lo), add(ffbl(hi), 32))
998 unsigned Opc = MI.getOpcode();
999
1000 // FFBH/FFBL return 0xFFFFFFFF on zero input, using uaddsat to avoid
1001 // wrapping. CTLZ/CTTZ guarantee non-zero input (zero_poison), so plain add
1002 // is fine.
1003 unsigned FFBOpc;
1004 unsigned AddOpc;
1005 bool SearchFromMSB;
1006 switch (Opc) {
1007 case AMDGPU::G_AMDGPU_FFBH_U32:
1008 FFBOpc = Opc;
1009 AddOpc = AMDGPU::G_UADDSAT;
1010 SearchFromMSB = true;
1011 break;
1012 case AMDGPU::G_AMDGPU_FFBL_B32:
1013 FFBOpc = Opc;
1014 AddOpc = AMDGPU::G_UADDSAT;
1015 SearchFromMSB = false;
1016 break;
1017 case AMDGPU::G_CTLZ_ZERO_POISON:
1018 FFBOpc = AMDGPU::G_AMDGPU_FFBH_U32;
1019 AddOpc = AMDGPU::G_ADD;
1020 SearchFromMSB = true;
1021 break;
1022 case AMDGPU::G_CTTZ_ZERO_POISON:
1023 FFBOpc = AMDGPU::G_AMDGPU_FFBL_B32;
1024 AddOpc = AMDGPU::G_ADD;
1025 SearchFromMSB = false;
1026 break;
1027 default:
1028 llvm_unreachable("unexpected opcode in lowerSplitBitCount64To32");
1029 }
1030
1031 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1032 Register Lo = Unmerge.getReg(Idx: 0);
1033 Register Hi = Unmerge.getReg(Idx: 1);
1034
1035 // MSB-first (FFBH/CTLZ) searches hi first; LSB-first (FFBL/CTTZ) searches
1036 // lo first. The secondary half adds 32 to account for the primary half's
1037 // width.
1038 auto Primary = B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Hi : Lo});
1039 auto Secondary =
1040 B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Lo : Hi});
1041
1042 auto Adjusted = B.buildInstr(Opc: AddOpc, DstOps: {VgprRB_I32},
1043 SrcOps: {Secondary, B.buildConstant(Res: VgprRB_I32, Val: 32)});
1044 B.buildUMin(Dst: MI.getOperand(i: 0).getReg(), Src0: Primary, Src1: Adjusted);
1045
1046 MI.eraseFromParent();
1047 return true;
1048}
1049
1050bool RegBankLegalizeHelper::lowerExtrVecEltToSel(MachineInstr &MI) {
1051 // Lower extract vector element to a compare-select chain:
1052 // result = elt[0]
1053 // for i in 1..N-1:
1054 // result = (idx == i) ? elt[i] : result
1055 //
1056 // When the index is divergent, each lane may want a different element, so
1057 // we must check every element per lane.
1058 Register Dst = MI.getOperand(i: 0).getReg();
1059 Register Src = MI.getOperand(i: 1).getReg();
1060 Register Idx = MI.getOperand(i: 2).getReg();
1061
1062 LLT VecTy = MRI.getType(Reg: Src);
1063 LLT ScalarTy = VecTy.getScalarType();
1064 unsigned NumElts = VecTy.getNumElements();
1065 MachineRegisterInfo::VRegAttrs VgprRB_EltTy = {.RCOrRB: VgprRB, .Ty: ScalarTy};
1066
1067 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_EltTy, Op: Src);
1068
1069 if (ScalarTy.getSizeInBits() == 32) {
1070 Register PrevSelect = Unmerge.getReg(Idx: 0);
1071 for (unsigned I = 1; I < NumElts; ++I) {
1072 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1073 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1074 PrevSelect =
1075 B.buildSelect(Res: VgprRB_EltTy, Tst: Cmp, Op0: Unmerge.getReg(Idx: I), Op1: PrevSelect)
1076 .getReg(Idx: 0);
1077 }
1078 B.buildCopy(Res: Dst, Op: PrevSelect);
1079 } else if (ScalarTy.getSizeInBits() == 64) {
1080 auto InitUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: 0));
1081 Register PrevLo = InitUnmerge.getReg(Idx: 0);
1082 Register PrevHi = InitUnmerge.getReg(Idx: 1);
1083 for (unsigned I = 1; I < NumElts; ++I) {
1084 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1085 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1086 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: I));
1087 PrevLo = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 0), Op1: PrevLo)
1088 .getReg(Idx: 0);
1089 PrevHi = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 1), Op1: PrevHi)
1090 .getReg(Idx: 0);
1091 }
1092 B.buildMergeLikeInstr(Res: Dst, Ops: {PrevLo, PrevHi});
1093 } else {
1094 reportGISelFailure(
1095 MF, MORE, DEBUG_TYPE,
1096 Msg: "AMDGPU RegBankLegalize: ExtrVecEltToSel unsupported element type", MI);
1097 return false;
1098 }
1099
1100 MI.eraseFromParent();
1101 return true;
1102}
1103
1104bool RegBankLegalizeHelper::lowerExtrVecEltTo32(MachineInstr &MI) {
1105 // Reduce a 64-bit element extract to two 32-bit extracts:
1106 // vec32 = bitcast <N x s64> to <2N x s32>
1107 // lo = vec32[idx * 2]
1108 // hi = vec32[idx * 2 + 1]
1109 // result = merge(lo, hi)
1110 //
1111 // When the index is uniform, all lanes extract the same element, so we can
1112 // just split the s64 extract into two s32 extracts which lower to MOVREL.
1113 Register Dst = MI.getOperand(i: 0).getReg();
1114 Register Src = MI.getOperand(i: 1).getReg();
1115 Register Idx = MI.getOperand(i: 2).getReg();
1116
1117 LLT SrcTy = MRI.getType(Reg: Src);
1118 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1119
1120 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1121 "expected VGPR src and SGPR idx");
1122
1123 auto CastSrc = B.buildBitcast(Dst: {VgprRB, Vec32Ty}, Src);
1124
1125 // Calculate new Lo and Hi indices
1126 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1127 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1128 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1129
1130 auto ExtLo = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxLo);
1131 auto ExtHi = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxHi);
1132
1133 B.buildMergeLikeInstr(Res: Dst, Ops: {ExtLo.getReg(Idx: 0), ExtHi.getReg(Idx: 0)});
1134
1135 MI.eraseFromParent();
1136 return true;
1137}
1138
1139bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
1140 // Lower insert vector element to a compare-select chain:
1141 // for i in 0..N-1:
1142 // result[i] = (idx == i) ? elt : srcVec[i]
1143 // dst = merge(result[0..N-1])
1144 //
1145 // VGPR B64 requires splitting to lo/hi s32 pairs since there is no
1146 // v_cndmask_b64. SGPR B64/B32 and VGPR B32 can be handled natively.
1147 Register Dst = MI.getOperand(i: 0).getReg();
1148 Register Src = MI.getOperand(i: 1).getReg();
1149 Register Elt = MI.getOperand(i: 2).getReg();
1150 Register Idx = MI.getOperand(i: 3).getReg();
1151
1152 LLT VecTy = MRI.getType(Reg: Src);
1153 LLT ScalarTy = VecTy.getScalarType();
1154 unsigned NumElts = VecTy.getNumElements();
1155 const RegisterBank *SrcRB = MRI.getRegBank(Reg: Src);
1156 bool IsSGPR = (SrcRB == SgprRB);
1157 SmallVector<Register, 16> Selects;
1158
1159 if (!IsSGPR && ScalarTy.getSizeInBits() == 64) {
1160 // VGPR B64: split to 32-bit lo/hi since there is no v_cndmask_b64.
1161 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1162 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1163 Register EltLo = EltUnmerge.getReg(Idx: 0);
1164 Register EltHi = EltUnmerge.getReg(Idx: 1);
1165 for (unsigned I = 0; I < NumElts; ++I) {
1166 auto IdxConst = B.buildConstant(Res: VgprRB_I32, Val: I);
1167 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1168 Selects.push_back(
1169 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltLo, Op1: Unmerge.getReg(Idx: 2 * I))
1170 .getReg(Idx: 0));
1171 Selects.push_back(
1172 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltHi, Op1: Unmerge.getReg(Idx: 2 * I + 1))
1173 .getReg(Idx: 0));
1174 }
1175 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * NumElts, ScalarSizeInBits: 32);
1176 auto Vec32 = B.buildBuildVector(Res: {VgprRB, Vec32Ty}, Ops: Selects);
1177 B.buildBitcast(Dst, Src: Vec32);
1178 } else if (ScalarTy.getSizeInBits() == 32 || ScalarTy.getSizeInBits() == 64) {
1179 // B32 (any bank) and SGPR B64: element-wise select at native width.
1180 MachineRegisterInfo::VRegAttrs SrcRB_EltTy = {.RCOrRB: SrcRB, .Ty: ScalarTy};
1181 MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_I32 : VccRB_S1;
1182 auto Unmerge = B.buildUnmerge(Attrs: SrcRB_EltTy, Op: Src);
1183 for (unsigned I = 0; I < NumElts; ++I) {
1184 auto IdxConst = B.buildConstant(Res: SgprRB_I32, Val: I);
1185 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpTy, Op0: Idx, Op1: IdxConst);
1186 Selects.push_back(
1187 Elt: B.buildSelect(Res: SrcRB_EltTy, Tst: Cmp, Op0: Elt, Op1: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
1188 }
1189 B.buildMergeLikeInstr(Res: Dst, Ops: Selects);
1190 } else {
1191 reportGISelFailure(
1192 MF, MORE, DEBUG_TYPE,
1193 Msg: "AMDGPU RegBankLegalize: InsVecEltToSel unsupported element type", MI);
1194 return false;
1195 }
1196
1197 MI.eraseFromParent();
1198 return true;
1199}
1200
1201bool RegBankLegalizeHelper::lowerInsVecEltTo32(MachineInstr &MI) {
1202 // Reduce a 64-bit element insert to two 32-bit inserts:
1203 // vec32 = bitcast <N x s64> to <2N x s32>
1204 // lo, hi = unmerge elt
1205 // vec32[idx * 2] = lo
1206 // vec32[idx * 2 + 1] = hi
1207 // dst = bitcast <2N x s32> to <N x s64>
1208 //
1209 // When the index is uniform, all lanes insert at the same position, so we
1210 // can split the s64 insert into two s32 inserts which lower to MOVREL/GPRIDX.
1211 Register Dst = MI.getOperand(i: 0).getReg();
1212 Register Src = MI.getOperand(i: 1).getReg();
1213 Register Elt = MI.getOperand(i: 2).getReg();
1214 Register Idx = MI.getOperand(i: 3).getReg();
1215
1216 LLT SrcTy = MRI.getType(Reg: Src);
1217 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1218
1219 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1220 "expected VGPR src and SGPR idx");
1221
1222 MachineRegisterInfo::VRegAttrs VgprRB_Vec32Ty = {.RCOrRB: VgprRB, .Ty: Vec32Ty};
1223
1224 auto CastSrc = B.buildBitcast(Dst: VgprRB_Vec32Ty, Src);
1225 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1226
1227 // Calculate new Lo and Hi indices
1228 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1229 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1230 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1231
1232 auto InsLo = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: CastSrc,
1233 Elt: EltUnmerge.getReg(Idx: 0), Idx: IdxLo);
1234 auto InsHi = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: InsLo,
1235 Elt: EltUnmerge.getReg(Idx: 1), Idx: IdxHi);
1236
1237 B.buildBitcast(Dst, Src: InsHi);
1238
1239 MI.eraseFromParent();
1240 return true;
1241}
1242
1243bool RegBankLegalizeHelper::lowerAbsToNegMax(MachineInstr &MI) {
1244 // Lower divergent G_ABS to smax(x, 0 - x) in the VGPR bank:
1245 // zero = 0
1246 // neg = G_SUB zero, x
1247 // dst = G_SMAX x, neg
1248 //
1249 // There is no integer v_abs instruction on AMDGPU, so divergent G_ABS is
1250 // expanded to this sub/smax pair.
1251 Register DstReg = MI.getOperand(i: 0).getReg();
1252 Register SrcReg = MI.getOperand(i: 1).getReg();
1253 LLT Ty = MRI.getType(Reg: DstReg);
1254
1255 Register Zero;
1256 if (Ty == V2S16) {
1257 // buildConstant cannot produce a V2S16 directly; pack two S16 zeros.
1258 Register Zero16 = B.buildConstant(Res: {VgprRB, I16}, Val: 0).getReg(Idx: 0);
1259 Zero = B.buildBuildVector(Res: {VgprRB, Ty}, Ops: {Zero16, Zero16}).getReg(Idx: 0);
1260 } else {
1261 assert((Ty == S32 || Ty == S16) && "unexpected type for AbsToNegMax");
1262 Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0).getReg(Idx: 0);
1263 }
1264
1265 auto Neg = B.buildSub(Dst: {VgprRB, Ty}, Src0: Zero, Src1: SrcReg);
1266 B.buildSMax(Dst: DstReg, Src0: SrcReg, Src1: Neg);
1267 MI.eraseFromParent();
1268 return true;
1269}
1270
1271bool RegBankLegalizeHelper::lowerAbsToS32(MachineInstr &MI) {
1272 // Lower uniform V2S16 abs by unpacking the values to two separate SGPR
1273 // registers and re-emitting G_ABS on each:
1274 // packed = bitcast <2 x s16> src to s32
1275 // lo = sext_inreg packed, 16
1276 // hi = ashr packed, 16
1277 // dst = build_vector_trunc G_ABS(lo), G_ABS(hi)
1278 //
1279 // SALU only has s_abs_i32, with no direct uniform V2S16 abs. The
1280 // re-emitted G_ABS(SgprRB, S32) selects to s_abs_i32 on each value.
1281 auto Bitcast = B.buildBitcast(Dst: {SgprRB_I32}, Src: MI.getOperand(i: 1).getReg());
1282 auto SextInReg = B.buildSExtInReg(Res: {SgprRB_I32}, Op: Bitcast, ImmOp: 16);
1283 auto ShiftHi =
1284 B.buildAShr(Dst: {SgprRB_I32}, Src0: Bitcast, Src1: B.buildConstant(Res: {SgprRB_I32}, Val: 16));
1285
1286 auto AbsLo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {SextInReg});
1287 auto AbsHi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {ShiftHi});
1288 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
1289 Ops: {AbsLo.getReg(Idx: 0), AbsHi.getReg(Idx: 0)});
1290
1291 MI.eraseFromParent();
1292 return true;
1293}
1294
1295// Ported from SITargetLowering::lowerSET_ROUNDING in SIISelLowering.cpp.
1296// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1297bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
1298 Register NewMode = MI.getOperand(i: 0).getReg();
1299
1300 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
1301 // hardware MODE.fp_round values.
1302 if (auto ConstMode = getIConstantVRegValWithLookThrough(VReg: NewMode, MRI)) {
1303 uint32_t ClampedVal = std::min(
1304 a: static_cast<uint32_t>(ConstMode->Value.getZExtValue()),
1305 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
1306 uint32_t DecodedVal = AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal);
1307 NewMode = B.buildConstant(Res: SgprRB_I32, Val: DecodedVal).getReg(Idx: 0);
1308 } else {
1309 // If we know the input can only be one of the supported standard modes in
1310 // the range 0-3, we can use a simplified mapping to hardware values.
1311 KnownBits Known = VT->getKnownBits(R: NewMode);
1312 const bool UseReducedTable = Known.countMinLeadingZeros() >= 30;
1313 // The supported standard values are 0-3. The extended values start at 8. We
1314 // need to offset by 4 if the value is in the extended range.
1315
1316 if (UseReducedTable) {
1317 // Truncate to the low 32-bits.
1318 auto BitTable = B.buildConstant(
1319 Res: SgprRB_I32, Val: AMDGPU::FltRoundToHWConversionTable & 0xffff);
1320
1321 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1322 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: NewMode, Src1: Two);
1323
1324 NewMode =
1325 B.buildLShr(Dst: SgprRB_I32, Src0: BitTable, Src1: RoundModeTimesNumBits).getReg(Idx: 0);
1326
1327 // TODO: A demanded-bits simplification on the setreg source here could
1328 // likely reduce the table extracted bits into inline immediates.
1329 } else {
1330 // table_index = umin(value, value - 4)
1331 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
1332 auto NegFour = B.buildConstant(Res: SgprRB_I32, Val: -4);
1333 auto OffsetEnum = B.buildAdd(Dst: SgprRB_I32, Src0: NewMode, Src1: NegFour);
1334 auto IndexVal = B.buildUMin(Dst: SgprRB_I32, Src0: NewMode, Src1: OffsetEnum);
1335
1336 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1337 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: IndexVal, Src1: Two);
1338
1339 auto BitTable =
1340 B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundToHWConversionTable);
1341 auto TableValue =
1342 B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1343 // No need to mask out the high bits since the setreg will ignore them
1344 // anyway.
1345 NewMode = B.buildTrunc(Res: SgprRB_I32, Op: TableValue).getReg(Idx: 0);
1346 }
1347 }
1348
1349 // N.B. The setreg will be later folded into s_round_mode on supported
1350 // targets.
1351 uint32_t BothRoundHwReg =
1352 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1353 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
1354 /*HasSideEffects=*/true, /*isConvergent=*/false)
1355 .addImm(Val: static_cast<int16_t>(BothRoundHwReg))
1356 .addReg(RegNo: NewMode);
1357
1358 MI.eraseFromParent();
1359 return true;
1360}
1361
1362// Ported from SITargetLowering::lowerGET_ROUNDING in SIISelLowering.cpp.
1363// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1364bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
1365 Register Dst = MI.getOperand(i: 0).getReg();
1366
1367 uint32_t BothRoundHwReg =
1368 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1369 auto GetReg =
1370 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {SgprRB_I32},
1371 /*HasSideEffects=*/true, /*isConvergent=*/false)
1372 .addImm(Val: BothRoundHwReg);
1373
1374 // There are two rounding modes, one for f32 and one for f64/f16. We only
1375 // report in the standard value range if both are the same.
1376 //
1377 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
1378 // ties away from zero is not supported, and the other values are rotated by
1379 // 1.
1380 //
1381 // If the two rounding modes are not the same, report a target defined value.
1382
1383 // Mode register rounding mode fields:
1384 //
1385 // [1:0] Single-precision round mode.
1386 // [3:2] Double/Half-precision round mode.
1387 //
1388 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
1389 //
1390 // Hardware Spec
1391 // Toward-0 3 0
1392 // Nearest Even 0 1
1393 // +Inf 1 2
1394 // -Inf 2 3
1395 // NearestAway0 N/A 4
1396 //
1397 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
1398 // table we can index by the raw hardware mode.
1399 //
1400 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
1401 auto BitTable = B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundConversionTable);
1402
1403 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1404 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: GetReg, Src1: Two);
1405
1406 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
1407 // knew only one mode was demanded.
1408 auto TableValue = B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1409 auto TruncTable = B.buildTrunc(Res: SgprRB_I32, Op: TableValue);
1410
1411 auto EntryMask = B.buildConstant(Res: SgprRB_I32, Val: 0xf);
1412 auto TableEntry = B.buildAnd(Dst: SgprRB_I32, Src0: TruncTable, Src1: EntryMask);
1413
1414 // There's a gap in the 4-bit encoded table and actual enum values, so offset
1415 // if it's an extended value.
1416 auto Four = B.buildConstant(Res: SgprRB_I32, Val: 4);
1417 auto EnumOffset = B.buildAdd(Dst: SgprRB_I32, Src0: TableEntry, Src1: Four);
1418 auto IsStandardMode =
1419 B.buildICmp(Pred: CmpInst::ICMP_ULT, Res: SgprRB_I32, Op0: TableEntry, Op1: Four);
1420 B.buildSelect(Res: Dst, Tst: IsStandardMode, Op0: TableEntry, Op1: EnumOffset);
1421
1422 MI.eraseFromParent();
1423 return true;
1424}
1425
1426bool RegBankLegalizeHelper::lower(MachineInstr &MI,
1427 const RegBankLLTMapping &Mapping,
1428 WaterfallInfo &WFI) {
1429
1430 switch (Mapping.LoweringMethod) {
1431 case DoNotLower:
1432 break;
1433 case VccExtToSel:
1434 return lowerVccExtToSel(MI);
1435 case UniExtToSel: {
1436 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1437 auto True = B.buildConstant(Res: {SgprRB, Ty},
1438 Val: MI.getOpcode() == AMDGPU::G_SEXT ? -1 : 1);
1439 auto False = B.buildConstant(Res: {SgprRB, Ty}, Val: 0);
1440 // Input to G_{Z|S}EXT is 'Legalizer legal' S1. Most common case is compare.
1441 // We are making select here. S1 cond was already 'any-extended to S32' +
1442 // 'AND with 1 to clean high bits' by Sgpr32AExtBoolInReg.
1443 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: MI.getOperand(i: 1).getReg(), Op0: True,
1444 Op1: False);
1445 MI.eraseFromParent();
1446 return true;
1447 }
1448 case UnpackBitShift:
1449 return lowerUnpackBitShift(MI);
1450 case UnpackMinMax:
1451 return lowerUnpackMinMax(MI);
1452 case ScalarizeToS16:
1453 return lowerSplitTo16(MI);
1454 case Ext32To64: {
1455 const RegisterBank *RB = MRI.getRegBank(Reg: MI.getOperand(i: 0).getReg());
1456 MachineInstrBuilder Hi;
1457 switch (MI.getOpcode()) {
1458 case AMDGPU::G_ZEXT: {
1459 Hi = B.buildConstant(Res: {RB, I32}, Val: 0);
1460 break;
1461 }
1462 case AMDGPU::G_SEXT: {
1463 // Replicate sign bit from 32-bit extended part.
1464 auto ShiftAmt = B.buildConstant(Res: {RB, I32}, Val: 31);
1465 Hi = B.buildAShr(Dst: {RB, MRI.getType(Reg: MI.getOperand(i: 1).getReg())},
1466 Src0: MI.getOperand(i: 1).getReg(), Src1: ShiftAmt);
1467 break;
1468 }
1469 case AMDGPU::G_ANYEXT: {
1470 Hi = B.buildUndef(Res: {RB, I32});
1471 break;
1472 }
1473 default:
1474 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1475 Msg: "AMDGPU RegBankLegalize: Ext32To64, unsuported opcode",
1476 MI);
1477 return false;
1478 }
1479
1480 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(),
1481 Ops: {MI.getOperand(i: 1).getReg(), Hi});
1482 MI.eraseFromParent();
1483 return true;
1484 }
1485 case UniCstExt: {
1486 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1487 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: ConstVal);
1488
1489 MI.eraseFromParent();
1490 return true;
1491 }
1492 case VgprToVccCopy: {
1493 Register Src = MI.getOperand(i: 1).getReg();
1494 LLT Ty = MRI.getType(Reg: Src);
1495 // Take lowest bit from each lane and put it in lane mask.
1496 // Lowering via compare, but we need to clean high bits first as compare
1497 // compares all bits in register.
1498 Register BoolSrc = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
1499 if (Ty == S64) {
1500 auto Src64 = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1501 auto One = B.buildConstant(Res: VgprRB_I32, Val: 1);
1502 auto AndLo = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 0), Src1: One);
1503 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
1504 auto AndHi = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 1), Src1: Zero);
1505 B.buildMergeLikeInstr(Res: BoolSrc, Ops: {AndLo, AndHi});
1506 } else {
1507 assert(Ty == S32 || Ty == S16);
1508 auto One = B.buildConstant(Res: {VgprRB, Ty}, Val: 1);
1509 B.buildAnd(Dst: BoolSrc, Src0: Src, Src1: One);
1510 }
1511 auto Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
1512 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 0).getReg(), Op0: BoolSrc, Op1: Zero);
1513 MI.eraseFromParent();
1514 return true;
1515 }
1516 case V_BFE:
1517 return lowerV_BFE(MI);
1518 case S_BFE:
1519 return lowerS_BFE(MI);
1520 case UniMAD64:
1521 return lowerUniMAD64(MI);
1522 case UniMul64: {
1523 B.buildMul(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2));
1524 MI.eraseFromParent();
1525 return true;
1526 }
1527 case DivSMulToMAD: {
1528 auto Op1 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 1));
1529 auto Op2 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 2));
1530 auto Zero = B.buildConstant(Res: VgprRB_I64, Val: 0);
1531
1532 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_AMDGPU_S_MUL_U64_U32
1533 ? AMDGPU::G_AMDGPU_MAD_U64_U32
1534 : AMDGPU::G_AMDGPU_MAD_I64_I32;
1535
1536 B.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0).getReg(), SgprRB_I32},
1537 SrcOps: {Op1, Op2, Zero});
1538 MI.eraseFromParent();
1539 return true;
1540 }
1541 case SplitTo32:
1542 return lowerSplitTo32(MI);
1543 case SplitTo32Mul:
1544 return lowerSplitTo32Mul(MI);
1545 case SplitTo32Select:
1546 return lowerSplitTo32Select(MI);
1547 case SplitTo32SExtInReg:
1548 return lowerSplitTo32SExtInReg(MI);
1549 case CtPop64To32: {
1550 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1551 auto LoPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 0));
1552 auto HiPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 1));
1553 // Max popcount of two 32-bit values is 64, so this add cannot overflow.
1554 B.buildAdd(Dst: MI.getOperand(i: 0).getReg(), Src0: LoPopCnt, Src1: HiPopCnt,
1555 Flags: MachineInstr::NoSWrap | MachineInstr::NoUWrap);
1556
1557 MI.eraseFromParent();
1558 break;
1559 }
1560 case S_BUF_to_BUF:
1561 return lowerSBufToBuf(MI, WFI);
1562 case SplitLoad: {
1563 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1564 unsigned Size = DstTy.getSizeInBits();
1565 // Even split to 128-bit loads
1566 if (Size > 128) {
1567 LLT B128;
1568 if (DstTy.isVector()) {
1569 LLT EltTy = DstTy.getElementType();
1570 B128 = LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1571 } else {
1572 B128 = LLT::integer(SizeInBits: 128);
1573 }
1574 if (Size / 128 == 2)
1575 splitLoad(MI, LLTBreakdown: {B128, B128});
1576 else if (Size / 128 == 4)
1577 splitLoad(MI, LLTBreakdown: {B128, B128, B128, B128});
1578 else {
1579 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1580 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1581 MI);
1582 return false;
1583 }
1584 }
1585 // 64 and 32 bit load
1586 else if (DstTy == S96)
1587 splitLoad(MI, LLTBreakdown: {S64, S32}, MergeTy: S32);
1588 else if (DstTy == V3S32)
1589 splitLoad(MI, LLTBreakdown: {V2S32, S32}, MergeTy: S32);
1590 else if (DstTy == V6S16)
1591 splitLoad(MI, LLTBreakdown: {V4S16, V2S16}, MergeTy: V2S16);
1592 else {
1593 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1594 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1595 MI);
1596 return false;
1597 }
1598 return true;
1599 }
1600 case DynStackAlloc: {
1601 const auto &TFI = *ST.getFrameLowering();
1602 // Guard in case the stack growth direction ever changes with scratch
1603 // instructions.
1604 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1605 "Stack grows upwards for AMDGPU");
1606
1607 Register Dst = MI.getOperand(i: 0).getReg();
1608 Register AllocSize = MI.getOperand(i: 1).getReg();
1609 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1610
1611 // Erase before building new instrs to avoid hitting multiple Dst assert
1612 // with CSE.
1613 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
1614 MI.eraseFromParent();
1615
1616 if (MRI.getRegBank(Reg: AllocSize) != SgprRB) {
1617 auto WaveReduction =
1618 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {SgprRB_I32})
1619 .addUse(RegNo: AllocSize)
1620 .addImm(Val: 0);
1621 AllocSize = WaveReduction.getReg(Idx: 0);
1622 }
1623
1624 LLT PtrTy = MRI.getType(Reg: Dst);
1625 assert(PtrTy.getSizeInBits() == 32 &&
1626 "Expected 32-bit pointer for stack allocation");
1627 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1628 Register SPReg = Info->getStackPtrOffsetReg();
1629
1630 // When using flat-scratch, the stack offset is unscaled.
1631 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1632 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1633
1634 Register AdjustedSize = AllocSize;
1635 if (!HasFlatScratch) {
1636 auto WaveSize = B.buildConstant(Res: SgprRB_I32, Val: WavefrontSizeLog2);
1637 AdjustedSize = B.buildShl(Dst: SgprRB_I32, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1638 }
1639 if (Alignment > TFI.getStackAlign()) {
1640 const uint64_t EffectiveAlignment =
1641 Alignment.value() << (HasFlatScratch ? 0 : WavefrontSizeLog2);
1642 auto OldSP = B.buildCopy(Res: {SgprRB, PtrTy}, Op: SPReg);
1643 auto Tmp1 =
1644 B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: OldSP,
1645 Op1: B.buildConstant(Res: SgprRB_I32, Val: EffectiveAlignment - 1));
1646 uint64_t Mask = maskTrailingZeros<uint64_t>(N: Log2_64(Value: EffectiveAlignment));
1647 B.buildPtrMask(Res: Dst, Op0: Tmp1, Op1: B.buildConstant(Res: SgprRB_I32, Val: Mask));
1648 } else {
1649 B.buildCopy(Res: Dst, Op: SPReg);
1650 }
1651 auto PtrAdd = B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: Dst, Op1: AdjustedSize);
1652 B.buildCopy(Res: SPReg, Op: PtrAdd);
1653 return true;
1654 }
1655 case WidenLoad: {
1656 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1657 if (DstTy == S96)
1658 widenLoad(MI, WideTy: S128);
1659 else if (DstTy == V3S32)
1660 widenLoad(MI, WideTy: V4S32, MergeTy: S32);
1661 else if (DstTy == V6S16)
1662 widenLoad(MI, WideTy: V8S16, MergeTy: V2S16);
1663 else {
1664 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1665 Msg: "AMDGPU RegBankLegalize: WidenLoad, unsuported type",
1666 MI);
1667 return false;
1668 }
1669 return true;
1670 }
1671 case UnpackAExt:
1672 return lowerUnpackAExt(MI);
1673 case WidenMMOToS32:
1674 return widenMMOToS32(MI&: cast<GAnyLoad>(Val&: MI));
1675 case VerifyAllSgpr: {
1676 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1677 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1678 }));
1679 return true;
1680 }
1681 case ApplyAllVgpr: {
1682 assert(llvm::all_of(MI.defs(), [&](const MachineOperand &Op) {
1683 return MRI.getRegBankOrNull(Op.getReg()) == VgprRB;
1684 }));
1685 B.setInstrAndDebugLoc(MI);
1686 for (unsigned i = MI.getNumDefs(); i < MI.getNumOperands(); ++i) {
1687 MachineOperand &Op = MI.getOperand(i);
1688 if (!Op.isReg())
1689 continue;
1690 Register Reg = Op.getReg();
1691 if (MRI.getRegBank(Reg) != VgprRB) {
1692 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
1693 Op.setReg(Copy.getReg(Idx: 0));
1694 }
1695 }
1696 return true;
1697 }
1698 case UnmergeToShiftTrunc: {
1699 GUnmerge *Unmerge = dyn_cast<GUnmerge>(Val: &MI);
1700 LLT Ty = MRI.getType(Reg: Unmerge->getSourceReg());
1701 if (Ty.getSizeInBits() % 32 != 0) {
1702 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1703 Msg: "AMDGPU RegBankLegalize: unmerge not multiple of 32",
1704 MI);
1705 return false;
1706 }
1707
1708 B.setInstrAndDebugLoc(MI);
1709 if (Ty.getSizeInBits() > 32) {
1710 auto UnmergeV2S16 =
1711 B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: V2S16}, Op: Unmerge->getSourceReg());
1712 for (unsigned i = 0; i < UnmergeV2S16->getNumDefs(); ++i) {
1713 auto [Dst0I32, Dst1I32] =
1714 unpackAExt(Reg: UnmergeV2S16->getOperand(i).getReg());
1715 B.buildTrunc(Res: MI.getOperand(i: i * 2).getReg(), Op: Dst0I32);
1716 B.buildTrunc(Res: MI.getOperand(i: i * 2 + 1).getReg(), Op: Dst1I32);
1717 }
1718 } else {
1719 auto [Dst0I32, Dst1I32] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
1720 B.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Dst0I32);
1721 B.buildTrunc(Res: MI.getOperand(i: 1).getReg(), Op: Dst1I32);
1722 }
1723
1724 MI.eraseFromParent();
1725 return true;
1726 }
1727 case AextToS32InIncomingBlockGPHI: {
1728 Register Dst = MI.getOperand(i: 0).getReg();
1729 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
1730 B.setInsertPt(MBB&: *MI.getParent(), II: MI.getParent()->getFirstNonPHI());
1731 MI.getOperand(i: 0).setReg(NewDst);
1732 B.buildTrunc(Res: Dst, Op: NewDst);
1733
1734 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1735 Register UseReg = MI.getOperand(i).getReg();
1736
1737 auto DefMI = MRI.getVRegDef(Reg: UseReg)->getIterator();
1738 MachineBasicBlock *DefMBB = DefMI->getParent();
1739
1740 B.setInsertPt(MBB&: *DefMBB, II: DefMBB->SkipPHIsAndLabels(I: std::next(x: DefMI)));
1741
1742 auto NewUse = B.buildAnyExt(Res: SgprRB_I32, Op: UseReg);
1743 MI.getOperand(i).setReg(NewUse.getReg(Idx: 0));
1744 }
1745 break;
1746 }
1747 case VerifyAllSgprGPHI: {
1748 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1749 if (Op.isMBB())
1750 return true;
1751 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1752 }));
1753 return true;
1754 }
1755 case VerifyAllSgprOrVgprGPHI: {
1756 assert(MRI.getRegBankOrNull(MI.getOperand(0).getReg()) == VgprRB);
1757 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1758 if (Op.isMBB())
1759 return true;
1760 const RegisterBank *RB = MRI.getRegBankOrNull(Op.getReg());
1761 return RB == VgprRB || RB == SgprRB;
1762 }));
1763 return true;
1764 }
1765 case ApplyINTRIN_IMAGE: {
1766 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
1767 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
1768 assert(RSrcIntrin && RSrcIntrin->IsImage);
1769 // The reported argument index is relative to the IR intrinsic call
1770 // arguments, so shift by the number of defs and the intrinsic ID.
1771 unsigned RsrcIdx = RSrcIntrin->RsrcArg + MI.getNumExplicitDefs() + 1;
1772 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1773 }
1774 case ApplyBVH_INTERSECT_RAY: {
1775 // Rsrc is the last register operand. Base BVH trails an A16 immediate
1776 // after rsrc; dual/BVH8 do not. Scan backwards for the last virtual
1777 // register.
1778 unsigned RsrcIdx = MI.getNumOperands();
1779 while (RsrcIdx-- > MI.getNumExplicitDefs()) {
1780 const MachineOperand &Op = MI.getOperand(i: RsrcIdx);
1781 if (Op.isReg() && Op.getReg().isVirtual())
1782 break;
1783 }
1784 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1785 }
1786 case SplitBitCount64To32:
1787 return lowerSplitBitCount64To32(MI);
1788 case ExtrVecEltToSel:
1789 return lowerExtrVecEltToSel(MI);
1790 case ExtrVecEltTo32:
1791 return lowerExtrVecEltTo32(MI);
1792 case InsVecEltToSel:
1793 return lowerInsVecEltToSel(MI);
1794 case InsVecEltTo32:
1795 return lowerInsVecEltTo32(MI);
1796 case AbsToNegMax:
1797 return lowerAbsToNegMax(MI);
1798 case AbsToS32:
1799 return lowerAbsToS32(MI);
1800 case DeletePrefetch:
1801 MI.eraseFromParent();
1802 return true;
1803 case LowerSetRounding:
1804 return lowerSetRounding(MI);
1805 case LowerGetRounding:
1806 return lowerGetRounding(MI);
1807 }
1808
1809 return true;
1810}
1811
1812LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
1813 switch (ID) {
1814 case Vcc:
1815 case UniInVcc:
1816 return LLT::scalar(SizeInBits: 1);
1817 case Sgpr16:
1818 case Vgpr16:
1819 case UniInVgprS16:
1820 return LLT::scalar(SizeInBits: 16);
1821 case Sgpr32:
1822 case Sgpr32_WF:
1823 case Sgpr32Trunc:
1824 case Sgpr32AExt:
1825 case Sgpr32AExtBoolInReg:
1826 case Sgpr32SExt:
1827 case Sgpr32ZExt:
1828 case UniInVgprS32:
1829 case Sgpr32ToVgprDst:
1830 case Vgpr32:
1831 case Vgpr32AExt:
1832 case Vgpr32SExt:
1833 case Vgpr32ZExt:
1834 return LLT::scalar(SizeInBits: 32);
1835 case Sgpr64:
1836 case Vgpr64:
1837 case UniInVgprS64:
1838 case Sgpr64ToVgprDst:
1839 return LLT::scalar(SizeInBits: 64);
1840 case Sgpr128:
1841 case Vgpr128:
1842 return LLT::scalar(SizeInBits: 128);
1843 case SgprP0:
1844 case SgprP0Call_WF:
1845 case VgprP0:
1846 return LLT::pointer(AddressSpace: 0, SizeInBits: 64);
1847 case SgprP1:
1848 case VgprP1:
1849 return LLT::pointer(AddressSpace: 1, SizeInBits: 64);
1850 case SgprP2:
1851 case VgprP2:
1852 return LLT::pointer(AddressSpace: 2, SizeInBits: 32);
1853 case SgprP3:
1854 case VgprP3:
1855 return LLT::pointer(AddressSpace: 3, SizeInBits: 32);
1856 case SgprP4:
1857 case SgprP4Call_WF:
1858 case VgprP4:
1859 return LLT::pointer(AddressSpace: 4, SizeInBits: 64);
1860 case SgprP5:
1861 case VgprP5:
1862 return LLT::pointer(AddressSpace: 5, SizeInBits: 32);
1863 case SgprP6:
1864 return LLT::pointer(AddressSpace: 6, SizeInBits: 32);
1865 case SgprP8:
1866 return LLT::pointer(AddressSpace: 8, SizeInBits: 128);
1867 case SgprV2S16:
1868 case VgprV2S16:
1869 case UniInVgprV2S16:
1870 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
1871 case SgprV2S32:
1872 case VgprV2S32:
1873 case UniInVgprV2S32:
1874 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
1875 case VgprV3S32:
1876 case UniInVgprV3S32:
1877 return LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
1878 case VgprV4S16:
1879 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
1880 case VgprV8S16:
1881 case UniInVgprV8S16:
1882 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
1883 case VgprV16S16:
1884 case UniInVgprV16S16:
1885 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
1886 case SgprV4S32:
1887 case SgprV4S32_WF:
1888 case SgprV4S32_ReadFirstLane:
1889 case VgprV4S32:
1890 case UniInVgprV4S32:
1891 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
1892 case VgprV8S32:
1893 case UniInVgprV8S32:
1894 case SgprV8S32_ReadFirstLane:
1895 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
1896 case VgprV2S64:
1897 case UniInVgprV2S64:
1898 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1899 case VgprV6S32:
1900 case UniInVgprV6S32:
1901 return LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32);
1902 case VgprV16S32:
1903 case UniInVgprV16S32:
1904 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
1905 case VgprV32S16:
1906 case UniInVgprV32S16:
1907 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
1908 case VgprV32S32:
1909 case UniInVgprV32S32:
1910 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32);
1911 default:
1912 return LLT();
1913 }
1914}
1915
1916LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
1917 switch (ID) {
1918 case SgprB32:
1919 case VgprB32:
1920 case SgprB32_M0:
1921 case SgprB32_ReadFirstLane:
1922 case UniInVgprB32:
1923 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
1924 isAnyPtr(Ty, Width: 32))
1925 return Ty;
1926 return LLT();
1927 case SgprPtr32:
1928 case VgprPtr32:
1929 return isAnyPtr(Ty, Width: 32) ? Ty : LLT();
1930 case SgprPtr64:
1931 case VgprPtr64:
1932 return isAnyPtr(Ty, Width: 64) ? Ty : LLT();
1933 case SgprPtr128:
1934 case VgprPtr128:
1935 return isAnyPtr(Ty, Width: 128) ? Ty : LLT();
1936 case SgprB64:
1937 case VgprB64:
1938 case SgprB64_ReadFirstLane:
1939 case UniInVgprB64:
1940 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
1941 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
1942 return Ty;
1943 return LLT();
1944 case SgprB96:
1945 case VgprB96:
1946 case UniInVgprB96:
1947 if (Ty == LLT::scalar(SizeInBits: 96) || Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) ||
1948 Ty == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 16))
1949 return Ty;
1950 return LLT();
1951 case SgprB128:
1952 case VgprB128:
1953 case UniInVgprB128:
1954 if (Ty.getSizeInBits() == 128)
1955 return Ty;
1956 return LLT();
1957 case VgprB160:
1958 case UniInVgprB160:
1959 if (Ty.getSizeInBits() == 160)
1960 return Ty;
1961 return LLT();
1962 case SgprB256:
1963 case VgprB256:
1964 case UniInVgprB256:
1965 if (Ty.getSizeInBits() == 256)
1966 return Ty;
1967 return LLT();
1968 case SgprB512:
1969 case VgprB512:
1970 case UniInVgprB512:
1971 if (Ty.getSizeInBits() == 512)
1972 return Ty;
1973 return LLT();
1974 case SgprBRC: {
1975 const SIRegisterInfo *TRI =
1976 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1977 unsigned LLTSize = Ty.getSizeInBits();
1978 if (LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize))
1979 return Ty;
1980 return LLT();
1981 }
1982 case VgprBRC: {
1983 const SIRegisterInfo *TRI =
1984 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1985 if (TRI->getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()))
1986 return Ty;
1987 return LLT();
1988 }
1989 default:
1990 return LLT();
1991 }
1992}
1993
1994const RegisterBank *
1995RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
1996 switch (ID) {
1997 case Vcc:
1998 return VccRB;
1999 case Sgpr16:
2000 case Sgpr32:
2001 case Sgpr32_WF:
2002 case Sgpr64:
2003 case Sgpr128:
2004 case SgprP0:
2005 case SgprP0Call_WF:
2006 case SgprP1:
2007 case SgprP2:
2008 case SgprP3:
2009 case SgprP4:
2010 case SgprP4Call_WF:
2011 case SgprP5:
2012 case SgprP6:
2013 case SgprP8:
2014 case SgprPtr32:
2015 case SgprPtr64:
2016 case SgprPtr128:
2017 case SgprV2S16:
2018 case SgprV2S32:
2019 case SgprV4S32:
2020 case SgprV4S32_WF:
2021 case SgprV4S32_ReadFirstLane:
2022 case SgprV8S32_ReadFirstLane:
2023 case SgprB32:
2024 case SgprB64:
2025 case SgprB96:
2026 case SgprB128:
2027 case SgprB256:
2028 case SgprB512:
2029 case SgprBRC:
2030 case UniInVcc:
2031 case UniInVgprS16:
2032 case UniInVgprS32:
2033 case UniInVgprS64:
2034 case UniInVgprV2S16:
2035 case UniInVgprV2S32:
2036 case UniInVgprV3S32:
2037 case UniInVgprV4S32:
2038 case UniInVgprV2S64:
2039 case UniInVgprV6S32:
2040 case UniInVgprV8S16:
2041 case UniInVgprV8S32:
2042 case UniInVgprV16S16:
2043 case UniInVgprV16S32:
2044 case UniInVgprV32S16:
2045 case UniInVgprV32S32:
2046 case UniInVgprB32:
2047 case UniInVgprB64:
2048 case UniInVgprB96:
2049 case UniInVgprB128:
2050 case UniInVgprB160:
2051 case UniInVgprB256:
2052 case UniInVgprB512:
2053 case Sgpr32Trunc:
2054 case Sgpr32AExt:
2055 case Sgpr32AExtBoolInReg:
2056 case Sgpr32SExt:
2057 case Sgpr32ZExt:
2058 return SgprRB;
2059 case AgprAnyTy:
2060 return AgprRB;
2061 case Vgpr16:
2062 case Vgpr32:
2063 case Vgpr64:
2064 case Vgpr128:
2065 case VgprP0:
2066 case VgprP1:
2067 case VgprP2:
2068 case VgprP3:
2069 case VgprP4:
2070 case VgprP5:
2071 case VgprPtr32:
2072 case VgprPtr64:
2073 case VgprPtr128:
2074 case VgprV2S16:
2075 case VgprV2S32:
2076 case VgprV2S64:
2077 case VgprV3S32:
2078 case VgprV4S16:
2079 case VgprV8S16:
2080 case VgprV16S16:
2081 case VgprV4S32:
2082 case VgprV6S32:
2083 case VgprV8S32:
2084 case VgprV16S32:
2085 case VgprV32S16:
2086 case VgprV32S32:
2087 case VgprB32:
2088 case VgprB64:
2089 case VgprB96:
2090 case VgprB128:
2091 case VgprB160:
2092 case VgprB256:
2093 case VgprB512:
2094 case VgprBRC:
2095 case VgprAnyTy:
2096 case Vgpr32AExt:
2097 case Vgpr32SExt:
2098 case Vgpr32ZExt:
2099 case Sgpr32ToVgprDst:
2100 case Sgpr64ToVgprDst:
2101 return VgprRB;
2102 default:
2103 return nullptr;
2104 }
2105}
2106
2107bool RegBankLegalizeHelper::applyMappingDst(
2108 MachineInstr &MI, unsigned &OpIdx,
2109 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs) {
2110 // Defs start from operand 0
2111 for (; OpIdx < MethodIDs.size(); ++OpIdx) {
2112 if (MethodIDs[OpIdx] == None)
2113 continue;
2114 MachineOperand &Op = MI.getOperand(i: OpIdx);
2115 Register Reg = Op.getReg();
2116 LLT Ty = MRI.getType(Reg);
2117 [[maybe_unused]] const RegisterBank *RB = MRI.getRegBank(Reg);
2118
2119 switch (MethodIDs[OpIdx]) {
2120 // vcc, sgpr and vgpr scalars, pointers and vectors
2121 case Vcc:
2122 case Sgpr16:
2123 case Sgpr32:
2124 case Sgpr64:
2125 case Sgpr128:
2126 case SgprP0:
2127 case SgprP1:
2128 case SgprP3:
2129 case SgprP4:
2130 case SgprP5:
2131 case SgprP6:
2132 case SgprP8:
2133 case SgprV2S16:
2134 case SgprV2S32:
2135 case SgprV4S32:
2136 case Vgpr16:
2137 case Vgpr32:
2138 case Vgpr64:
2139 case Vgpr128:
2140 case VgprP0:
2141 case VgprP1:
2142 case VgprP2:
2143 case VgprP3:
2144 case VgprP4:
2145 case VgprP5:
2146 case VgprV2S16:
2147 case VgprV2S32:
2148 case VgprV2S64:
2149 case VgprV3S32:
2150 case VgprV4S16:
2151 case VgprV8S16:
2152 case VgprV16S16:
2153 case VgprV4S32:
2154 case VgprV6S32:
2155 case VgprV8S32:
2156 case VgprV16S32:
2157 case VgprV32S16:
2158 case VgprV32S32: {
2159 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2160 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2161 break;
2162 }
2163 // sgpr and vgpr B-types
2164 case SgprB32:
2165 case SgprB64:
2166 case SgprB96:
2167 case SgprB128:
2168 case SgprB256:
2169 case SgprB512:
2170 case SgprBRC:
2171 case SgprPtr32:
2172 case SgprPtr64:
2173 case SgprPtr128:
2174 case VgprB32:
2175 case VgprB64:
2176 case VgprB96:
2177 case VgprB128:
2178 case VgprB160:
2179 case VgprB256:
2180 case VgprB512:
2181 case VgprBRC:
2182 case VgprPtr32:
2183 case VgprPtr64:
2184 case VgprPtr128: {
2185 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2186 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2187 break;
2188 }
2189 case VgprAnyTy: {
2190 assert(RB == VgprRB);
2191 break;
2192 }
2193 case AgprAnyTy: {
2194 if (RB == AgprRB)
2195 break;
2196 Register NewAgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: AgprRB, .Ty: Ty});
2197 Op.setReg(NewAgprDst);
2198 if (!MRI.use_nodbg_empty(RegNo: Reg))
2199 B.buildCopy(Res: Reg, Op: NewAgprDst);
2200 break;
2201 }
2202 case VgprOrAgprAnyTy: {
2203 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2204 const RegisterBank *DstRB =
2205 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2206 if (RB == DstRB)
2207 break;
2208 Register NewDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: DstRB, .Ty: Ty});
2209 Op.setReg(NewDst);
2210 if (!MRI.use_nodbg_empty(RegNo: Reg))
2211 B.buildCopy(Res: Reg, Op: NewDst);
2212 break;
2213 }
2214 // uniform in vcc/vgpr: scalars, vectors and B-types
2215 case UniInVcc: {
2216 assert(Ty == S1);
2217 assert(RB == SgprRB);
2218 Register NewDst = MRI.createVirtualRegister(RegAttr: VccRB_S1);
2219 Op.setReg(NewDst);
2220 if (!MRI.use_empty(RegNo: Reg)) {
2221 auto CopyS32_Vcc =
2222 B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_SCC_VCC, DstOps: {SgprRB_I32}, SrcOps: {NewDst});
2223 B.buildTrunc(Res: Reg, Op: CopyS32_Vcc);
2224 }
2225 break;
2226 }
2227 case UniInVgprS16: {
2228 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2229 assert(RB == SgprRB);
2230 Register NewVgprDst16 = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2231 Register NewVgprDstI32 = MRI.createVirtualRegister(RegAttr: VgprRB_I32);
2232 Register NewSgprDstI32 = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2233 Op.setReg(NewVgprDst16);
2234 B.buildAnyExt(Res: NewVgprDstI32, Op: NewVgprDst16);
2235 buildReadAnyLane(B, SgprDst: NewSgprDstI32, VgprSrc: NewVgprDstI32, RBI);
2236 B.buildTrunc(Res: Reg, Op: NewSgprDstI32);
2237 break;
2238 }
2239 case UniInVgprS32:
2240 case UniInVgprS64:
2241 case UniInVgprV2S16:
2242 case UniInVgprV2S32:
2243 case UniInVgprV3S32:
2244 case UniInVgprV4S32:
2245 case UniInVgprV2S64:
2246 case UniInVgprV6S32:
2247 case UniInVgprV8S16:
2248 case UniInVgprV8S32:
2249 case UniInVgprV16S16:
2250 case UniInVgprV16S32:
2251 case UniInVgprV32S16:
2252 case UniInVgprV32S32: {
2253 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2254 assert(RB == SgprRB);
2255 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2256 Op.setReg(NewVgprDst);
2257 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2258 break;
2259 }
2260 case UniInVgprB32:
2261 case UniInVgprB64:
2262 case UniInVgprB96:
2263 case UniInVgprB128:
2264 case UniInVgprB160:
2265 case UniInVgprB256:
2266 case UniInVgprB512: {
2267 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2268 assert(RB == SgprRB);
2269 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2270 Op.setReg(NewVgprDst);
2271 AMDGPU::buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2272 break;
2273 }
2274 // sgpr trunc
2275 case Sgpr32Trunc: {
2276 assert(Ty.getSizeInBits() < 32);
2277 assert(RB == SgprRB);
2278 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2279 Op.setReg(NewDst);
2280 if (!MRI.use_empty(RegNo: Reg))
2281 B.buildTrunc(Res: Reg, Op: NewDst);
2282 break;
2283 }
2284 case Sgpr32ToVgprDst:
2285 case Sgpr64ToVgprDst: {
2286 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2287 assert(RB == VgprRB);
2288 Op.setReg(MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty}));
2289 B.buildCopy(Res: Reg, Op: Op.getReg());
2290 break;
2291 }
2292 case InvalidMapping: {
2293 reportGISelFailure(
2294 MF, MORE, DEBUG_TYPE,
2295 Msg: "AMDGPU RegBankLegalize: missing fast rule ('Div' or 'Uni') for", MI);
2296 return false;
2297 }
2298 default:
2299 reportGISelFailure(
2300 MF, MORE, DEBUG_TYPE,
2301 Msg: "AMDGPU RegBankLegalize: applyMappingDst, ID not supported", MI);
2302 return false;
2303 }
2304 }
2305
2306 return true;
2307}
2308
2309bool RegBankLegalizeHelper::applyMappingSrc(
2310 MachineInstr &MI, unsigned &OpIdx,
2311 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
2312 WaterfallInfo &WFI) {
2313 for (unsigned i = 0; i < MethodIDs.size(); ++OpIdx, ++i) {
2314 if (MethodIDs[i] == None || MethodIDs[i] == IntrId || MethodIDs[i] == Imm)
2315 continue;
2316
2317 MachineOperand &Op = MI.getOperand(i: OpIdx);
2318 Register Reg = Op.getReg();
2319 LLT Ty = MRI.getType(Reg);
2320 const RegisterBank *RB = MRI.getRegBank(Reg);
2321
2322 switch (MethodIDs[i]) {
2323 case Vcc: {
2324 assert(Ty == S1);
2325 assert(RB == VccRB || RB == SgprRB);
2326 if (RB == SgprRB) {
2327 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2328 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2329 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2330 auto CopyVcc_Scc = B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_VCC_SCC,
2331 DstOps: {VccRB_S1}, SrcOps: {BoolInReg});
2332 Op.setReg(CopyVcc_Scc.getReg(Idx: 0));
2333 }
2334 break;
2335 }
2336 // sgpr scalars, pointers and vectors
2337 case Sgpr16:
2338 case Sgpr32:
2339 case Sgpr64:
2340 case Sgpr128:
2341 case SgprP0:
2342 case SgprP1:
2343 case SgprP3:
2344 case SgprP4:
2345 case SgprP5:
2346 case SgprP6:
2347 case SgprP8:
2348 case SgprV2S16:
2349 case SgprV2S32:
2350 case SgprV4S32: {
2351 assert(Ty == getTyFromID(MethodIDs[i]));
2352 assert(RB == getRegBankFromID(MethodIDs[i]));
2353 break;
2354 }
2355 // sgpr B-types
2356 case SgprB32:
2357 case SgprB64:
2358 case SgprB96:
2359 case SgprB128:
2360 case SgprB256:
2361 case SgprB512:
2362 case SgprBRC:
2363 case SgprPtr32:
2364 case SgprPtr64:
2365 case SgprPtr128: {
2366 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2367 assert(RB == getRegBankFromID(MethodIDs[i]));
2368 break;
2369 }
2370 // vgpr scalars, pointers and vectors
2371 case Vgpr16:
2372 case Vgpr32:
2373 case Vgpr64:
2374 case Vgpr128:
2375 case VgprP0:
2376 case VgprP1:
2377 case VgprP2:
2378 case VgprP3:
2379 case VgprP4:
2380 case VgprP5:
2381 case VgprV2S16:
2382 case VgprV2S32:
2383 case VgprV2S64:
2384 case VgprV3S32:
2385 case VgprV4S16:
2386 case VgprV8S16:
2387 case VgprV16S16:
2388 case VgprV4S32:
2389 case VgprV6S32:
2390 case VgprV8S32:
2391 case VgprV16S32:
2392 case VgprV32S16:
2393 case VgprV32S32: {
2394 assert(Ty == getTyFromID(MethodIDs[i]));
2395 if (RB != VgprRB) {
2396 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2397 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2398 }
2399 break;
2400 }
2401 // vgpr B-types
2402 case VgprB32:
2403 case VgprB64:
2404 case VgprB96:
2405 case VgprB128:
2406 case VgprB160:
2407 case VgprB256:
2408 case VgprB512:
2409 case VgprBRC:
2410 case VgprPtr32:
2411 case VgprPtr64:
2412 case VgprPtr128: {
2413 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2414 if (RB != VgprRB) {
2415 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2416 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2417 }
2418 break;
2419 }
2420 case VgprAnyTy: {
2421 if (RB != VgprRB) {
2422 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2423 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2424 }
2425 break;
2426 }
2427 case AgprAnyTy: {
2428 if (RB != AgprRB) {
2429 auto CopyToAgpr = B.buildCopy(Res: {AgprRB, Ty}, Op: Reg);
2430 Op.setReg(CopyToAgpr.getReg(Idx: 0));
2431 }
2432 break;
2433 }
2434 case VgprOrAgprAnyTy: {
2435 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2436 const RegisterBank *SrcRB =
2437 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2438 if (RB != SrcRB)
2439 Op.setReg(B.buildCopy(Res: {SrcRB, Ty}, Op: Reg).getReg(Idx: 0));
2440 break;
2441 }
2442 // sgpr waterfall, scalars, and vectors
2443 case Sgpr32_WF:
2444 case SgprV4S32_WF: {
2445 assert(Ty == getTyFromID(MethodIDs[i]));
2446 if (RB != SgprRB) {
2447 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2448 if (!WFI.Start.isValid()) {
2449 WFI.Start = MI.getIterator();
2450 WFI.End = std::next(x: MI.getIterator());
2451 }
2452 }
2453 break;
2454 }
2455 case SgprP0Call_WF:
2456 case SgprP4Call_WF: {
2457 assert(Ty == getTyFromID(MethodIDs[i]));
2458 if (RB != SgprRB) {
2459 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2460
2461 // Find the ADJCALLSTACKUP before the call.
2462 MachineBasicBlock::iterator Start = MI.getIterator();
2463 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
2464 --Start;
2465
2466 // Find the ADJCALLSTACKDOWN after the call (include it in range).
2467 MachineBasicBlock::iterator End = MI.getIterator();
2468 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
2469 ++End;
2470 ++End;
2471
2472 WFI.Start = Start;
2473 WFI.End = End;
2474 }
2475 break;
2476 }
2477 case SgprB32_M0:
2478 case SgprB32_ReadFirstLane:
2479 case SgprB64_ReadFirstLane: {
2480 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2481 if (RB == SgprRB)
2482 break;
2483 assert(RB == VgprRB);
2484 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2485 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2486 Op.setReg(NewSGPR);
2487 break;
2488 }
2489 case SgprV4S32_ReadFirstLane:
2490 case SgprV8S32_ReadFirstLane: {
2491 assert(Ty == getTyFromID(MethodIDs[i]));
2492 if (RB == SgprRB)
2493 break;
2494 assert(RB == VgprRB);
2495 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2496 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2497 Op.setReg(NewSGPR);
2498 break;
2499 }
2500 // sgpr and vgpr scalars with extend
2501 case Sgpr32AExt: {
2502 // Note: this ext allows S1, and it is meant to be combined away.
2503 assert(Ty.getSizeInBits() < 32);
2504 assert(RB == SgprRB);
2505 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2506 Op.setReg(Aext.getReg(Idx: 0));
2507 break;
2508 }
2509 case Sgpr32AExtBoolInReg: {
2510 // Note: this ext allows S1, and it is meant to be combined away.
2511 assert(Ty.getSizeInBits() == 1);
2512 assert(RB == SgprRB);
2513 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2514 // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
2515 // most of times meant to be combined away in AMDGPURegBankCombiner.
2516 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2517 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2518 Op.setReg(BoolInReg.getReg(Idx: 0));
2519 break;
2520 }
2521 case Sgpr32SExt: {
2522 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2523 assert(RB == SgprRB);
2524 auto Sext = B.buildSExt(Res: SgprRB_I32, Op: Reg);
2525 Op.setReg(Sext.getReg(Idx: 0));
2526 break;
2527 }
2528 case Sgpr32ZExt: {
2529 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2530 assert(RB == SgprRB);
2531 auto Zext = B.buildZExt(Res: SgprRB_I32, Op: Reg);
2532 Op.setReg(Zext.getReg(Idx: 0));
2533 break;
2534 }
2535 case Vgpr32AExt: {
2536 assert(Ty.getSizeInBits() < 32);
2537 assert(RB == VgprRB);
2538 auto Aext = B.buildAnyExt(Res: VgprRB_I32, Op: Reg);
2539 Op.setReg(Aext.getReg(Idx: 0));
2540 break;
2541 }
2542 case Vgpr32SExt: {
2543 // Note this ext allows S1, and it is meant to be combined away.
2544 assert(Ty.getSizeInBits() < 32);
2545 assert(RB == VgprRB);
2546 auto Sext = B.buildSExt(Res: VgprRB_I32, Op: Reg);
2547 Op.setReg(Sext.getReg(Idx: 0));
2548 break;
2549 }
2550 case Vgpr32ZExt: {
2551 // Note this ext allows S1, and it is meant to be combined away.
2552 assert(Ty.getSizeInBits() < 32);
2553 assert(RB == VgprRB);
2554 auto Zext = B.buildZExt(Res: VgprRB_I32, Op: Reg);
2555 Op.setReg(Zext.getReg(Idx: 0));
2556 break;
2557 }
2558 default:
2559 reportGISelFailure(
2560 MF, MORE, DEBUG_TYPE,
2561 Msg: "AMDGPU RegBankLegalize: applyMappingSrc, ID not supported", MI);
2562 return false;
2563 }
2564 }
2565 return true;
2566}
2567
2568[[maybe_unused]] static bool verifyRegBankOnOperands(MachineInstr &MI,
2569 const RegisterBank *RB,
2570 MachineRegisterInfo &MRI,
2571 unsigned StartOpIdx,
2572 unsigned EndOpIdx) {
2573 for (unsigned i = StartOpIdx; i <= EndOpIdx; ++i) {
2574 if (MRI.getRegBankOrNull(Reg: MI.getOperand(i).getReg()) != RB)
2575 return false;
2576 }
2577 return true;
2578}
2579
2580bool RegBankLegalizeHelper::applyRegisterBanksVgprWithSgprRsrc(
2581 MachineInstr &MI, unsigned RsrcIdx) {
2582 const unsigned NumDefs = MI.getNumExplicitDefs();
2583
2584 MachineBasicBlock *MBB = MI.getParent();
2585 B.setInsertPt(MBB&: *MBB, II: MBB->SkipPHIsAndLabels(I: std::next(x: MI.getIterator())));
2586
2587 // Defs are vgpr.
2588 for (unsigned i = 0; i < NumDefs; ++i) {
2589 Register Reg = MI.getOperand(i).getReg();
2590 if (MRI.getRegBank(Reg) == VgprRB)
2591 continue;
2592
2593 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: MRI.getType(Reg)});
2594 MI.getOperand(i).setReg(NewVgprDst);
2595 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2596 }
2597
2598 B.setInstrAndDebugLoc(MI);
2599
2600 // Register uses before RsrcIdx are vgpr.
2601 for (unsigned i = NumDefs; i < RsrcIdx; ++i) {
2602 MachineOperand &Op = MI.getOperand(i);
2603 if (!Op.isReg())
2604 continue;
2605
2606 Register Reg = Op.getReg();
2607 if (!Reg.isVirtual())
2608 continue;
2609
2610 if (MRI.getRegBank(Reg) == VgprRB)
2611 continue;
2612
2613 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
2614 Op.setReg(Copy.getReg(Idx: 0));
2615 }
2616
2617 SmallSet<Register, 4> OpsToWaterfall;
2618
2619 // Register use RsrcIdx (and later register operands) is sgpr.
2620 for (unsigned i = RsrcIdx; i < MI.getNumOperands(); ++i) {
2621 MachineOperand &Op = MI.getOperand(i);
2622 if (!Op.isReg())
2623 continue;
2624
2625 Register Reg = Op.getReg();
2626 if (MRI.getRegBank(Reg) != SgprRB)
2627 OpsToWaterfall.insert(V: Reg);
2628 }
2629
2630 if (!OpsToWaterfall.empty()) {
2631 MachineBasicBlock::iterator MII = MI.getIterator();
2632 executeInWaterfallLoop(B, WFI: {.SgprWaterfallOperandRegs: OpsToWaterfall, .Start: MII, .End: std::next(x: MII)});
2633 }
2634
2635 return true;
2636}
2637