1//===-- AMDGPURegBankLegalizeHelper.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Implements actual lowering algorithms for each ID that can be used in
10/// Rule.OperandMapping. Similar to legalizer helper but with register banks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPURegBankLegalizeHelper.h"
15#include "AMDGPUGlobalISelUtils.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPURegBankLegalizeRules.h"
19#include "AMDGPURegisterBankInfo.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIMachineFunctionInfo.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
27#include "llvm/CodeGen/MachineInstr.h"
28#include "llvm/CodeGen/MachineUniformityAnalysis.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30
31#define DEBUG_TYPE "amdgpu-reg-bank-legalize"
32
33using namespace llvm;
34using namespace AMDGPU;
35
36RegBankLegalizeHelper::RegBankLegalizeHelper(
37 MachineIRBuilder &B, const MachineUniformityInfo &MUI,
38 GISelValueTracking *VT, const RegisterBankInfo &RBI,
39 const RegBankLegalizeRules &RBLRules)
40 : MF(B.getMF()), MFI(MF.getInfo<SIMachineFunctionInfo>()),
41 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()), B(B),
42 MRI(*B.getMRI()), MUI(MUI), VT(VT), RBI(RBI), MORE(MF, nullptr),
43 RBLRules(RBLRules), IsWave32(ST.isWave32()),
44 SgprRB(&RBI.getRegBank(ID: AMDGPU::SGPRRegBankID)),
45 VgprRB(&RBI.getRegBank(ID: AMDGPU::VGPRRegBankID)),
46 AgprRB(&RBI.getRegBank(ID: AMDGPU::AGPRRegBankID)),
47 VccRB(&RBI.getRegBank(ID: AMDGPU::VCCRegBankID)) {}
48
49bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
50 const SetOfRulesForOpcode *RuleSet = RBLRules.getRulesForOpc(MI);
51 if (!RuleSet) {
52 reportGISelFailure(MF, MORE, DEBUG_TYPE,
53 Msg: "No AMDGPU RegBankLegalize rules defined for opcode",
54 MI);
55 return false;
56 }
57
58 const RegBankLLTMapping *Mapping = RuleSet->findMappingForMI(MI, MRI, MUI);
59 if (!Mapping) {
60 reportGISelFailure(MF, MORE, DEBUG_TYPE,
61 Msg: "AMDGPU RegBankLegalize: none of the rules defined with "
62 "'Any' for MI's opcode matched MI",
63 MI);
64 return false;
65 }
66
67 WaterfallInfo WFI;
68 unsigned OpIdx = 0;
69 if (!Mapping->DstOpMapping.empty()) {
70 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
71 if (!applyMappingDst(MI, OpIdx, MethodIDs: Mapping->DstOpMapping))
72 return false;
73 }
74 if (!Mapping->SrcOpMapping.empty()) {
75 B.setInstr(MI);
76 if (!applyMappingSrc(MI, OpIdx, MethodIDs: Mapping->SrcOpMapping, WFI))
77 return false;
78 }
79
80 if (!lower(MI, Mapping: *Mapping, WFI))
81 return false;
82
83 if (!WFI.SgprWaterfallOperandRegs.empty()) {
84 if (!executeInWaterfallLoop(B, WFI))
85 return false;
86 }
87
88 return true;
89}
90
91bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
92 const WaterfallInfo &WFI) {
93 assert(WFI.Start.isValid() && WFI.End.isValid() &&
94 "Waterfall range not initialized");
95
96 // Track use registers which have already been expanded with a readfirstlane
97 // sequence. This may have multiple uses if moving a sequence.
98 DenseMap<Register, Register> WaterfalledRegMap;
99
100 MachineBasicBlock &MBB = B.getMBB();
101 MachineFunction &MF = B.getMF();
102
103 MachineBasicBlock::iterator BeginIt = WFI.Start;
104 MachineBasicBlock::iterator EndIt = WFI.End;
105
106 const SIRegisterInfo *TRI = ST.getRegisterInfo();
107 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
108 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
109
110#ifndef NDEBUG
111 const int OrigRangeSize = std::distance(BeginIt, EndIt);
112#endif
113
114 MachineRegisterInfo &MRI = *B.getMRI();
115 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
116 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
117
118 // Don't bother using generic instructions/registers for the exec mask.
119 B.setInstr(*WFI.Start);
120 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: InitSaveExecReg);
121
122 Register SavedExec = MRI.createVirtualRegister(RegClass: WaveRC);
123
124 // To insert the loop we need to split the block. Move everything before
125 // this point to a new block, and insert a new empty block before this
126 // instruction.
127 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
128 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
129 MachineBasicBlock *RestoreExecBB = MF.CreateMachineBasicBlock();
130 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
131 MachineFunction::iterator MBBI(MBB);
132 ++MBBI;
133 MF.insert(MBBI, MBB: LoopBB);
134 MF.insert(MBBI, MBB: BodyBB);
135 MF.insert(MBBI, MBB: RestoreExecBB);
136 MF.insert(MBBI, MBB: RemainderBB);
137
138 LoopBB->addSuccessor(Succ: BodyBB);
139 BodyBB->addSuccessor(Succ: RestoreExecBB);
140 BodyBB->addSuccessor(Succ: LoopBB);
141
142 // Move the rest of the block into a new block.
143 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
144 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: EndIt, To: MBB.end());
145
146 MBB.addSuccessor(Succ: LoopBB);
147 RestoreExecBB->addSuccessor(Succ: RemainderBB);
148
149 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
150
151 // +-MBB:------------+
152 // | ... |
153 // | %0 = G_INST_1 |
154 // | %Dst = MI %Vgpr |
155 // | %1 = G_INST_2 |
156 // | ... |
157 // +-----------------+
158 // ->
159 // +-MBB-------------------------------+
160 // | ... |
161 // | %0 = G_INST_1 |
162 // | %SaveExecReg = S_MOV_B32 $exec_lo |
163 // +----------------|------------------+
164 // | /------------------------------|
165 // V V |
166 // +-LoopBB---------------------------------------------------------------+ |
167 // | %CurrentLaneReg:sgpr(s32) = READFIRSTLANE %Vgpr | |
168 // | instead of executing for each lane, see if other lanes had | |
169 // | same value for %Vgpr and execute for them also. | |
170 // | %CondReg:vcc(s1) = G_ICMP eq %CurrentLaneReg, %Vgpr | |
171 // | %CondRegLM:sreg_32 = ballot %CondReg // copy vcc to sreg32 lane mask | |
172 // | %SavedExec = S_AND_SAVEEXEC_B32 %CondRegLM | |
173 // | exec is active for lanes with the same "CurrentLane value" in Vgpr | |
174 // +----------------|-----------------------------------------------------+ |
175 // V |
176 // +-BodyBB------------------------------------------------------------+ |
177 // | %Dst = MI %CurrentLaneReg:sgpr(s32) | |
178 // | executed only for active lanes and written to Dst | |
179 // | $exec = S_XOR_B32 $exec, %SavedExec | |
180 // | set active lanes to 0 in SavedExec, lanes that did not write to | |
181 // | Dst yet, and set this as new exec (for READFIRSTLANE and ICMP) | |
182 // | SI_WATERFALL_LOOP LoopBB |-----|
183 // +----------------|--------------------------------------------------+
184 // V
185 // +-RestoreExecBB--------------------------+
186 // | $exec_lo = S_MOV_B32_term %SaveExecReg |
187 // +----------------|-----------------------+
188 // V
189 // +-RemainderBB:----------------------+
190 // | %1 = G_INST_2 |
191 // | ... |
192 // +---------------------------------- +
193
194 // Move the instruction into the loop body. Note we moved everything after
195 // Range.end() already into a new block, so Range.end() is no longer valid.
196 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: BeginIt, To: MBB.end());
197
198 // Figure out the iterator range after splicing the instructions.
199 MachineBasicBlock::iterator NewBegin = BeginIt;
200 auto NewEnd = BodyBB->end();
201 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
202
203 B.setMBB(*LoopBB);
204 Register CondReg;
205
206 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
207 for (MachineOperand &Op : MI.all_uses()) {
208 Register OldReg = Op.getReg();
209 if (!WFI.SgprWaterfallOperandRegs.count(V: OldReg))
210 continue;
211
212 // See if we already processed this register in another instruction in
213 // the sequence.
214 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
215 if (OldVal != WaterfalledRegMap.end()) {
216 Op.setReg(OldVal->second);
217 continue;
218 }
219
220 Register OpReg = Op.getReg();
221 LLT OpTy = MRI.getType(Reg: OpReg);
222
223 // TODO: support for agpr
224 assert(MRI.getRegBank(OpReg) == VgprRB);
225 Register CurrentLaneReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: OpTy});
226 buildReadFirstLane(B, SgprDst: CurrentLaneReg, VgprSrc: OpReg, RBI);
227
228 // Build the comparison(s), CurrentLaneReg == OpReg.
229 unsigned OpSize = OpTy.getSizeInBits();
230 unsigned PartSize = (OpSize % 64 == 0) ? 64 : 32;
231 LLT PartTy = LLT::integer(SizeInBits: PartSize);
232 unsigned NumParts = OpSize / PartSize;
233 SmallVector<Register, 8> OpParts;
234 SmallVector<Register, 8> CurrentLaneParts;
235
236 if (NumParts == 1) {
237 OpParts.push_back(Elt: OpReg);
238 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
239 } else {
240 auto UnmergeOp = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: PartTy}, Op: OpReg);
241 auto UnmergeCurrLane = B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: PartTy}, Op: CurrentLaneReg);
242 for (unsigned i = 0; i < NumParts; ++i) {
243 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
244 CurrentLaneParts.push_back(Elt: UnmergeCurrLane.getReg(Idx: i));
245 }
246 }
247
248 for (unsigned i = 0; i < NumParts; ++i) {
249 Register CmpReg = MRI.createVirtualRegister(RegAttr: VccRB_S1);
250 B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpReg, Op0: CurrentLaneParts[i], Op1: OpParts[i]);
251
252 if (!CondReg)
253 CondReg = CmpReg;
254 else
255 CondReg = B.buildAnd(Dst: VccRB_S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
256 }
257
258 Op.setReg(CurrentLaneReg);
259
260 // Make sure we don't re-process this register again.
261 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
262 }
263 }
264
265 // Copy vcc to sgpr32/64, ballot becomes a no-op during instruction selection.
266 Register CondRegLM =
267 MRI.createVirtualRegister(RegAttr: {.RCOrRB: WaveRC, .Ty: LLT::integer(SizeInBits: IsWave32 ? 32 : 64)});
268 B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot, Res: CondRegLM).addReg(RegNo: CondReg);
269
270 // Update EXEC, save the original EXEC value to SavedExec.
271 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
272 .addDef(RegNo: SavedExec)
273 .addReg(RegNo: CondRegLM, Flags: RegState::Kill);
274 MRI.setSimpleHint(VReg: SavedExec, PrefReg: CondRegLM);
275
276 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
277
278 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
279 B.buildInstr(Opcode: LMC.XorTermOpc)
280 .addDef(RegNo: LMC.ExecReg)
281 .addReg(RegNo: LMC.ExecReg)
282 .addReg(RegNo: SavedExec);
283
284 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
285 // s_cbranch_scc0?
286
287 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
288 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
289
290 // Save the EXEC mask before the loop.
291 B.setInsertPt(MBB, II: MBB.end());
292 B.buildInstr(Opcode: LMC.MovOpc).addDef(RegNo: SaveExecReg).addReg(RegNo: LMC.ExecReg);
293
294 // Restore the EXEC mask after the loop.
295 B.setInsertPt(MBB&: *RestoreExecBB, II: RestoreExecBB->begin());
296 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
297
298 // Set the insert point after the original instruction, so any new
299 // instructions will be in the remainder.
300 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
301
302 return true;
303}
304
305// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
306// the three offsets (voffset, soffset and instoffset)
307unsigned RegBankLegalizeHelper::setBufferOffsets(
308 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
309 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) {
310 if (std::optional<int64_t> Imm =
311 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI)) {
312 uint32_t SOffset, ImmOffset;
313 if (TII.splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
314 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
315 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
316 InstOffsetVal = ImmOffset;
317 return SOffset + ImmOffset;
318 }
319 }
320 const bool CheckNUW = ST.hasGFX1250Insts();
321 auto [Base, Offset] = AMDGPU::getBaseWithConstantOffset(
322 MRI, Reg: CombinedOffset, /*KnownBits=*/ValueTracking: nullptr,
323 /*CheckNUW=*/CheckNUW);
324 uint32_t SOffset, ImmOffset;
325 if (static_cast<int32_t>(Offset) > 0 &&
326 TII.splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
327 if (Base.isValid() && MRI.getRegBank(Reg: Base) == VgprRB) {
328 VOffsetReg = Base;
329 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
330 InstOffsetVal = ImmOffset;
331 return 0;
332 }
333 // If we have SGPR base, we can use it for soffset.
334 if (SOffset == 0) {
335 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
336 SOffsetReg = Base;
337 InstOffsetVal = ImmOffset;
338 return 0;
339 }
340 }
341 // Handle the variable sgpr + vgpr case.
342 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI);
343 if (Add && static_cast<int32_t>(Offset) >= 0 &&
344 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
345 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI);
346 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI);
347 const RegisterBank *Src0Bank = MRI.getRegBank(Reg: Src0);
348 const RegisterBank *Src1Bank = MRI.getRegBank(Reg: Src1);
349 if (Src0Bank == VgprRB && Src1Bank == SgprRB) {
350 VOffsetReg = Src0;
351 SOffsetReg = Src1;
352 return 0;
353 }
354 if (Src0Bank == SgprRB && Src1Bank == VgprRB) {
355 VOffsetReg = Src1;
356 SOffsetReg = Src0;
357 return 0;
358 }
359 }
360 // Ensure we have a VGPR for the combined offset. This could be an issue if we
361 // have an SGPR offset and a VGPR resource.
362 if (MRI.getRegBank(Reg: CombinedOffset) == VgprRB) {
363 VOffsetReg = CombinedOffset;
364 } else {
365 VOffsetReg = B.buildCopy(Res: VgprRB_I32, Op: CombinedOffset).getReg(Idx: 0);
366 }
367 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: 0).getReg(Idx: 0);
368 return 0;
369}
370
371bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
372 ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
373 MachineFunction &MF = B.getMF();
374 assert(MI.getNumMemOperands() == 1);
375 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
376 Register Dst = MI.getOperand(i: 0).getReg();
377 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
378 Register Base = MI.getOperand(i: 1).getReg();
379 LLT PtrTy = MRI.getType(Reg: Base);
380 const RegisterBank *PtrRB = MRI.getRegBankOrNull(Reg: Base);
381 LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
382 SmallVector<Register, 4> LoadPartRegs;
383
384 unsigned ByteOffset = 0;
385 for (LLT PartTy : LLTBreakdown) {
386 Register BasePlusOffset;
387 if (ByteOffset == 0) {
388 BasePlusOffset = Base;
389 } else {
390 auto Offset = B.buildConstant(Res: {PtrRB, OffsetTy}, Val: ByteOffset);
391 BasePlusOffset =
392 B.buildObjectPtrOffset(Res: {PtrRB, PtrTy}, Op0: Base, Op1: Offset).getReg(Idx: 0);
393 }
394 auto *OffsetMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: ByteOffset, Ty: PartTy);
395 auto LoadPart = B.buildLoad(Res: {DstRB, PartTy}, Addr: BasePlusOffset, MMO&: *OffsetMMO);
396 LoadPartRegs.push_back(Elt: LoadPart.getReg(Idx: 0));
397 ByteOffset += PartTy.getSizeInBytes();
398 }
399
400 if (!MergeTy.isValid()) {
401 // Loads are of same size, concat or merge them together.
402 B.buildMergeLikeInstr(Res: Dst, Ops: LoadPartRegs);
403 } else {
404 // Loads are not all of same size, need to unmerge them to smaller pieces
405 // of MergeTy type, then merge pieces to Dst.
406 SmallVector<Register, 4> MergeTyParts;
407 for (Register Reg : LoadPartRegs) {
408 if (MRI.getType(Reg) == MergeTy) {
409 MergeTyParts.push_back(Elt: Reg);
410 } else {
411 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: Reg);
412 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i)
413 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
414 }
415 }
416 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
417 }
418 MI.eraseFromParent();
419 return true;
420}
421
422bool RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
423 LLT MergeTy) {
424 MachineFunction &MF = B.getMF();
425 assert(MI.getNumMemOperands() == 1);
426 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
427 Register Dst = MI.getOperand(i: 0).getReg();
428 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
429 Register Base = MI.getOperand(i: 1).getReg();
430
431 MachineMemOperand *WideMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: 0, Ty: WideTy);
432 auto WideLoad = B.buildLoad(Res: {DstRB, WideTy}, Addr: Base, MMO&: *WideMMO);
433
434 if (WideTy.isScalar()) {
435 B.buildTrunc(Res: Dst, Op: WideLoad);
436 } else {
437 SmallVector<Register, 4> MergeTyParts;
438 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: WideLoad);
439
440 LLT DstTy = MRI.getType(Reg: Dst);
441 unsigned NumElts = DstTy.getSizeInBits() / MergeTy.getSizeInBits();
442 for (unsigned i = 0; i < NumElts; ++i) {
443 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
444 }
445 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
446 }
447 MI.eraseFromParent();
448 return true;
449}
450
451bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
452 Register Dst = MI.getDstReg();
453 Register Ptr = MI.getPointerReg();
454 MachineMemOperand &MMO = MI.getMMO();
455 unsigned MemSize = 8 * MMO.getSize().getValue();
456
457 MachineMemOperand *WideMMO = B.getMF().getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: S32);
458
459 if (MI.getOpcode() == G_LOAD) {
460 B.buildLoad(Res: Dst, Addr: Ptr, MMO&: *WideMMO);
461 } else {
462 auto Load = B.buildLoad(Res: SgprRB_I32, Addr: Ptr, MMO&: *WideMMO);
463
464 if (MI.getOpcode() == G_ZEXTLOAD) {
465 APInt Mask = APInt::getLowBitsSet(numBits: S32.getSizeInBits(), loBitsSet: MemSize);
466 auto MaskCst = B.buildConstant(Res: SgprRB_I32, Val: Mask);
467 B.buildAnd(Dst, Src0: Load, Src1: MaskCst);
468 } else {
469 assert(MI.getOpcode() == G_SEXTLOAD);
470 B.buildSExtInReg(Res: Dst, Op: Load, ImmOp: MemSize);
471 }
472 }
473
474 MI.eraseFromParent();
475 return true;
476}
477
478bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
479 Register Dst = MI.getOperand(i: 0).getReg();
480 LLT Ty = MRI.getType(Reg: Dst);
481 Register Src = MI.getOperand(i: 1).getReg();
482 unsigned Opc = MI.getOpcode();
483 int TrueExtCst = Opc == G_SEXT ? -1 : 1;
484 if (Ty == S32 || Ty == S16) {
485 auto True = B.buildConstant(Res: {VgprRB, Ty}, Val: TrueExtCst);
486 auto False = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
487 B.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
488 } else if (Ty == S64) {
489 auto True = B.buildConstant(Res: {VgprRB_I32}, Val: TrueExtCst);
490 auto False = B.buildConstant(Res: {VgprRB_I32}, Val: 0);
491 auto Lo = B.buildSelect(Res: {VgprRB_I32}, Tst: Src, Op0: True, Op1: False);
492 MachineInstrBuilder Hi;
493 switch (Opc) {
494 case G_SEXT:
495 Hi = Lo;
496 break;
497 case G_ZEXT:
498 Hi = False;
499 break;
500 case G_ANYEXT:
501 Hi = B.buildUndef(Res: {VgprRB_I32});
502 break;
503 default:
504 reportGISelFailure(
505 MF, MORE, DEBUG_TYPE,
506 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Opcode not supported", MI);
507 return false;
508 }
509
510 B.buildMergeValues(Res: Dst, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
511 } else {
512 reportGISelFailure(
513 MF, MORE, DEBUG_TYPE,
514 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Type not supported", MI);
515 return false;
516 }
517
518 MI.eraseFromParent();
519 return true;
520}
521
522std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
523 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
524 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: 0x0000ffff);
525 auto Lo = B.buildAnd(Dst: SgprRB_I32, Src0: PackedI32, Src1: Mask);
526 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
527 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
528}
529
530std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
531 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
532 auto Lo = B.buildSExtInReg(Res: SgprRB_I32, Op: PackedI32, ImmOp: 16);
533 auto Hi = B.buildAShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
534 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
535}
536
537std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
538 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
539 auto Lo = PackedI32;
540 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
541 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
542}
543
544std::pair<Register, Register>
545RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
546 auto [Lo32, Hi32] = unpackAExt(Reg);
547 LLT EltTy = MRI.getType(Reg).getElementType();
548 return {B.buildTrunc(Res: {SgprRB, EltTy}, Op: Lo32).getReg(Idx: 0),
549 B.buildTrunc(Res: {SgprRB, EltTy}, Op: Hi32).getReg(Idx: 0)};
550}
551
552bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
553 Register Lo, Hi;
554 switch (MI.getOpcode()) {
555 case AMDGPU::G_SHL: {
556 auto [Val0, Val1] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
557 auto [Amt0, Amt1] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
558 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
559 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
560 break;
561 }
562 case AMDGPU::G_LSHR: {
563 auto [Val0, Val1] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
564 auto [Amt0, Amt1] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
565 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
566 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
567 break;
568 }
569 case AMDGPU::G_ASHR: {
570 auto [Val0, Val1] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
571 auto [Amt0, Amt1] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
572 Lo = B.buildAShr(Dst: SgprRB_I32, Src0: Val0, Src1: Amt0).getReg(Idx: 0);
573 Hi = B.buildAShr(Dst: SgprRB_I32, Src0: Val1, Src1: Amt1).getReg(Idx: 0);
574 break;
575 }
576 default:
577 reportGISelFailure(
578 MF, MORE, DEBUG_TYPE,
579 Msg: "AMDGPU RegBankLegalize: lowerUnpackBitShift, case not implemented",
580 MI);
581 return false;
582 }
583 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
584 MI.eraseFromParent();
585 return true;
586}
587
588bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
589 Register Lo, Hi;
590 switch (MI.getOpcode()) {
591 case AMDGPU::G_SMIN:
592 case AMDGPU::G_SMAX: {
593 // For signed operations, use sign extension
594 auto [Val0_Lo, Val0_Hi] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
595 auto [Val1_Lo, Val1_Hi] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
596 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
597 .getReg(Idx: 0);
598 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
599 .getReg(Idx: 0);
600 break;
601 }
602 case AMDGPU::G_UMIN:
603 case AMDGPU::G_UMAX: {
604 // For unsigned operations, use zero extension
605 auto [Val0_Lo, Val0_Hi] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
606 auto [Val1_Lo, Val1_Hi] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
607 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
608 .getReg(Idx: 0);
609 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
610 .getReg(Idx: 0);
611 break;
612 }
613 default:
614 reportGISelFailure(
615 MF, MORE, DEBUG_TYPE,
616 Msg: "AMDGPU RegBankLegalize: lowerUnpackMinMax, case not implemented", MI);
617 return false;
618 }
619 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
620 MI.eraseFromParent();
621 return true;
622}
623
624bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
625 auto [Op1Lo, Op1Hi] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
626 auto [Op2Lo, Op2Hi] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
627 auto ResLo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Lo, Op2Lo});
628 auto ResHi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Hi, Op2Hi});
629 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
630 Ops: {ResLo.getReg(Idx: 0), ResHi.getReg(Idx: 0)});
631 MI.eraseFromParent();
632 return true;
633}
634
635bool RegBankLegalizeHelper::lowerSBufToBuf(MachineInstr &MI,
636 WaterfallInfo &WFI) {
637 Register Dst = MI.getOperand(i: 0).getReg();
638 LLT Ty = MRI.getType(Reg: Dst);
639 const RegisterBank *RSrcBank = MRI.getRegBank(Reg: MI.getOperand(i: 1).getReg());
640 unsigned LoadSize = Ty.getSizeInBits();
641 int NumLoads = 1;
642 SmallVector<Register, 4> LoadParts;
643 if (LoadSize == 256 || LoadSize == 512) {
644 NumLoads = LoadSize / 128;
645 Ty = Ty.divide(Factor: NumLoads);
646 }
647 for (int I = 0; I < NumLoads; ++I)
648 LoadParts.emplace_back(Args: MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty}));
649 MachineMemOperand *OrigMMO = *MI.memoperands_begin();
650 const Align Alignment = OrigMMO->getAlign();
651 MachineFunction &MF = B.getMF();
652 Register SOffset;
653 Register VOffset;
654 int64_t ImmOffset = 0;
655 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
656 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
657 const unsigned MemSize = divideCeil(Numerator: OrigMMO->getSize().getValue(), Denominator: NumLoads);
658 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(MMO: OrigMMO, Offset: 0, Size: MemSize);
659 if (MMOOffset != 0)
660 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
661 // If only the offset is divergent, emit a MUBUF buffer load
662 // instead. We can assume that the buffer is unswizzled.
663 Register RSrc = MI.getOperand(i: 1).getReg();
664 Register VIndex = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
665 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
666 unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
667 switch (MI.getOpcode()) {
668 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
669 Opc = G_AMDGPU_BUFFER_LOAD_SBYTE;
670 break;
671 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
672 Opc = G_AMDGPU_BUFFER_LOAD_UBYTE;
673 break;
674 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
675 Opc = G_AMDGPU_BUFFER_LOAD_SSHORT;
676 break;
677 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
678 Opc = G_AMDGPU_BUFFER_LOAD_USHORT;
679 break;
680 default:
681 break;
682 }
683 for (int I = 0; I < NumLoads; ++I) {
684 B.buildInstr(Opcode: Opc)
685 .addDef(RegNo: LoadParts[I]) // vdata
686 .addUse(RegNo: RSrc) // rsrc
687 .addUse(RegNo: VIndex) // vindex
688 .addUse(RegNo: VOffset) // voffset
689 .addUse(RegNo: SOffset) // soffset
690 .addImm(Val: ImmOffset + 16 * I) // offset(imm)
691 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
692 .addImm(Val: 0) // idxen(imm)
693 .addMemOperand(MMO: MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16 * I, Size: MemSize));
694 }
695 if (NumLoads == 1)
696 B.buildCopy(Res: Dst, Op: LoadParts[0]);
697 else
698 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
699 B.setInstr(*MRI.getVRegDef(Reg: LoadParts[0]));
700 if (RSrcBank != SgprRB) {
701 WFI.SgprWaterfallOperandRegs.insert(V: RSrc);
702 WFI.Start = MRI.getVRegDef(Reg: LoadParts.front());
703 WFI.End = std::next(x: MRI.getVRegDef(Reg: LoadParts.back())->getIterator());
704 }
705 MI.eraseFromParent();
706 return true;
707}
708
709static bool isSignedBFE(MachineInstr &MI) {
710 if (GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI))
711 return (GI->is(ID: Intrinsic::amdgcn_sbfe));
712
713 return MI.getOpcode() == AMDGPU::G_SBFX;
714}
715
716bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
717 Register Dst = MI.getOperand(i: 0).getReg();
718 assert(MRI.getType(Dst) == LLT::scalar(64));
719 bool Signed = isSignedBFE(MI);
720 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
721 // Extract bitfield from Src, LSBit is the least-significant bit for the
722 // extraction (field offset) and Width is size of bitfield.
723 Register Src = MI.getOperand(i: FirstOpnd).getReg();
724 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
725 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
726 // Comments are for signed bitfield extract, similar for unsigned. x is sign
727 // bit. s is sign, l is LSB and y are remaining bits of bitfield to extract.
728
729 // Src >> LSBit Hi|Lo: x?????syyyyyyl??? -> xxxx?????syyyyyyl
730 unsigned SHROpc = Signed ? AMDGPU::G_ASHR : AMDGPU::G_LSHR;
731 auto SHRSrc = B.buildInstr(Opc: SHROpc, DstOps: {VgprRB_I64}, SrcOps: {Src, LSBit});
732
733 auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: Width, MRI);
734
735 // Expand to Src >> LSBit << (64 - Width) >> (64 - Width)
736 // << (64 - Width): Hi|Lo: xxxx?????syyyyyyl -> syyyyyyl000000000
737 // >> (64 - Width): Hi|Lo: syyyyyyl000000000 -> ssssssssssyyyyyyl
738 if (!ConstWidth) {
739 auto Amt = B.buildSub(Dst: VgprRB_I32, Src0: B.buildConstant(Res: SgprRB_I32, Val: 64), Src1: Width);
740 auto SignBit = B.buildShl(Dst: VgprRB_I64, Src0: SHRSrc, Src1: Amt);
741 B.buildInstr(Opc: SHROpc, DstOps: {Dst}, SrcOps: {SignBit, Amt});
742 MI.eraseFromParent();
743 return true;
744 }
745
746 uint64_t WidthImm = ConstWidth->Value.getZExtValue();
747 auto UnmergeSHRSrc = B.buildUnmerge(Attrs: VgprRB_I32, Op: SHRSrc);
748 Register SHRSrcLo = UnmergeSHRSrc.getReg(Idx: 0);
749 Register SHRSrcHi = UnmergeSHRSrc.getReg(Idx: 1);
750 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
751 unsigned BFXOpc = Signed ? AMDGPU::G_SBFX : AMDGPU::G_UBFX;
752
753 if (WidthImm <= 32) {
754 // SHRSrc Hi|Lo: ????????|???syyyl -> ????????|ssssyyyl
755 Register Lo = SHRSrcLo;
756 // V_BFE masks its width to 5 bits, so 32 would extract zero bits.
757 if (WidthImm < 32) {
758 Lo =
759 B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcLo, Zero, Width}).getReg(Idx: 0);
760 }
761 MachineInstrBuilder Hi;
762 if (Signed) {
763 // SHRSrc Hi|Lo: ????????|ssssyyyl -> ssssssss|ssssyyyl
764 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: B.buildConstant(Res: VgprRB_I32, Val: 31));
765 } else {
766 // SHRSrc Hi|Lo: ????????|000syyyl -> 00000000|000syyyl
767 Hi = Zero;
768 }
769 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
770 } else {
771 auto Amt = B.buildConstant(Res: VgprRB_I32, Val: WidthImm - 32);
772 // SHRSrc Hi|Lo: ??????sy|yyyyyyyl -> sssssssy|yyyyyyyl
773 auto Hi = B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcHi, Zero, Amt});
774 B.buildMergeLikeInstr(Res: Dst, Ops: {SHRSrcLo, Hi});
775 }
776
777 MI.eraseFromParent();
778 return true;
779}
780
781bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
782 Register DstReg = MI.getOperand(i: 0).getReg();
783 LLT Ty = MRI.getType(Reg: DstReg);
784 bool Signed = isSignedBFE(MI);
785 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
786 Register Src = MI.getOperand(i: FirstOpnd).getReg();
787 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
788 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
789 // For uniform bit field extract there are 4 available instructions, but
790 // LSBit(field offset) and Width(size of bitfield) need to be packed in S32,
791 // field offset in low and size in high 16 bits.
792
793 // Src1 Hi16|Lo16 = Size|FieldOffset
794 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: maskTrailingOnes<unsigned>(N: 6));
795 auto FieldOffset = B.buildAnd(Dst: SgprRB_I32, Src0: LSBit, Src1: Mask);
796 auto Size = B.buildShl(Dst: SgprRB_I32, Src0: Width, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
797 auto Src1 = B.buildOr(Dst: SgprRB_I32, Src0: FieldOffset, Src1: Size);
798 unsigned Opc32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
799 unsigned Opc64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
800 unsigned Opc = Ty == S32 ? Opc32 : Opc64;
801
802 // Select machine instruction, because of reg class constraining, insert
803 // copies from reg class to reg bank.
804 auto S_BFE = B.buildInstr(Opc, DstOps: {{SgprRB, Ty}},
805 SrcOps: {B.buildCopy(Res: Ty, Op: Src), B.buildCopy(Res: I32, Op: Src1)});
806 constrainSelectedInstRegOperands(I&: *S_BFE, TII: *ST.getInstrInfo(),
807 TRI: *ST.getRegisterInfo(), RBI);
808
809 B.buildCopy(Res: DstReg, Op: S_BFE->getOperand(i: 0).getReg());
810 MI.eraseFromParent();
811 return true;
812}
813
814bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
815 Register Dst = MI.getOperand(i: 0).getReg();
816 LLT DstTy = MRI.getType(Reg: Dst);
817 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
818 LLT Ty = DstTy.divide(Factor: 2);
819 auto Op1 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 1).getReg());
820 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
821 unsigned Opc = MI.getOpcode();
822 auto Flags = MI.getFlags();
823 auto Lo =
824 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 0), Op2.getReg(Idx: 0)}, Flags);
825 auto Hi =
826 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 1), Op2.getReg(Idx: 1)}, Flags);
827 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
828 MI.eraseFromParent();
829 return true;
830}
831
832bool RegBankLegalizeHelper::lowerSplitTo32Mul(MachineInstr &MI) {
833 Register Dst = MI.getOperand(i: 0).getReg();
834 assert(MRI.getType(Dst) == S64);
835 auto Op1 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 1).getReg());
836 auto Op2 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 2).getReg());
837
838 // TODO: G_AMDGPU_MAD_* optimizations for G_MUL divergent S64 operation to
839 // match GlobalISel with old regbankselect.
840 auto Lo = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
841 auto Carry = B.buildUMulH(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
842 auto MulLo0Hi1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 1));
843 auto MulHi0Lo1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 1), Src1: Op2.getReg(Idx: 0));
844 auto Sum = B.buildAdd(Dst: VgprRB_I32, Src0: MulLo0Hi1, Src1: MulHi0Lo1);
845 auto Hi = B.buildAdd(Dst: VgprRB_I32, Src0: Sum, Src1: Carry);
846
847 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
848 MI.eraseFromParent();
849 return true;
850}
851
852bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
853 Register Dst = MI.getOperand(i: 0).getReg();
854 assert(MRI.getType(Dst) == V2S16);
855 unsigned Opc = MI.getOpcode();
856 unsigned NumOps = MI.getNumOperands();
857 auto Flags = MI.getFlags();
858
859 auto [Op1Lo, Op1Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 1).getReg());
860 LLT EltTy = MRI.getType(Reg: Dst).getElementType();
861
862 if (NumOps == 2) {
863 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo}, Flags);
864 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi}, Flags);
865 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
866 MI.eraseFromParent();
867 return true;
868 }
869
870 auto [Op2Lo, Op2Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 2).getReg());
871
872 if (NumOps == 3) {
873 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo}, Flags);
874 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi}, Flags);
875 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
876 MI.eraseFromParent();
877 return true;
878 }
879
880 assert(NumOps == 4);
881 auto [Op3Lo, Op3Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 3).getReg());
882 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo, Op3Lo}, Flags);
883 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi, Op3Hi}, Flags);
884 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
885 MI.eraseFromParent();
886 return true;
887}
888
889bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
890 Register Dst0 = MI.getOperand(i: 0).getReg();
891 Register Dst1 = MI.getOperand(i: 1).getReg();
892 Register Src0 = MI.getOperand(i: 2).getReg();
893 Register Src1 = MI.getOperand(i: 3).getReg();
894 Register Src2 = MI.getOperand(i: 4).getReg();
895
896 const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
897
898 // Keep the multiplication on the SALU.
899 Register DstLo = B.buildMul(Dst: SgprRB_I32, Src0, Src1).getReg(Idx: 0);
900 Register DstHi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
901 if (ST.hasScalarMulHiInsts()) {
902 B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {{DstHi}}, SrcOps: {Src0, Src1});
903 } else {
904 auto VSrc0 = B.buildCopy(Res: VgprRB_I32, Op: Src0);
905 auto VSrc1 = B.buildCopy(Res: VgprRB_I32, Op: Src1);
906 auto MulHi = B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {VgprRB_I32}, SrcOps: {VSrc0, VSrc1});
907 buildReadAnyLane(B, SgprDst: DstHi, VgprSrc: MulHi.getReg(Idx: 0), RBI);
908 }
909
910 // Accumulate and produce the "carry-out" bit.
911
912 // The "carry-out" is defined as bit 64 of the result when computed as a
913 // big integer. For unsigned multiply-add, this matches the usual
914 // definition of carry-out.
915 if (mi_match(R: Src2, MRI, P: MIPatternMatch::m_ZeroInt())) {
916 // No accumulate: result is just the multiplication, carry is 0.
917 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
918 B.buildConstant(Res: Dst1, Val: 0);
919 } else {
920 // Accumulate: add Src2 to the multiplication result with carry chain.
921 Register Src2Lo = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
922 Register Src2Hi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
923 B.buildUnmerge(Res: {Src2Lo, Src2Hi}, Op: Src2);
924
925 auto AddLo = B.buildUAddo(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstLo, Op1: Src2Lo);
926 auto AddHi =
927 B.buildUAdde(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstHi, Op1: Src2Hi, CarryIn: AddLo.getReg(Idx: 1));
928 B.buildMergeLikeInstr(Res: Dst0, Ops: {AddLo.getReg(Idx: 0), AddHi.getReg(Idx: 0)});
929 B.buildCopy(Res: Dst1, Op: AddHi.getReg(Idx: 1));
930 }
931
932 MI.eraseFromParent();
933 return true;
934}
935
936bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
937 Register Dst = MI.getOperand(i: 0).getReg();
938 LLT DstTy = MRI.getType(Reg: Dst);
939 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
940 (DstTy.isPointer() && DstTy.getSizeInBits() == 64));
941 LLT Ty = DstTy.isFloat() ? LLT::float32() : DstTy.divide(Factor: 2);
942 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
943 auto Op3 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 3).getReg());
944 Register Cond = MI.getOperand(i: 1).getReg();
945 auto Flags = MI.getFlags();
946 auto Lo =
947 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 0), Op1: Op3.getReg(Idx: 0), Flags);
948 auto Hi =
949 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 1), Op1: Op3.getReg(Idx: 1), Flags);
950
951 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
952 MI.eraseFromParent();
953 return true;
954}
955
956bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
957 auto Op1 = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
958 int Amt = MI.getOperand(i: 2).getImm();
959 Register Lo, Hi;
960 // Hi|Lo: s sign bit, ?/x bits changed/not changed by sign-extend
961 if (Amt <= 32) {
962 auto Freeze = B.buildFreeze(Dst: VgprRB_I32, Src: Op1.getReg(Idx: 0));
963 if (Amt == 32) {
964 // Hi|Lo: ????????|sxxxxxxx -> ssssssss|sxxxxxxx
965 Lo = Freeze.getReg(Idx: 0);
966 } else {
967 // Hi|Lo: ????????|???sxxxx -> ssssssss|ssssxxxx
968 Lo = B.buildSExtInReg(Res: VgprRB_I32, Op: Freeze, ImmOp: Amt).getReg(Idx: 0);
969 }
970
971 auto SignExtCst = B.buildConstant(Res: SgprRB_I32, Val: 31);
972 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: SignExtCst).getReg(Idx: 0);
973 } else {
974 // Hi|Lo: ?????sxx|xxxxxxxx -> ssssssxx|xxxxxxxx
975 Lo = Op1.getReg(Idx: 0);
976 Hi = B.buildSExtInReg(Res: VgprRB_I32, Op: Op1.getReg(Idx: 1), ImmOp: Amt - 32).getReg(Idx: 0);
977 }
978
979 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
980 MI.eraseFromParent();
981 return true;
982}
983
984bool RegBankLegalizeHelper::lowerSplitBitCount64To32(MachineInstr &MI) {
985 // Split 64-bit find-first-bit operations into 32-bit halves:
986 // (ffbh hi:lo) -> umin(ffbh(hi), uaddsat(ffbh(lo), 32))
987 // (ffbl hi:lo) -> umin(ffbl(lo), uaddsat(ffbl(hi), 32))
988 // (ctlz_zero_poison hi:lo) -> umin(ffbh(hi), add(ffbh(lo), 32))
989 // (cttz_zero_poison hi:lo) -> umin(ffbl(lo), add(ffbl(hi), 32))
990 unsigned Opc = MI.getOpcode();
991
992 // FFBH/FFBL return 0xFFFFFFFF on zero input, using uaddsat to avoid
993 // wrapping. CTLZ/CTTZ guarantee non-zero input (zero_poison), so plain add
994 // is fine.
995 unsigned FFBOpc;
996 unsigned AddOpc;
997 bool SearchFromMSB;
998 switch (Opc) {
999 case AMDGPU::G_AMDGPU_FFBH_U32:
1000 FFBOpc = Opc;
1001 AddOpc = AMDGPU::G_UADDSAT;
1002 SearchFromMSB = true;
1003 break;
1004 case AMDGPU::G_AMDGPU_FFBL_B32:
1005 FFBOpc = Opc;
1006 AddOpc = AMDGPU::G_UADDSAT;
1007 SearchFromMSB = false;
1008 break;
1009 case AMDGPU::G_CTLZ_ZERO_POISON:
1010 FFBOpc = AMDGPU::G_AMDGPU_FFBH_U32;
1011 AddOpc = AMDGPU::G_ADD;
1012 SearchFromMSB = true;
1013 break;
1014 case AMDGPU::G_CTTZ_ZERO_POISON:
1015 FFBOpc = AMDGPU::G_AMDGPU_FFBL_B32;
1016 AddOpc = AMDGPU::G_ADD;
1017 SearchFromMSB = false;
1018 break;
1019 default:
1020 llvm_unreachable("unexpected opcode in lowerSplitBitCount64To32");
1021 }
1022
1023 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1024 Register Lo = Unmerge.getReg(Idx: 0);
1025 Register Hi = Unmerge.getReg(Idx: 1);
1026
1027 // MSB-first (FFBH/CTLZ) searches hi first; LSB-first (FFBL/CTTZ) searches
1028 // lo first. The secondary half adds 32 to account for the primary half's
1029 // width.
1030 auto Primary = B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Hi : Lo});
1031 auto Secondary =
1032 B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Lo : Hi});
1033
1034 auto Adjusted = B.buildInstr(Opc: AddOpc, DstOps: {VgprRB_I32},
1035 SrcOps: {Secondary, B.buildConstant(Res: VgprRB_I32, Val: 32)});
1036 B.buildUMin(Dst: MI.getOperand(i: 0).getReg(), Src0: Primary, Src1: Adjusted);
1037
1038 MI.eraseFromParent();
1039 return true;
1040}
1041
1042bool RegBankLegalizeHelper::lowerExtrVecEltToSel(MachineInstr &MI) {
1043 // Lower extract vector element to a compare-select chain:
1044 // result = elt[0]
1045 // for i in 1..N-1:
1046 // result = (idx == i) ? elt[i] : result
1047 //
1048 // When the index is divergent, each lane may want a different element, so
1049 // we must check every element per lane.
1050 Register Dst = MI.getOperand(i: 0).getReg();
1051 Register Src = MI.getOperand(i: 1).getReg();
1052 Register Idx = MI.getOperand(i: 2).getReg();
1053
1054 LLT VecTy = MRI.getType(Reg: Src);
1055 LLT ScalarTy = VecTy.getScalarType();
1056 unsigned NumElts = VecTy.getNumElements();
1057 MachineRegisterInfo::VRegAttrs VgprRB_EltTy = {.RCOrRB: VgprRB, .Ty: ScalarTy};
1058
1059 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_EltTy, Op: Src);
1060
1061 if (ScalarTy.getSizeInBits() == 32) {
1062 Register PrevSelect = Unmerge.getReg(Idx: 0);
1063 for (unsigned I = 1; I < NumElts; ++I) {
1064 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1065 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1066 PrevSelect =
1067 B.buildSelect(Res: VgprRB_EltTy, Tst: Cmp, Op0: Unmerge.getReg(Idx: I), Op1: PrevSelect)
1068 .getReg(Idx: 0);
1069 }
1070 B.buildCopy(Res: Dst, Op: PrevSelect);
1071 } else if (ScalarTy.getSizeInBits() == 64) {
1072 auto InitUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: 0));
1073 Register PrevLo = InitUnmerge.getReg(Idx: 0);
1074 Register PrevHi = InitUnmerge.getReg(Idx: 1);
1075 for (unsigned I = 1; I < NumElts; ++I) {
1076 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1077 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1078 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: I));
1079 PrevLo = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 0), Op1: PrevLo)
1080 .getReg(Idx: 0);
1081 PrevHi = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 1), Op1: PrevHi)
1082 .getReg(Idx: 0);
1083 }
1084 B.buildMergeLikeInstr(Res: Dst, Ops: {PrevLo, PrevHi});
1085 } else {
1086 reportGISelFailure(
1087 MF, MORE, DEBUG_TYPE,
1088 Msg: "AMDGPU RegBankLegalize: ExtrVecEltToSel unsupported element type", MI);
1089 return false;
1090 }
1091
1092 MI.eraseFromParent();
1093 return true;
1094}
1095
1096bool RegBankLegalizeHelper::lowerExtrVecEltTo32(MachineInstr &MI) {
1097 // Reduce a 64-bit element extract to two 32-bit extracts:
1098 // vec32 = bitcast <N x s64> to <2N x s32>
1099 // lo = vec32[idx * 2]
1100 // hi = vec32[idx * 2 + 1]
1101 // result = merge(lo, hi)
1102 //
1103 // When the index is uniform, all lanes extract the same element, so we can
1104 // just split the s64 extract into two s32 extracts which lower to MOVREL.
1105 Register Dst = MI.getOperand(i: 0).getReg();
1106 Register Src = MI.getOperand(i: 1).getReg();
1107 Register Idx = MI.getOperand(i: 2).getReg();
1108
1109 LLT SrcTy = MRI.getType(Reg: Src);
1110 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1111
1112 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1113 "expected VGPR src and SGPR idx");
1114
1115 auto CastSrc = B.buildBitcast(Dst: {VgprRB, Vec32Ty}, Src);
1116
1117 // Calculate new Lo and Hi indices
1118 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1119 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1120 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1121
1122 auto ExtLo = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxLo);
1123 auto ExtHi = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxHi);
1124
1125 B.buildMergeLikeInstr(Res: Dst, Ops: {ExtLo.getReg(Idx: 0), ExtHi.getReg(Idx: 0)});
1126
1127 MI.eraseFromParent();
1128 return true;
1129}
1130
1131bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
1132 // Lower insert vector element to a compare-select chain:
1133 // for i in 0..N-1:
1134 // result[i] = (idx == i) ? elt : srcVec[i]
1135 // dst = merge(result[0..N-1])
1136 //
1137 // VGPR B64 requires splitting to lo/hi s32 pairs since there is no
1138 // v_cndmask_b64. SGPR B64/B32 and VGPR B32 can be handled natively.
1139 Register Dst = MI.getOperand(i: 0).getReg();
1140 Register Src = MI.getOperand(i: 1).getReg();
1141 Register Elt = MI.getOperand(i: 2).getReg();
1142 Register Idx = MI.getOperand(i: 3).getReg();
1143
1144 LLT VecTy = MRI.getType(Reg: Src);
1145 LLT ScalarTy = VecTy.getScalarType();
1146 unsigned NumElts = VecTy.getNumElements();
1147 const RegisterBank *SrcRB = MRI.getRegBank(Reg: Src);
1148 bool IsSGPR = (SrcRB == SgprRB);
1149 SmallVector<Register, 16> Selects;
1150
1151 if (!IsSGPR && ScalarTy.getSizeInBits() == 64) {
1152 // VGPR B64: split to 32-bit lo/hi since there is no v_cndmask_b64.
1153 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1154 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1155 Register EltLo = EltUnmerge.getReg(Idx: 0);
1156 Register EltHi = EltUnmerge.getReg(Idx: 1);
1157 for (unsigned I = 0; I < NumElts; ++I) {
1158 auto IdxConst = B.buildConstant(Res: VgprRB_I32, Val: I);
1159 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1160 Selects.push_back(
1161 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltLo, Op1: Unmerge.getReg(Idx: 2 * I))
1162 .getReg(Idx: 0));
1163 Selects.push_back(
1164 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltHi, Op1: Unmerge.getReg(Idx: 2 * I + 1))
1165 .getReg(Idx: 0));
1166 }
1167 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * NumElts, ScalarSizeInBits: 32);
1168 auto Vec32 = B.buildBuildVector(Res: {VgprRB, Vec32Ty}, Ops: Selects);
1169 B.buildBitcast(Dst, Src: Vec32);
1170 } else if (ScalarTy.getSizeInBits() == 32 || ScalarTy.getSizeInBits() == 64) {
1171 // B32 (any bank) and SGPR B64: element-wise select at native width.
1172 MachineRegisterInfo::VRegAttrs SrcRB_EltTy = {.RCOrRB: SrcRB, .Ty: ScalarTy};
1173 MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_I32 : VccRB_S1;
1174 auto Unmerge = B.buildUnmerge(Attrs: SrcRB_EltTy, Op: Src);
1175 for (unsigned I = 0; I < NumElts; ++I) {
1176 auto IdxConst = B.buildConstant(Res: SgprRB_I32, Val: I);
1177 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpTy, Op0: Idx, Op1: IdxConst);
1178 Selects.push_back(
1179 Elt: B.buildSelect(Res: SrcRB_EltTy, Tst: Cmp, Op0: Elt, Op1: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
1180 }
1181 B.buildMergeLikeInstr(Res: Dst, Ops: Selects);
1182 } else {
1183 reportGISelFailure(
1184 MF, MORE, DEBUG_TYPE,
1185 Msg: "AMDGPU RegBankLegalize: InsVecEltToSel unsupported element type", MI);
1186 return false;
1187 }
1188
1189 MI.eraseFromParent();
1190 return true;
1191}
1192
1193bool RegBankLegalizeHelper::lowerInsVecEltTo32(MachineInstr &MI) {
1194 // Reduce a 64-bit element insert to two 32-bit inserts:
1195 // vec32 = bitcast <N x s64> to <2N x s32>
1196 // lo, hi = unmerge elt
1197 // vec32[idx * 2] = lo
1198 // vec32[idx * 2 + 1] = hi
1199 // dst = bitcast <2N x s32> to <N x s64>
1200 //
1201 // When the index is uniform, all lanes insert at the same position, so we
1202 // can split the s64 insert into two s32 inserts which lower to MOVREL/GPRIDX.
1203 Register Dst = MI.getOperand(i: 0).getReg();
1204 Register Src = MI.getOperand(i: 1).getReg();
1205 Register Elt = MI.getOperand(i: 2).getReg();
1206 Register Idx = MI.getOperand(i: 3).getReg();
1207
1208 LLT SrcTy = MRI.getType(Reg: Src);
1209 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1210
1211 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1212 "expected VGPR src and SGPR idx");
1213
1214 MachineRegisterInfo::VRegAttrs VgprRB_Vec32Ty = {.RCOrRB: VgprRB, .Ty: Vec32Ty};
1215
1216 auto CastSrc = B.buildBitcast(Dst: VgprRB_Vec32Ty, Src);
1217 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1218
1219 // Calculate new Lo and Hi indices
1220 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1221 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1222 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1223
1224 auto InsLo = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: CastSrc,
1225 Elt: EltUnmerge.getReg(Idx: 0), Idx: IdxLo);
1226 auto InsHi = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: InsLo,
1227 Elt: EltUnmerge.getReg(Idx: 1), Idx: IdxHi);
1228
1229 B.buildBitcast(Dst, Src: InsHi);
1230
1231 MI.eraseFromParent();
1232 return true;
1233}
1234
1235bool RegBankLegalizeHelper::lowerAbsToNegMax(MachineInstr &MI) {
1236 // Lower divergent G_ABS to smax(x, 0 - x) in the VGPR bank:
1237 // zero = 0
1238 // neg = G_SUB zero, x
1239 // dst = G_SMAX x, neg
1240 //
1241 // There is no integer v_abs instruction on AMDGPU, so divergent G_ABS is
1242 // expanded to this sub/smax pair.
1243 Register DstReg = MI.getOperand(i: 0).getReg();
1244 Register SrcReg = MI.getOperand(i: 1).getReg();
1245 LLT Ty = MRI.getType(Reg: DstReg);
1246
1247 Register Zero;
1248 if (Ty == V2S16) {
1249 // buildConstant cannot produce a V2S16 directly; pack two S16 zeros.
1250 Register Zero16 = B.buildConstant(Res: {VgprRB, I16}, Val: 0).getReg(Idx: 0);
1251 Zero = B.buildBuildVector(Res: {VgprRB, Ty}, Ops: {Zero16, Zero16}).getReg(Idx: 0);
1252 } else {
1253 assert((Ty == S32 || Ty == S16) && "unexpected type for AbsToNegMax");
1254 Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0).getReg(Idx: 0);
1255 }
1256
1257 auto Neg = B.buildSub(Dst: {VgprRB, Ty}, Src0: Zero, Src1: SrcReg);
1258 B.buildSMax(Dst: DstReg, Src0: SrcReg, Src1: Neg);
1259 MI.eraseFromParent();
1260 return true;
1261}
1262
1263bool RegBankLegalizeHelper::lowerAbsToS32(MachineInstr &MI) {
1264 // Lower uniform V2S16 abs by unpacking the values to two separate SGPR
1265 // registers and re-emitting G_ABS on each:
1266 // packed = bitcast <2 x s16> src to s32
1267 // lo = sext_inreg packed, 16
1268 // hi = ashr packed, 16
1269 // dst = build_vector_trunc G_ABS(lo), G_ABS(hi)
1270 //
1271 // SALU only has s_abs_i32, with no direct uniform V2S16 abs. The
1272 // re-emitted G_ABS(SgprRB, S32) selects to s_abs_i32 on each value.
1273 auto Bitcast = B.buildBitcast(Dst: {SgprRB_I32}, Src: MI.getOperand(i: 1).getReg());
1274 auto SextInReg = B.buildSExtInReg(Res: {SgprRB_I32}, Op: Bitcast, ImmOp: 16);
1275 auto ShiftHi =
1276 B.buildAShr(Dst: {SgprRB_I32}, Src0: Bitcast, Src1: B.buildConstant(Res: {SgprRB_I32}, Val: 16));
1277
1278 auto AbsLo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {SextInReg});
1279 auto AbsHi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {ShiftHi});
1280 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
1281 Ops: {AbsLo.getReg(Idx: 0), AbsHi.getReg(Idx: 0)});
1282
1283 MI.eraseFromParent();
1284 return true;
1285}
1286
1287// Ported from SITargetLowering::lowerSET_ROUNDING in SIISelLowering.cpp.
1288// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1289bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
1290 Register NewMode = MI.getOperand(i: 0).getReg();
1291
1292 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
1293 // hardware MODE.fp_round values.
1294 if (auto ConstMode = getIConstantVRegValWithLookThrough(VReg: NewMode, MRI)) {
1295 uint32_t ClampedVal = std::min(
1296 a: static_cast<uint32_t>(ConstMode->Value.getZExtValue()),
1297 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
1298 uint32_t DecodedVal = AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal);
1299 NewMode = B.buildConstant(Res: SgprRB_I32, Val: DecodedVal).getReg(Idx: 0);
1300 } else {
1301 // If we know the input can only be one of the supported standard modes in
1302 // the range 0-3, we can use a simplified mapping to hardware values.
1303 KnownBits Known = VT->getKnownBits(R: NewMode);
1304 const bool UseReducedTable = Known.countMinLeadingZeros() >= 30;
1305 // The supported standard values are 0-3. The extended values start at 8. We
1306 // need to offset by 4 if the value is in the extended range.
1307
1308 if (UseReducedTable) {
1309 // Truncate to the low 32-bits.
1310 auto BitTable = B.buildConstant(
1311 Res: SgprRB_I32, Val: AMDGPU::FltRoundToHWConversionTable & 0xffff);
1312
1313 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1314 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: NewMode, Src1: Two);
1315
1316 NewMode =
1317 B.buildLShr(Dst: SgprRB_I32, Src0: BitTable, Src1: RoundModeTimesNumBits).getReg(Idx: 0);
1318
1319 // TODO: A demanded-bits simplification on the setreg source here could
1320 // likely reduce the table extracted bits into inline immediates.
1321 } else {
1322 // table_index = umin(value, value - 4)
1323 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
1324 auto NegFour = B.buildConstant(Res: SgprRB_I32, Val: -4);
1325 auto OffsetEnum = B.buildAdd(Dst: SgprRB_I32, Src0: NewMode, Src1: NegFour);
1326 auto IndexVal = B.buildUMin(Dst: SgprRB_I32, Src0: NewMode, Src1: OffsetEnum);
1327
1328 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1329 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: IndexVal, Src1: Two);
1330
1331 auto BitTable =
1332 B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundToHWConversionTable);
1333 auto TableValue =
1334 B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1335 // No need to mask out the high bits since the setreg will ignore them
1336 // anyway.
1337 NewMode = B.buildTrunc(Res: SgprRB_I32, Op: TableValue).getReg(Idx: 0);
1338 }
1339 }
1340
1341 // N.B. The setreg will be later folded into s_round_mode on supported
1342 // targets.
1343 uint32_t BothRoundHwReg =
1344 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1345 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
1346 /*HasSideEffects=*/true, /*isConvergent=*/false)
1347 .addImm(Val: static_cast<int16_t>(BothRoundHwReg))
1348 .addReg(RegNo: NewMode);
1349
1350 MI.eraseFromParent();
1351 return true;
1352}
1353
1354// Ported from SITargetLowering::lowerGET_ROUNDING in SIISelLowering.cpp.
1355// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1356bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
1357 Register Dst = MI.getOperand(i: 0).getReg();
1358
1359 uint32_t BothRoundHwReg =
1360 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1361 auto GetReg =
1362 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {SgprRB_I32},
1363 /*HasSideEffects=*/true, /*isConvergent=*/false)
1364 .addImm(Val: BothRoundHwReg);
1365
1366 // There are two rounding modes, one for f32 and one for f64/f16. We only
1367 // report in the standard value range if both are the same.
1368 //
1369 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
1370 // ties away from zero is not supported, and the other values are rotated by
1371 // 1.
1372 //
1373 // If the two rounding modes are not the same, report a target defined value.
1374
1375 // Mode register rounding mode fields:
1376 //
1377 // [1:0] Single-precision round mode.
1378 // [3:2] Double/Half-precision round mode.
1379 //
1380 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
1381 //
1382 // Hardware Spec
1383 // Toward-0 3 0
1384 // Nearest Even 0 1
1385 // +Inf 1 2
1386 // -Inf 2 3
1387 // NearestAway0 N/A 4
1388 //
1389 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
1390 // table we can index by the raw hardware mode.
1391 //
1392 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
1393 auto BitTable = B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundConversionTable);
1394
1395 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1396 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: GetReg, Src1: Two);
1397
1398 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
1399 // knew only one mode was demanded.
1400 auto TableValue = B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1401 auto TruncTable = B.buildTrunc(Res: SgprRB_I32, Op: TableValue);
1402
1403 auto EntryMask = B.buildConstant(Res: SgprRB_I32, Val: 0xf);
1404 auto TableEntry = B.buildAnd(Dst: SgprRB_I32, Src0: TruncTable, Src1: EntryMask);
1405
1406 // There's a gap in the 4-bit encoded table and actual enum values, so offset
1407 // if it's an extended value.
1408 auto Four = B.buildConstant(Res: SgprRB_I32, Val: 4);
1409 auto EnumOffset = B.buildAdd(Dst: SgprRB_I32, Src0: TableEntry, Src1: Four);
1410 auto IsStandardMode =
1411 B.buildICmp(Pred: CmpInst::ICMP_ULT, Res: SgprRB_I32, Op0: TableEntry, Op1: Four);
1412 B.buildSelect(Res: Dst, Tst: IsStandardMode, Op0: TableEntry, Op1: EnumOffset);
1413
1414 MI.eraseFromParent();
1415 return true;
1416}
1417
1418bool RegBankLegalizeHelper::lower(MachineInstr &MI,
1419 const RegBankLLTMapping &Mapping,
1420 WaterfallInfo &WFI) {
1421
1422 switch (Mapping.LoweringMethod) {
1423 case DoNotLower:
1424 break;
1425 case VccExtToSel:
1426 return lowerVccExtToSel(MI);
1427 case UniExtToSel: {
1428 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1429 auto True = B.buildConstant(Res: {SgprRB, Ty},
1430 Val: MI.getOpcode() == AMDGPU::G_SEXT ? -1 : 1);
1431 auto False = B.buildConstant(Res: {SgprRB, Ty}, Val: 0);
1432 // Input to G_{Z|S}EXT is 'Legalizer legal' S1. Most common case is compare.
1433 // We are making select here. S1 cond was already 'any-extended to S32' +
1434 // 'AND with 1 to clean high bits' by Sgpr32AExtBoolInReg.
1435 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: MI.getOperand(i: 1).getReg(), Op0: True,
1436 Op1: False);
1437 MI.eraseFromParent();
1438 return true;
1439 }
1440 case UnpackBitShift:
1441 return lowerUnpackBitShift(MI);
1442 case UnpackMinMax:
1443 return lowerUnpackMinMax(MI);
1444 case ScalarizeToS16:
1445 return lowerSplitTo16(MI);
1446 case Ext32To64: {
1447 const RegisterBank *RB = MRI.getRegBank(Reg: MI.getOperand(i: 0).getReg());
1448 MachineInstrBuilder Hi;
1449 switch (MI.getOpcode()) {
1450 case AMDGPU::G_ZEXT: {
1451 Hi = B.buildConstant(Res: {RB, I32}, Val: 0);
1452 break;
1453 }
1454 case AMDGPU::G_SEXT: {
1455 // Replicate sign bit from 32-bit extended part.
1456 auto ShiftAmt = B.buildConstant(Res: {RB, I32}, Val: 31);
1457 Hi = B.buildAShr(Dst: {RB, MRI.getType(Reg: MI.getOperand(i: 1).getReg())},
1458 Src0: MI.getOperand(i: 1).getReg(), Src1: ShiftAmt);
1459 break;
1460 }
1461 case AMDGPU::G_ANYEXT: {
1462 Hi = B.buildUndef(Res: {RB, I32});
1463 break;
1464 }
1465 default:
1466 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1467 Msg: "AMDGPU RegBankLegalize: Ext32To64, unsuported opcode",
1468 MI);
1469 return false;
1470 }
1471
1472 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(),
1473 Ops: {MI.getOperand(i: 1).getReg(), Hi});
1474 MI.eraseFromParent();
1475 return true;
1476 }
1477 case UniCstExt: {
1478 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1479 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: ConstVal);
1480
1481 MI.eraseFromParent();
1482 return true;
1483 }
1484 case VgprToVccCopy: {
1485 Register Src = MI.getOperand(i: 1).getReg();
1486 LLT Ty = MRI.getType(Reg: Src);
1487 // Take lowest bit from each lane and put it in lane mask.
1488 // Lowering via compare, but we need to clean high bits first as compare
1489 // compares all bits in register.
1490 Register BoolSrc = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
1491 if (Ty == S64) {
1492 auto Src64 = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1493 auto One = B.buildConstant(Res: VgprRB_I32, Val: 1);
1494 auto AndLo = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 0), Src1: One);
1495 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
1496 auto AndHi = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 1), Src1: Zero);
1497 B.buildMergeLikeInstr(Res: BoolSrc, Ops: {AndLo, AndHi});
1498 } else {
1499 assert(Ty == S32 || Ty == S16);
1500 auto One = B.buildConstant(Res: {VgprRB, Ty}, Val: 1);
1501 B.buildAnd(Dst: BoolSrc, Src0: Src, Src1: One);
1502 }
1503 auto Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
1504 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 0).getReg(), Op0: BoolSrc, Op1: Zero);
1505 MI.eraseFromParent();
1506 return true;
1507 }
1508 case V_BFE:
1509 return lowerV_BFE(MI);
1510 case S_BFE:
1511 return lowerS_BFE(MI);
1512 case UniMAD64:
1513 return lowerUniMAD64(MI);
1514 case UniMul64: {
1515 B.buildMul(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2));
1516 MI.eraseFromParent();
1517 return true;
1518 }
1519 case DivSMulToMAD: {
1520 auto Op1 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 1));
1521 auto Op2 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 2));
1522 auto Zero = B.buildConstant(Res: VgprRB_I64, Val: 0);
1523
1524 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_AMDGPU_S_MUL_U64_U32
1525 ? AMDGPU::G_AMDGPU_MAD_U64_U32
1526 : AMDGPU::G_AMDGPU_MAD_I64_I32;
1527
1528 B.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0).getReg(), SgprRB_I32},
1529 SrcOps: {Op1, Op2, Zero});
1530 MI.eraseFromParent();
1531 return true;
1532 }
1533 case SplitTo32:
1534 return lowerSplitTo32(MI);
1535 case SplitTo32Mul:
1536 return lowerSplitTo32Mul(MI);
1537 case SplitTo32Select:
1538 return lowerSplitTo32Select(MI);
1539 case SplitTo32SExtInReg:
1540 return lowerSplitTo32SExtInReg(MI);
1541 case CtPop64To32: {
1542 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1543 auto LoPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 0));
1544 auto HiPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 1));
1545 // Max popcount of two 32-bit values is 64, so this add cannot overflow.
1546 B.buildAdd(Dst: MI.getOperand(i: 0).getReg(), Src0: LoPopCnt, Src1: HiPopCnt,
1547 Flags: MachineInstr::NoSWrap | MachineInstr::NoUWrap);
1548
1549 MI.eraseFromParent();
1550 break;
1551 }
1552 case S_BUF_to_BUF:
1553 return lowerSBufToBuf(MI, WFI);
1554 case SplitLoad: {
1555 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1556 unsigned Size = DstTy.getSizeInBits();
1557 // Even split to 128-bit loads
1558 if (Size > 128) {
1559 LLT B128;
1560 if (DstTy.isVector()) {
1561 LLT EltTy = DstTy.getElementType();
1562 B128 = LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1563 } else {
1564 B128 = LLT::integer(SizeInBits: 128);
1565 }
1566 if (Size / 128 == 2)
1567 splitLoad(MI, LLTBreakdown: {B128, B128});
1568 else if (Size / 128 == 4)
1569 splitLoad(MI, LLTBreakdown: {B128, B128, B128, B128});
1570 else {
1571 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1572 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1573 MI);
1574 return false;
1575 }
1576 }
1577 // 64 and 32 bit load
1578 else if (DstTy == S96)
1579 splitLoad(MI, LLTBreakdown: {S64, S32}, MergeTy: S32);
1580 else if (DstTy == V3S32)
1581 splitLoad(MI, LLTBreakdown: {V2S32, S32}, MergeTy: S32);
1582 else if (DstTy == V6S16)
1583 splitLoad(MI, LLTBreakdown: {V4S16, V2S16}, MergeTy: V2S16);
1584 else {
1585 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1586 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1587 MI);
1588 return false;
1589 }
1590 return true;
1591 }
1592 case DynStackAlloc: {
1593 const auto &TFI = *ST.getFrameLowering();
1594 // Guard in case the stack growth direction ever changes with scratch
1595 // instructions.
1596 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1597 "Stack grows upwards for AMDGPU");
1598
1599 Register Dst = MI.getOperand(i: 0).getReg();
1600 Register AllocSize = MI.getOperand(i: 1).getReg();
1601 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1602
1603 // Erase before building new instrs to avoid hitting multiple Dst assert
1604 // with CSE.
1605 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
1606 MI.eraseFromParent();
1607
1608 if (MRI.getRegBank(Reg: AllocSize) != SgprRB) {
1609 auto WaveReduction =
1610 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {SgprRB_I32})
1611 .addUse(RegNo: AllocSize)
1612 .addImm(Val: 0);
1613 AllocSize = WaveReduction.getReg(Idx: 0);
1614 }
1615
1616 LLT PtrTy = MRI.getType(Reg: Dst);
1617 assert(PtrTy.getSizeInBits() == 32 &&
1618 "Expected 32-bit pointer for stack allocation");
1619 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1620 Register SPReg = Info->getStackPtrOffsetReg();
1621
1622 // When using flat-scratch, the stack offset is unscaled.
1623 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1624 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1625
1626 Register AdjustedSize = AllocSize;
1627 if (!HasFlatScratch) {
1628 auto WaveSize = B.buildConstant(Res: SgprRB_I32, Val: WavefrontSizeLog2);
1629 AdjustedSize = B.buildShl(Dst: SgprRB_I32, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1630 }
1631 if (Alignment > TFI.getStackAlign()) {
1632 const uint64_t EffectiveAlignment =
1633 Alignment.value() << (HasFlatScratch ? 0 : WavefrontSizeLog2);
1634 auto OldSP = B.buildCopy(Res: {SgprRB, PtrTy}, Op: SPReg);
1635 auto Tmp1 =
1636 B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: OldSP,
1637 Op1: B.buildConstant(Res: SgprRB_I32, Val: EffectiveAlignment - 1));
1638 uint64_t Mask = maskTrailingZeros<uint64_t>(N: Log2_64(Value: EffectiveAlignment));
1639 B.buildPtrMask(Res: Dst, Op0: Tmp1, Op1: B.buildConstant(Res: SgprRB_I32, Val: Mask));
1640 } else {
1641 B.buildCopy(Res: Dst, Op: SPReg);
1642 }
1643 auto PtrAdd = B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: Dst, Op1: AdjustedSize);
1644 B.buildCopy(Res: SPReg, Op: PtrAdd);
1645 return true;
1646 }
1647 case WidenLoad: {
1648 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1649 if (DstTy == S96)
1650 widenLoad(MI, WideTy: S128);
1651 else if (DstTy == V3S32)
1652 widenLoad(MI, WideTy: V4S32, MergeTy: S32);
1653 else if (DstTy == V6S16)
1654 widenLoad(MI, WideTy: V8S16, MergeTy: V2S16);
1655 else {
1656 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1657 Msg: "AMDGPU RegBankLegalize: WidenLoad, unsuported type",
1658 MI);
1659 return false;
1660 }
1661 return true;
1662 }
1663 case UnpackAExt:
1664 return lowerUnpackAExt(MI);
1665 case WidenMMOToS32:
1666 return widenMMOToS32(MI&: cast<GAnyLoad>(Val&: MI));
1667 case VerifyAllSgpr: {
1668 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1669 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1670 }));
1671 return true;
1672 }
1673 case ApplyAllVgpr: {
1674 assert(llvm::all_of(MI.defs(), [&](const MachineOperand &Op) {
1675 return MRI.getRegBankOrNull(Op.getReg()) == VgprRB;
1676 }));
1677 B.setInstrAndDebugLoc(MI);
1678 for (unsigned i = MI.getNumDefs(); i < MI.getNumOperands(); ++i) {
1679 MachineOperand &Op = MI.getOperand(i);
1680 if (!Op.isReg())
1681 continue;
1682 Register Reg = Op.getReg();
1683 if (MRI.getRegBank(Reg) != VgprRB) {
1684 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
1685 Op.setReg(Copy.getReg(Idx: 0));
1686 }
1687 }
1688 return true;
1689 }
1690 case UnmergeToShiftTrunc: {
1691 GUnmerge *Unmerge = dyn_cast<GUnmerge>(Val: &MI);
1692 LLT Ty = MRI.getType(Reg: Unmerge->getSourceReg());
1693 if (Ty.getSizeInBits() % 32 != 0) {
1694 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1695 Msg: "AMDGPU RegBankLegalize: unmerge not multiple of 32",
1696 MI);
1697 return false;
1698 }
1699
1700 B.setInstrAndDebugLoc(MI);
1701 if (Ty.getSizeInBits() > 32) {
1702 auto UnmergeV2S16 =
1703 B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: V2S16}, Op: Unmerge->getSourceReg());
1704 for (unsigned i = 0; i < UnmergeV2S16->getNumDefs(); ++i) {
1705 auto [Dst0I32, Dst1I32] =
1706 unpackAExt(Reg: UnmergeV2S16->getOperand(i).getReg());
1707 B.buildTrunc(Res: MI.getOperand(i: i * 2).getReg(), Op: Dst0I32);
1708 B.buildTrunc(Res: MI.getOperand(i: i * 2 + 1).getReg(), Op: Dst1I32);
1709 }
1710 } else {
1711 auto [Dst0I32, Dst1I32] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
1712 B.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Dst0I32);
1713 B.buildTrunc(Res: MI.getOperand(i: 1).getReg(), Op: Dst1I32);
1714 }
1715
1716 MI.eraseFromParent();
1717 return true;
1718 }
1719 case AextToS32InIncomingBlockGPHI: {
1720 Register Dst = MI.getOperand(i: 0).getReg();
1721 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
1722 B.setInsertPt(MBB&: *MI.getParent(), II: MI.getParent()->getFirstNonPHI());
1723 MI.getOperand(i: 0).setReg(NewDst);
1724 B.buildTrunc(Res: Dst, Op: NewDst);
1725
1726 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1727 Register UseReg = MI.getOperand(i).getReg();
1728
1729 auto DefMI = MRI.getVRegDef(Reg: UseReg)->getIterator();
1730 MachineBasicBlock *DefMBB = DefMI->getParent();
1731
1732 B.setInsertPt(MBB&: *DefMBB, II: DefMBB->SkipPHIsAndLabels(I: std::next(x: DefMI)));
1733
1734 auto NewUse = B.buildAnyExt(Res: SgprRB_I32, Op: UseReg);
1735 MI.getOperand(i).setReg(NewUse.getReg(Idx: 0));
1736 }
1737 break;
1738 }
1739 case VerifyAllSgprGPHI: {
1740 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1741 if (Op.isMBB())
1742 return true;
1743 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1744 }));
1745 return true;
1746 }
1747 case VerifyAllSgprOrVgprGPHI: {
1748 assert(MRI.getRegBankOrNull(MI.getOperand(0).getReg()) == VgprRB);
1749 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1750 if (Op.isMBB())
1751 return true;
1752 const RegisterBank *RB = MRI.getRegBankOrNull(Op.getReg());
1753 return RB == VgprRB || RB == SgprRB;
1754 }));
1755 return true;
1756 }
1757 case ApplyINTRIN_IMAGE: {
1758 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
1759 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
1760 assert(RSrcIntrin && RSrcIntrin->IsImage);
1761 // The reported argument index is relative to the IR intrinsic call
1762 // arguments, so shift by the number of defs and the intrinsic ID.
1763 unsigned RsrcIdx = RSrcIntrin->RsrcArg + MI.getNumExplicitDefs() + 1;
1764 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1765 }
1766 case ApplyBVH_INTERSECT_RAY: {
1767 // Rsrc is the last register operand. Base BVH trails an A16 immediate
1768 // after rsrc; dual/BVH8 do not. Scan backwards for the last virtual
1769 // register.
1770 unsigned RsrcIdx = MI.getNumOperands();
1771 while (RsrcIdx-- > MI.getNumExplicitDefs()) {
1772 const MachineOperand &Op = MI.getOperand(i: RsrcIdx);
1773 if (Op.isReg() && Op.getReg().isVirtual())
1774 break;
1775 }
1776 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1777 }
1778 case SplitBitCount64To32:
1779 return lowerSplitBitCount64To32(MI);
1780 case ExtrVecEltToSel:
1781 return lowerExtrVecEltToSel(MI);
1782 case ExtrVecEltTo32:
1783 return lowerExtrVecEltTo32(MI);
1784 case InsVecEltToSel:
1785 return lowerInsVecEltToSel(MI);
1786 case InsVecEltTo32:
1787 return lowerInsVecEltTo32(MI);
1788 case AbsToNegMax:
1789 return lowerAbsToNegMax(MI);
1790 case AbsToS32:
1791 return lowerAbsToS32(MI);
1792 case DeletePrefetch:
1793 MI.eraseFromParent();
1794 return true;
1795 case LowerSetRounding:
1796 return lowerSetRounding(MI);
1797 case LowerGetRounding:
1798 return lowerGetRounding(MI);
1799 }
1800
1801 return true;
1802}
1803
1804LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
1805 switch (ID) {
1806 case Vcc:
1807 case UniInVcc:
1808 return LLT::scalar(SizeInBits: 1);
1809 case Sgpr16:
1810 case Vgpr16:
1811 case UniInVgprS16:
1812 return LLT::scalar(SizeInBits: 16);
1813 case Sgpr32:
1814 case Sgpr32_WF:
1815 case Sgpr32Trunc:
1816 case Sgpr32AExt:
1817 case Sgpr32AExtBoolInReg:
1818 case Sgpr32SExt:
1819 case Sgpr32ZExt:
1820 case UniInVgprS32:
1821 case Sgpr32ToVgprDst:
1822 case Vgpr32:
1823 case Vgpr32AExt:
1824 case Vgpr32SExt:
1825 case Vgpr32ZExt:
1826 return LLT::scalar(SizeInBits: 32);
1827 case Sgpr64:
1828 case Vgpr64:
1829 case UniInVgprS64:
1830 case Sgpr64ToVgprDst:
1831 return LLT::scalar(SizeInBits: 64);
1832 case Sgpr128:
1833 case Vgpr128:
1834 return LLT::scalar(SizeInBits: 128);
1835 case SgprP0:
1836 case SgprP0Call_WF:
1837 case VgprP0:
1838 return LLT::pointer(AddressSpace: 0, SizeInBits: 64);
1839 case SgprP1:
1840 case VgprP1:
1841 return LLT::pointer(AddressSpace: 1, SizeInBits: 64);
1842 case SgprP2:
1843 case VgprP2:
1844 return LLT::pointer(AddressSpace: 2, SizeInBits: 32);
1845 case SgprP3:
1846 case VgprP3:
1847 return LLT::pointer(AddressSpace: 3, SizeInBits: 32);
1848 case SgprP4:
1849 case SgprP4Call_WF:
1850 case VgprP4:
1851 return LLT::pointer(AddressSpace: 4, SizeInBits: 64);
1852 case SgprP5:
1853 case VgprP5:
1854 return LLT::pointer(AddressSpace: 5, SizeInBits: 32);
1855 case SgprP6:
1856 return LLT::pointer(AddressSpace: 6, SizeInBits: 32);
1857 case SgprP8:
1858 return LLT::pointer(AddressSpace: 8, SizeInBits: 128);
1859 case SgprV2S16:
1860 case VgprV2S16:
1861 case UniInVgprV2S16:
1862 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
1863 case SgprV2S32:
1864 case VgprV2S32:
1865 case UniInVgprV2S32:
1866 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
1867 case VgprV3S32:
1868 case UniInVgprV3S32:
1869 return LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
1870 case VgprV4S16:
1871 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
1872 case VgprV8S16:
1873 case UniInVgprV8S16:
1874 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
1875 case VgprV16S16:
1876 case UniInVgprV16S16:
1877 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
1878 case SgprV4S32:
1879 case SgprV4S32_WF:
1880 case SgprV4S32_ReadFirstLane:
1881 case VgprV4S32:
1882 case UniInVgprV4S32:
1883 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
1884 case VgprV8S32:
1885 case UniInVgprV8S32:
1886 case SgprV8S32_ReadFirstLane:
1887 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
1888 case VgprV2S64:
1889 case UniInVgprV2S64:
1890 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1891 case VgprV6S32:
1892 case UniInVgprV6S32:
1893 return LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32);
1894 case VgprV16S32:
1895 case UniInVgprV16S32:
1896 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
1897 case VgprV32S16:
1898 case UniInVgprV32S16:
1899 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
1900 case VgprV32S32:
1901 case UniInVgprV32S32:
1902 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32);
1903 default:
1904 return LLT();
1905 }
1906}
1907
1908LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
1909 switch (ID) {
1910 case SgprB32:
1911 case VgprB32:
1912 case SgprB32_M0:
1913 case SgprB32_ReadFirstLane:
1914 case UniInVgprB32:
1915 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
1916 isAnyPtr(Ty, Width: 32))
1917 return Ty;
1918 return LLT();
1919 case SgprPtr32:
1920 case VgprPtr32:
1921 return isAnyPtr(Ty, Width: 32) ? Ty : LLT();
1922 case SgprPtr64:
1923 case VgprPtr64:
1924 return isAnyPtr(Ty, Width: 64) ? Ty : LLT();
1925 case SgprPtr128:
1926 case VgprPtr128:
1927 return isAnyPtr(Ty, Width: 128) ? Ty : LLT();
1928 case SgprB64:
1929 case VgprB64:
1930 case SgprB64_ReadFirstLane:
1931 case UniInVgprB64:
1932 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
1933 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
1934 return Ty;
1935 return LLT();
1936 case SgprB96:
1937 case VgprB96:
1938 case UniInVgprB96:
1939 if (Ty == LLT::scalar(SizeInBits: 96) || Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) ||
1940 Ty == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 16))
1941 return Ty;
1942 return LLT();
1943 case SgprB128:
1944 case VgprB128:
1945 case UniInVgprB128:
1946 if (Ty.getSizeInBits() == 128)
1947 return Ty;
1948 return LLT();
1949 case VgprB160:
1950 case UniInVgprB160:
1951 if (Ty.getSizeInBits() == 160)
1952 return Ty;
1953 return LLT();
1954 case SgprB256:
1955 case VgprB256:
1956 case UniInVgprB256:
1957 if (Ty.getSizeInBits() == 256)
1958 return Ty;
1959 return LLT();
1960 case SgprB512:
1961 case VgprB512:
1962 case UniInVgprB512:
1963 if (Ty.getSizeInBits() == 512)
1964 return Ty;
1965 return LLT();
1966 case SgprBRC: {
1967 const SIRegisterInfo *TRI =
1968 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1969 unsigned LLTSize = Ty.getSizeInBits();
1970 if (LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize))
1971 return Ty;
1972 return LLT();
1973 }
1974 case VgprBRC: {
1975 const SIRegisterInfo *TRI =
1976 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1977 if (TRI->getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()))
1978 return Ty;
1979 return LLT();
1980 }
1981 default:
1982 return LLT();
1983 }
1984}
1985
1986const RegisterBank *
1987RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
1988 switch (ID) {
1989 case Vcc:
1990 return VccRB;
1991 case Sgpr16:
1992 case Sgpr32:
1993 case Sgpr32_WF:
1994 case Sgpr64:
1995 case Sgpr128:
1996 case SgprP0:
1997 case SgprP0Call_WF:
1998 case SgprP1:
1999 case SgprP2:
2000 case SgprP3:
2001 case SgprP4:
2002 case SgprP4Call_WF:
2003 case SgprP5:
2004 case SgprP6:
2005 case SgprP8:
2006 case SgprPtr32:
2007 case SgprPtr64:
2008 case SgprPtr128:
2009 case SgprV2S16:
2010 case SgprV2S32:
2011 case SgprV4S32:
2012 case SgprV4S32_WF:
2013 case SgprV4S32_ReadFirstLane:
2014 case SgprV8S32_ReadFirstLane:
2015 case SgprB32:
2016 case SgprB64:
2017 case SgprB96:
2018 case SgprB128:
2019 case SgprB256:
2020 case SgprB512:
2021 case SgprBRC:
2022 case UniInVcc:
2023 case UniInVgprS16:
2024 case UniInVgprS32:
2025 case UniInVgprS64:
2026 case UniInVgprV2S16:
2027 case UniInVgprV2S32:
2028 case UniInVgprV3S32:
2029 case UniInVgprV4S32:
2030 case UniInVgprV2S64:
2031 case UniInVgprV6S32:
2032 case UniInVgprV8S16:
2033 case UniInVgprV8S32:
2034 case UniInVgprV16S16:
2035 case UniInVgprV16S32:
2036 case UniInVgprV32S16:
2037 case UniInVgprV32S32:
2038 case UniInVgprB32:
2039 case UniInVgprB64:
2040 case UniInVgprB96:
2041 case UniInVgprB128:
2042 case UniInVgprB160:
2043 case UniInVgprB256:
2044 case UniInVgprB512:
2045 case Sgpr32Trunc:
2046 case Sgpr32AExt:
2047 case Sgpr32AExtBoolInReg:
2048 case Sgpr32SExt:
2049 case Sgpr32ZExt:
2050 return SgprRB;
2051 case AgprAnyTy:
2052 return AgprRB;
2053 case Vgpr16:
2054 case Vgpr32:
2055 case Vgpr64:
2056 case Vgpr128:
2057 case VgprP0:
2058 case VgprP1:
2059 case VgprP2:
2060 case VgprP3:
2061 case VgprP4:
2062 case VgprP5:
2063 case VgprPtr32:
2064 case VgprPtr64:
2065 case VgprPtr128:
2066 case VgprV2S16:
2067 case VgprV2S32:
2068 case VgprV2S64:
2069 case VgprV3S32:
2070 case VgprV4S16:
2071 case VgprV8S16:
2072 case VgprV16S16:
2073 case VgprV4S32:
2074 case VgprV6S32:
2075 case VgprV8S32:
2076 case VgprV16S32:
2077 case VgprV32S16:
2078 case VgprV32S32:
2079 case VgprB32:
2080 case VgprB64:
2081 case VgprB96:
2082 case VgprB128:
2083 case VgprB160:
2084 case VgprB256:
2085 case VgprB512:
2086 case VgprBRC:
2087 case VgprAnyTy:
2088 case Vgpr32AExt:
2089 case Vgpr32SExt:
2090 case Vgpr32ZExt:
2091 case Sgpr32ToVgprDst:
2092 case Sgpr64ToVgprDst:
2093 return VgprRB;
2094 default:
2095 return nullptr;
2096 }
2097}
2098
2099bool RegBankLegalizeHelper::applyMappingDst(
2100 MachineInstr &MI, unsigned &OpIdx,
2101 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs) {
2102 // Defs start from operand 0
2103 for (; OpIdx < MethodIDs.size(); ++OpIdx) {
2104 if (MethodIDs[OpIdx] == None)
2105 continue;
2106 MachineOperand &Op = MI.getOperand(i: OpIdx);
2107 Register Reg = Op.getReg();
2108 LLT Ty = MRI.getType(Reg);
2109 [[maybe_unused]] const RegisterBank *RB = MRI.getRegBank(Reg);
2110
2111 switch (MethodIDs[OpIdx]) {
2112 // vcc, sgpr and vgpr scalars, pointers and vectors
2113 case Vcc:
2114 case Sgpr16:
2115 case Sgpr32:
2116 case Sgpr64:
2117 case Sgpr128:
2118 case SgprP0:
2119 case SgprP1:
2120 case SgprP3:
2121 case SgprP4:
2122 case SgprP5:
2123 case SgprP6:
2124 case SgprP8:
2125 case SgprV2S16:
2126 case SgprV2S32:
2127 case SgprV4S32:
2128 case Vgpr16:
2129 case Vgpr32:
2130 case Vgpr64:
2131 case Vgpr128:
2132 case VgprP0:
2133 case VgprP1:
2134 case VgprP2:
2135 case VgprP3:
2136 case VgprP4:
2137 case VgprP5:
2138 case VgprV2S16:
2139 case VgprV2S32:
2140 case VgprV2S64:
2141 case VgprV3S32:
2142 case VgprV4S16:
2143 case VgprV8S16:
2144 case VgprV16S16:
2145 case VgprV4S32:
2146 case VgprV6S32:
2147 case VgprV8S32:
2148 case VgprV16S32:
2149 case VgprV32S16:
2150 case VgprV32S32: {
2151 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2152 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2153 break;
2154 }
2155 // sgpr and vgpr B-types
2156 case SgprB32:
2157 case SgprB64:
2158 case SgprB96:
2159 case SgprB128:
2160 case SgprB256:
2161 case SgprB512:
2162 case SgprBRC:
2163 case SgprPtr32:
2164 case SgprPtr64:
2165 case SgprPtr128:
2166 case VgprB32:
2167 case VgprB64:
2168 case VgprB96:
2169 case VgprB128:
2170 case VgprB160:
2171 case VgprB256:
2172 case VgprB512:
2173 case VgprBRC:
2174 case VgprPtr32:
2175 case VgprPtr64:
2176 case VgprPtr128: {
2177 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2178 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2179 break;
2180 }
2181 case VgprAnyTy: {
2182 assert(RB == VgprRB);
2183 break;
2184 }
2185 case AgprAnyTy: {
2186 if (RB == AgprRB)
2187 break;
2188 Register NewAgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: AgprRB, .Ty: Ty});
2189 Op.setReg(NewAgprDst);
2190 if (!MRI.use_nodbg_empty(RegNo: Reg))
2191 B.buildCopy(Res: Reg, Op: NewAgprDst);
2192 break;
2193 }
2194 case VgprOrAgprAnyTy: {
2195 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2196 const RegisterBank *DstRB =
2197 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2198 if (RB == DstRB)
2199 break;
2200 Register NewDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: DstRB, .Ty: Ty});
2201 Op.setReg(NewDst);
2202 if (!MRI.use_nodbg_empty(RegNo: Reg))
2203 B.buildCopy(Res: Reg, Op: NewDst);
2204 break;
2205 }
2206 // uniform in vcc/vgpr: scalars, vectors and B-types
2207 case UniInVcc: {
2208 assert(Ty == S1);
2209 assert(RB == SgprRB);
2210 Register NewDst = MRI.createVirtualRegister(RegAttr: VccRB_S1);
2211 Op.setReg(NewDst);
2212 if (!MRI.use_empty(RegNo: Reg)) {
2213 auto CopyS32_Vcc =
2214 B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_SCC_VCC, DstOps: {SgprRB_I32}, SrcOps: {NewDst});
2215 B.buildTrunc(Res: Reg, Op: CopyS32_Vcc);
2216 }
2217 break;
2218 }
2219 case UniInVgprS16: {
2220 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2221 assert(RB == SgprRB);
2222 Register NewVgprDst16 = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2223 Register NewVgprDstI32 = MRI.createVirtualRegister(RegAttr: VgprRB_I32);
2224 Register NewSgprDstI32 = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2225 Op.setReg(NewVgprDst16);
2226 B.buildAnyExt(Res: NewVgprDstI32, Op: NewVgprDst16);
2227 buildReadAnyLane(B, SgprDst: NewSgprDstI32, VgprSrc: NewVgprDstI32, RBI);
2228 B.buildTrunc(Res: Reg, Op: NewSgprDstI32);
2229 break;
2230 }
2231 case UniInVgprS32:
2232 case UniInVgprS64:
2233 case UniInVgprV2S16:
2234 case UniInVgprV2S32:
2235 case UniInVgprV3S32:
2236 case UniInVgprV4S32:
2237 case UniInVgprV2S64:
2238 case UniInVgprV6S32:
2239 case UniInVgprV8S16:
2240 case UniInVgprV8S32:
2241 case UniInVgprV16S16:
2242 case UniInVgprV16S32:
2243 case UniInVgprV32S16:
2244 case UniInVgprV32S32: {
2245 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2246 assert(RB == SgprRB);
2247 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2248 Op.setReg(NewVgprDst);
2249 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2250 break;
2251 }
2252 case UniInVgprB32:
2253 case UniInVgprB64:
2254 case UniInVgprB96:
2255 case UniInVgprB128:
2256 case UniInVgprB160:
2257 case UniInVgprB256:
2258 case UniInVgprB512: {
2259 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2260 assert(RB == SgprRB);
2261 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2262 Op.setReg(NewVgprDst);
2263 AMDGPU::buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2264 break;
2265 }
2266 // sgpr trunc
2267 case Sgpr32Trunc: {
2268 assert(Ty.getSizeInBits() < 32);
2269 assert(RB == SgprRB);
2270 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2271 Op.setReg(NewDst);
2272 if (!MRI.use_empty(RegNo: Reg))
2273 B.buildTrunc(Res: Reg, Op: NewDst);
2274 break;
2275 }
2276 case Sgpr32ToVgprDst:
2277 case Sgpr64ToVgprDst: {
2278 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2279 assert(RB == VgprRB);
2280 Op.setReg(MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty}));
2281 B.buildCopy(Res: Reg, Op: Op.getReg());
2282 break;
2283 }
2284 case InvalidMapping: {
2285 reportGISelFailure(
2286 MF, MORE, DEBUG_TYPE,
2287 Msg: "AMDGPU RegBankLegalize: missing fast rule ('Div' or 'Uni') for", MI);
2288 return false;
2289 }
2290 default:
2291 reportGISelFailure(
2292 MF, MORE, DEBUG_TYPE,
2293 Msg: "AMDGPU RegBankLegalize: applyMappingDst, ID not supported", MI);
2294 return false;
2295 }
2296 }
2297
2298 return true;
2299}
2300
2301bool RegBankLegalizeHelper::applyMappingSrc(
2302 MachineInstr &MI, unsigned &OpIdx,
2303 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
2304 WaterfallInfo &WFI) {
2305 for (unsigned i = 0; i < MethodIDs.size(); ++OpIdx, ++i) {
2306 if (MethodIDs[i] == None || MethodIDs[i] == IntrId || MethodIDs[i] == Imm)
2307 continue;
2308
2309 MachineOperand &Op = MI.getOperand(i: OpIdx);
2310 Register Reg = Op.getReg();
2311 LLT Ty = MRI.getType(Reg);
2312 const RegisterBank *RB = MRI.getRegBank(Reg);
2313
2314 switch (MethodIDs[i]) {
2315 case Vcc: {
2316 assert(Ty == S1);
2317 assert(RB == VccRB || RB == SgprRB);
2318 if (RB == SgprRB) {
2319 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2320 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2321 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2322 auto CopyVcc_Scc = B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_VCC_SCC,
2323 DstOps: {VccRB_S1}, SrcOps: {BoolInReg});
2324 Op.setReg(CopyVcc_Scc.getReg(Idx: 0));
2325 }
2326 break;
2327 }
2328 // sgpr scalars, pointers and vectors
2329 case Sgpr16:
2330 case Sgpr32:
2331 case Sgpr64:
2332 case Sgpr128:
2333 case SgprP0:
2334 case SgprP1:
2335 case SgprP3:
2336 case SgprP4:
2337 case SgprP5:
2338 case SgprP6:
2339 case SgprP8:
2340 case SgprV2S16:
2341 case SgprV2S32:
2342 case SgprV4S32: {
2343 assert(Ty == getTyFromID(MethodIDs[i]));
2344 assert(RB == getRegBankFromID(MethodIDs[i]));
2345 break;
2346 }
2347 // sgpr B-types
2348 case SgprB32:
2349 case SgprB64:
2350 case SgprB96:
2351 case SgprB128:
2352 case SgprB256:
2353 case SgprB512:
2354 case SgprBRC:
2355 case SgprPtr32:
2356 case SgprPtr64:
2357 case SgprPtr128: {
2358 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2359 assert(RB == getRegBankFromID(MethodIDs[i]));
2360 break;
2361 }
2362 // vgpr scalars, pointers and vectors
2363 case Vgpr16:
2364 case Vgpr32:
2365 case Vgpr64:
2366 case Vgpr128:
2367 case VgprP0:
2368 case VgprP1:
2369 case VgprP2:
2370 case VgprP3:
2371 case VgprP4:
2372 case VgprP5:
2373 case VgprV2S16:
2374 case VgprV2S32:
2375 case VgprV2S64:
2376 case VgprV3S32:
2377 case VgprV4S16:
2378 case VgprV8S16:
2379 case VgprV16S16:
2380 case VgprV4S32:
2381 case VgprV6S32:
2382 case VgprV8S32:
2383 case VgprV16S32:
2384 case VgprV32S16:
2385 case VgprV32S32: {
2386 assert(Ty == getTyFromID(MethodIDs[i]));
2387 if (RB != VgprRB) {
2388 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2389 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2390 }
2391 break;
2392 }
2393 // vgpr B-types
2394 case VgprB32:
2395 case VgprB64:
2396 case VgprB96:
2397 case VgprB128:
2398 case VgprB160:
2399 case VgprB256:
2400 case VgprB512:
2401 case VgprBRC:
2402 case VgprPtr32:
2403 case VgprPtr64:
2404 case VgprPtr128: {
2405 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2406 if (RB != VgprRB) {
2407 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2408 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2409 }
2410 break;
2411 }
2412 case VgprAnyTy: {
2413 if (RB != VgprRB) {
2414 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2415 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2416 }
2417 break;
2418 }
2419 case AgprAnyTy: {
2420 if (RB != AgprRB) {
2421 auto CopyToAgpr = B.buildCopy(Res: {AgprRB, Ty}, Op: Reg);
2422 Op.setReg(CopyToAgpr.getReg(Idx: 0));
2423 }
2424 break;
2425 }
2426 case VgprOrAgprAnyTy: {
2427 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2428 const RegisterBank *SrcRB =
2429 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2430 if (RB != SrcRB)
2431 Op.setReg(B.buildCopy(Res: {SrcRB, Ty}, Op: Reg).getReg(Idx: 0));
2432 break;
2433 }
2434 // sgpr waterfall, scalars, and vectors
2435 case Sgpr32_WF:
2436 case SgprV4S32_WF: {
2437 assert(Ty == getTyFromID(MethodIDs[i]));
2438 if (RB != SgprRB) {
2439 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2440 if (!WFI.Start.isValid()) {
2441 WFI.Start = MI.getIterator();
2442 WFI.End = std::next(x: MI.getIterator());
2443 }
2444 }
2445 break;
2446 }
2447 case SgprP0Call_WF:
2448 case SgprP4Call_WF: {
2449 assert(Ty == getTyFromID(MethodIDs[i]));
2450 if (RB != SgprRB) {
2451 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2452
2453 // Find the ADJCALLSTACKUP before the call.
2454 MachineBasicBlock::iterator Start = MI.getIterator();
2455 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
2456 --Start;
2457
2458 // Find the ADJCALLSTACKDOWN after the call (include it in range).
2459 MachineBasicBlock::iterator End = MI.getIterator();
2460 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
2461 ++End;
2462 ++End;
2463
2464 WFI.Start = Start;
2465 WFI.End = End;
2466 }
2467 break;
2468 }
2469 case SgprB32_M0:
2470 case SgprB32_ReadFirstLane:
2471 case SgprB64_ReadFirstLane: {
2472 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2473 if (RB == SgprRB)
2474 break;
2475 assert(RB == VgprRB);
2476 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2477 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2478 Op.setReg(NewSGPR);
2479 break;
2480 }
2481 case SgprV4S32_ReadFirstLane:
2482 case SgprV8S32_ReadFirstLane: {
2483 assert(Ty == getTyFromID(MethodIDs[i]));
2484 if (RB == SgprRB)
2485 break;
2486 assert(RB == VgprRB);
2487 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2488 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2489 Op.setReg(NewSGPR);
2490 break;
2491 }
2492 // sgpr and vgpr scalars with extend
2493 case Sgpr32AExt: {
2494 // Note: this ext allows S1, and it is meant to be combined away.
2495 assert(Ty.getSizeInBits() < 32);
2496 assert(RB == SgprRB);
2497 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2498 Op.setReg(Aext.getReg(Idx: 0));
2499 break;
2500 }
2501 case Sgpr32AExtBoolInReg: {
2502 // Note: this ext allows S1, and it is meant to be combined away.
2503 assert(Ty.getSizeInBits() == 1);
2504 assert(RB == SgprRB);
2505 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2506 // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
2507 // most of times meant to be combined away in AMDGPURegBankCombiner.
2508 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2509 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2510 Op.setReg(BoolInReg.getReg(Idx: 0));
2511 break;
2512 }
2513 case Sgpr32SExt: {
2514 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2515 assert(RB == SgprRB);
2516 auto Sext = B.buildSExt(Res: SgprRB_I32, Op: Reg);
2517 Op.setReg(Sext.getReg(Idx: 0));
2518 break;
2519 }
2520 case Sgpr32ZExt: {
2521 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2522 assert(RB == SgprRB);
2523 auto Zext = B.buildZExt(Res: SgprRB_I32, Op: Reg);
2524 Op.setReg(Zext.getReg(Idx: 0));
2525 break;
2526 }
2527 case Vgpr32AExt: {
2528 assert(Ty.getSizeInBits() < 32);
2529 assert(RB == VgprRB);
2530 auto Aext = B.buildAnyExt(Res: VgprRB_I32, Op: Reg);
2531 Op.setReg(Aext.getReg(Idx: 0));
2532 break;
2533 }
2534 case Vgpr32SExt: {
2535 // Note this ext allows S1, and it is meant to be combined away.
2536 assert(Ty.getSizeInBits() < 32);
2537 assert(RB == VgprRB);
2538 auto Sext = B.buildSExt(Res: VgprRB_I32, Op: Reg);
2539 Op.setReg(Sext.getReg(Idx: 0));
2540 break;
2541 }
2542 case Vgpr32ZExt: {
2543 // Note this ext allows S1, and it is meant to be combined away.
2544 assert(Ty.getSizeInBits() < 32);
2545 assert(RB == VgprRB);
2546 auto Zext = B.buildZExt(Res: VgprRB_I32, Op: Reg);
2547 Op.setReg(Zext.getReg(Idx: 0));
2548 break;
2549 }
2550 default:
2551 reportGISelFailure(
2552 MF, MORE, DEBUG_TYPE,
2553 Msg: "AMDGPU RegBankLegalize: applyMappingSrc, ID not supported", MI);
2554 return false;
2555 }
2556 }
2557 return true;
2558}
2559
2560[[maybe_unused]] static bool verifyRegBankOnOperands(MachineInstr &MI,
2561 const RegisterBank *RB,
2562 MachineRegisterInfo &MRI,
2563 unsigned StartOpIdx,
2564 unsigned EndOpIdx) {
2565 for (unsigned i = StartOpIdx; i <= EndOpIdx; ++i) {
2566 if (MRI.getRegBankOrNull(Reg: MI.getOperand(i).getReg()) != RB)
2567 return false;
2568 }
2569 return true;
2570}
2571
2572bool RegBankLegalizeHelper::applyRegisterBanksVgprWithSgprRsrc(
2573 MachineInstr &MI, unsigned RsrcIdx) {
2574 const unsigned NumDefs = MI.getNumExplicitDefs();
2575
2576 MachineBasicBlock *MBB = MI.getParent();
2577 B.setInsertPt(MBB&: *MBB, II: MBB->SkipPHIsAndLabels(I: std::next(x: MI.getIterator())));
2578
2579 // Defs are vgpr.
2580 for (unsigned i = 0; i < NumDefs; ++i) {
2581 Register Reg = MI.getOperand(i).getReg();
2582 if (MRI.getRegBank(Reg) == VgprRB)
2583 continue;
2584
2585 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: MRI.getType(Reg)});
2586 MI.getOperand(i).setReg(NewVgprDst);
2587 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2588 }
2589
2590 B.setInstrAndDebugLoc(MI);
2591
2592 // Register uses before RsrcIdx are vgpr.
2593 for (unsigned i = NumDefs; i < RsrcIdx; ++i) {
2594 MachineOperand &Op = MI.getOperand(i);
2595 if (!Op.isReg())
2596 continue;
2597
2598 Register Reg = Op.getReg();
2599 if (!Reg.isVirtual())
2600 continue;
2601
2602 if (MRI.getRegBank(Reg) == VgprRB)
2603 continue;
2604
2605 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
2606 Op.setReg(Copy.getReg(Idx: 0));
2607 }
2608
2609 SmallSet<Register, 4> OpsToWaterfall;
2610
2611 // Register use RsrcIdx (and later register operands) is sgpr.
2612 for (unsigned i = RsrcIdx; i < MI.getNumOperands(); ++i) {
2613 MachineOperand &Op = MI.getOperand(i);
2614 if (!Op.isReg())
2615 continue;
2616
2617 Register Reg = Op.getReg();
2618 if (MRI.getRegBank(Reg) != SgprRB)
2619 OpsToWaterfall.insert(V: Reg);
2620 }
2621
2622 if (!OpsToWaterfall.empty()) {
2623 MachineBasicBlock::iterator MII = MI.getIterator();
2624 executeInWaterfallLoop(B, WFI: {.SgprWaterfallOperandRegs: OpsToWaterfall, .Start: MII, .End: std::next(x: MII)});
2625 }
2626
2627 return true;
2628}
2629