1//===-- AMDGPURegBankLegalizeHelper.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Implements actual lowering algorithms for each ID that can be used in
10/// Rule.OperandMapping. Similar to legalizer helper but with register banks.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPURegBankLegalizeHelper.h"
15#include "AMDGPUGlobalISelUtils.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPURegBankLegalizeRules.h"
19#include "AMDGPURegisterBankInfo.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineInstr.h"
27#include "llvm/CodeGen/MachineUniformityAnalysis.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29
30#define DEBUG_TYPE "amdgpu-reg-bank-legalize"
31
32using namespace llvm;
33using namespace AMDGPU;
34
35RegBankLegalizeHelper::RegBankLegalizeHelper(
36 MachineIRBuilder &B, const MachineUniformityInfo &MUI,
37 GISelValueTracking *VT, const RegisterBankInfo &RBI,
38 const RegBankLegalizeRules &RBLRules)
39 : MF(B.getMF()), MFI(MF.getInfo<SIMachineFunctionInfo>()),
40 ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()), B(B),
41 MRI(*B.getMRI()), MUI(MUI), VT(VT), RBI(RBI), MORE(MF, nullptr),
42 RBLRules(RBLRules), IsWave32(ST.isWave32()),
43 SgprRB(&RBI.getRegBank(ID: AMDGPU::SGPRRegBankID)),
44 VgprRB(&RBI.getRegBank(ID: AMDGPU::VGPRRegBankID)),
45 AgprRB(&RBI.getRegBank(ID: AMDGPU::AGPRRegBankID)),
46 VccRB(&RBI.getRegBank(ID: AMDGPU::VCCRegBankID)) {}
47
48bool RegBankLegalizeHelper::findRuleAndApplyMapping(MachineInstr &MI) {
49 const SetOfRulesForOpcode *RuleSet = RBLRules.getRulesForOpc(MI);
50 if (!RuleSet) {
51 reportGISelFailure(MF, MORE, DEBUG_TYPE,
52 Msg: "No AMDGPU RegBankLegalize rules defined for opcode",
53 MI);
54 return false;
55 }
56
57 const RegBankLLTMapping *Mapping = RuleSet->findMappingForMI(MI, MRI, MUI);
58 if (!Mapping) {
59 reportGISelFailure(MF, MORE, DEBUG_TYPE,
60 Msg: "AMDGPU RegBankLegalize: none of the rules defined with "
61 "'Any' for MI's opcode matched MI",
62 MI);
63 return false;
64 }
65
66 WaterfallInfo WFI;
67 unsigned OpIdx = 0;
68 if (!Mapping->DstOpMapping.empty()) {
69 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
70 if (!applyMappingDst(MI, OpIdx, MethodIDs: Mapping->DstOpMapping))
71 return false;
72 }
73 if (!Mapping->SrcOpMapping.empty()) {
74 B.setInstr(MI);
75 if (!applyMappingSrc(MI, OpIdx, MethodIDs: Mapping->SrcOpMapping, WFI))
76 return false;
77 }
78
79 if (!lower(MI, Mapping: *Mapping, WFI))
80 return false;
81
82 if (!WFI.SgprWaterfallOperandRegs.empty()) {
83 if (!executeInWaterfallLoop(B, WFI))
84 return false;
85 }
86
87 return true;
88}
89
90bool RegBankLegalizeHelper::executeInWaterfallLoop(MachineIRBuilder &B,
91 const WaterfallInfo &WFI) {
92 assert(WFI.Start.isValid() && WFI.End.isValid() &&
93 "Waterfall range not initialized");
94
95 // Track use registers which have already been expanded with a readfirstlane
96 // sequence. This may have multiple uses if moving a sequence.
97 DenseMap<Register, Register> WaterfalledRegMap;
98
99 MachineBasicBlock &MBB = B.getMBB();
100 MachineFunction &MF = B.getMF();
101
102 MachineBasicBlock::iterator BeginIt = WFI.Start;
103 MachineBasicBlock::iterator EndIt = WFI.End;
104
105 const SIRegisterInfo *TRI = ST.getRegisterInfo();
106 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass();
107 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
108
109#ifndef NDEBUG
110 const int OrigRangeSize = std::distance(BeginIt, EndIt);
111#endif
112
113 MachineRegisterInfo &MRI = *B.getMRI();
114 Register SaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
115 Register InitSaveExecReg = MRI.createVirtualRegister(RegClass: WaveRC);
116
117 // Don't bother using generic instructions/registers for the exec mask.
118 B.setInstr(*WFI.Start);
119 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: InitSaveExecReg);
120
121 Register SavedExec = MRI.createVirtualRegister(RegClass: WaveRC);
122
123 // To insert the loop we need to split the block. Move everything before
124 // this point to a new block, and insert a new empty block before this
125 // instruction.
126 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
127 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
128 MachineBasicBlock *RestoreExecBB = MF.CreateMachineBasicBlock();
129 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
130 MachineFunction::iterator MBBI(MBB);
131 ++MBBI;
132 MF.insert(MBBI, MBB: LoopBB);
133 MF.insert(MBBI, MBB: BodyBB);
134 MF.insert(MBBI, MBB: RestoreExecBB);
135 MF.insert(MBBI, MBB: RemainderBB);
136
137 LoopBB->addSuccessor(Succ: BodyBB);
138 BodyBB->addSuccessor(Succ: RestoreExecBB);
139 BodyBB->addSuccessor(Succ: LoopBB);
140
141 // Move the rest of the block into a new block.
142 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
143 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: EndIt, To: MBB.end());
144
145 MBB.addSuccessor(Succ: LoopBB);
146 RestoreExecBB->addSuccessor(Succ: RemainderBB);
147
148 B.setInsertPt(MBB&: *LoopBB, II: LoopBB->end());
149
150 // +-MBB:------------+
151 // | ... |
152 // | %0 = G_INST_1 |
153 // | %Dst = MI %Vgpr |
154 // | %1 = G_INST_2 |
155 // | ... |
156 // +-----------------+
157 // ->
158 // +-MBB-------------------------------+
159 // | ... |
160 // | %0 = G_INST_1 |
161 // | %SaveExecReg = S_MOV_B32 $exec_lo |
162 // +----------------|------------------+
163 // | /------------------------------|
164 // V V |
165 // +-LoopBB---------------------------------------------------------------+ |
166 // | %CurrentLaneReg:sgpr(s32) = READFIRSTLANE %Vgpr | |
167 // | instead of executing for each lane, see if other lanes had | |
168 // | same value for %Vgpr and execute for them also. | |
169 // | %CondReg:vcc(s1) = G_ICMP eq %CurrentLaneReg, %Vgpr | |
170 // | %CondRegLM:sreg_32 = ballot %CondReg // copy vcc to sreg32 lane mask | |
171 // | %SavedExec = S_AND_SAVEEXEC_B32 %CondRegLM | |
172 // | exec is active for lanes with the same "CurrentLane value" in Vgpr | |
173 // +----------------|-----------------------------------------------------+ |
174 // V |
175 // +-BodyBB------------------------------------------------------------+ |
176 // | %Dst = MI %CurrentLaneReg:sgpr(s32) | |
177 // | executed only for active lanes and written to Dst | |
178 // | $exec = S_XOR_B32 $exec, %SavedExec | |
179 // | set active lanes to 0 in SavedExec, lanes that did not write to | |
180 // | Dst yet, and set this as new exec (for READFIRSTLANE and ICMP) | |
181 // | SI_WATERFALL_LOOP LoopBB |-----|
182 // +----------------|--------------------------------------------------+
183 // V
184 // +-RestoreExecBB--------------------------+
185 // | $exec_lo = S_MOV_B32_term %SaveExecReg |
186 // +----------------|-----------------------+
187 // V
188 // +-RemainderBB:----------------------+
189 // | %1 = G_INST_2 |
190 // | ... |
191 // +---------------------------------- +
192
193 // Move the instruction into the loop body. Note we moved everything after
194 // Range.end() already into a new block, so Range.end() is no longer valid.
195 BodyBB->splice(Where: BodyBB->end(), Other: &MBB, From: BeginIt, To: MBB.end());
196
197 // Figure out the iterator range after splicing the instructions.
198 MachineBasicBlock::iterator NewBegin = BeginIt;
199 auto NewEnd = BodyBB->end();
200 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize);
201
202 B.setMBB(*LoopBB);
203 Register CondReg;
204
205 for (MachineInstr &MI : make_range(x: NewBegin, y: NewEnd)) {
206 for (MachineOperand &Op : MI.all_uses()) {
207 Register OldReg = Op.getReg();
208 if (!WFI.SgprWaterfallOperandRegs.count(V: OldReg))
209 continue;
210
211 // See if we already processed this register in another instruction in
212 // the sequence.
213 auto OldVal = WaterfalledRegMap.find(Val: OldReg);
214 if (OldVal != WaterfalledRegMap.end()) {
215 Op.setReg(OldVal->second);
216 continue;
217 }
218
219 Register OpReg = Op.getReg();
220 LLT OpTy = MRI.getType(Reg: OpReg);
221
222 // TODO: support for agpr
223 assert(MRI.getRegBank(OpReg) == VgprRB);
224 Register CurrentLaneReg = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: OpTy});
225 buildReadFirstLane(B, SgprDst: CurrentLaneReg, VgprSrc: OpReg, RBI);
226
227 // Build the comparison(s), CurrentLaneReg == OpReg.
228 unsigned OpSize = OpTy.getSizeInBits();
229 unsigned PartSize = (OpSize % 64 == 0) ? 64 : 32;
230 LLT PartTy = LLT::integer(SizeInBits: PartSize);
231 unsigned NumParts = OpSize / PartSize;
232 SmallVector<Register, 8> OpParts;
233 SmallVector<Register, 8> CurrentLaneParts;
234
235 if (NumParts == 1) {
236 OpParts.push_back(Elt: OpReg);
237 CurrentLaneParts.push_back(Elt: CurrentLaneReg);
238 } else {
239 auto UnmergeOp = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: PartTy}, Op: OpReg);
240 auto UnmergeCurrLane = B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: PartTy}, Op: CurrentLaneReg);
241 for (unsigned i = 0; i < NumParts; ++i) {
242 OpParts.push_back(Elt: UnmergeOp.getReg(Idx: i));
243 CurrentLaneParts.push_back(Elt: UnmergeCurrLane.getReg(Idx: i));
244 }
245 }
246
247 for (unsigned i = 0; i < NumParts; ++i) {
248 Register CmpReg = MRI.createVirtualRegister(RegAttr: VccRB_S1);
249 B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpReg, Op0: CurrentLaneParts[i], Op1: OpParts[i]);
250
251 if (!CondReg)
252 CondReg = CmpReg;
253 else
254 CondReg = B.buildAnd(Dst: VccRB_S1, Src0: CondReg, Src1: CmpReg).getReg(Idx: 0);
255 }
256
257 Op.setReg(CurrentLaneReg);
258
259 // Make sure we don't re-process this register again.
260 WaterfalledRegMap.insert(KV: std::pair(OldReg, Op.getReg()));
261 }
262 }
263
264 // Copy vcc to sgpr32/64, ballot becomes a no-op during instruction selection.
265 Register CondRegLM =
266 MRI.createVirtualRegister(RegAttr: {.RCOrRB: WaveRC, .Ty: LLT::integer(SizeInBits: IsWave32 ? 32 : 64)});
267 B.buildIntrinsic(ID: Intrinsic::amdgcn_ballot, Res: CondRegLM).addReg(RegNo: CondReg);
268
269 // Update EXEC, save the original EXEC value to SavedExec.
270 B.buildInstr(Opcode: LMC.AndSaveExecOpc)
271 .addDef(RegNo: SavedExec)
272 .addReg(RegNo: CondRegLM, Flags: RegState::Kill);
273 MRI.setSimpleHint(VReg: SavedExec, PrefReg: CondRegLM);
274
275 B.setInsertPt(MBB&: *BodyBB, II: BodyBB->end());
276
277 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
278 B.buildInstr(Opcode: LMC.XorTermOpc)
279 .addDef(RegNo: LMC.ExecReg)
280 .addReg(RegNo: LMC.ExecReg)
281 .addReg(RegNo: SavedExec);
282
283 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
284 // s_cbranch_scc0?
285
286 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
287 B.buildInstr(Opcode: AMDGPU::SI_WATERFALL_LOOP).addMBB(MBB: LoopBB);
288
289 // Save the EXEC mask before the loop.
290 B.setInsertPt(MBB, II: MBB.end());
291 B.buildInstr(Opcode: LMC.MovOpc).addDef(RegNo: SaveExecReg).addReg(RegNo: LMC.ExecReg);
292
293 // Restore the EXEC mask after the loop.
294 B.setInsertPt(MBB&: *RestoreExecBB, II: RestoreExecBB->begin());
295 B.buildInstr(Opcode: LMC.MovTermOpc).addDef(RegNo: LMC.ExecReg).addReg(RegNo: SaveExecReg);
296
297 // Set the insert point after the original instruction, so any new
298 // instructions will be in the remainder.
299 B.setInsertPt(MBB&: *RemainderBB, II: RemainderBB->begin());
300
301 return true;
302}
303
304// Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store
305// the three offsets (voffset, soffset and instoffset)
306unsigned RegBankLegalizeHelper::setBufferOffsets(
307 MachineIRBuilder &B, Register CombinedOffset, Register &VOffsetReg,
308 Register &SOffsetReg, int64_t &InstOffsetVal, Align Alignment) {
309 if (std::optional<int64_t> Imm =
310 getIConstantVRegSExtVal(VReg: CombinedOffset, MRI)) {
311 uint32_t SOffset, ImmOffset;
312 if (TII.splitMUBUFOffset(Imm: *Imm, SOffset, ImmOffset, Alignment)) {
313 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
314 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
315 InstOffsetVal = ImmOffset;
316 return SOffset + ImmOffset;
317 }
318 }
319 const bool CheckNUW = ST.hasGFX1250Insts();
320 auto [Base, Offset] = AMDGPU::getBaseWithConstantOffset(
321 MRI, Reg: CombinedOffset, /*KnownBits=*/ValueTracking: nullptr,
322 /*CheckNUW=*/CheckNUW);
323 uint32_t SOffset, ImmOffset;
324 if (static_cast<int32_t>(Offset) > 0 &&
325 TII.splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
326 if (Base.isValid() && MRI.getRegBank(Reg: Base) == VgprRB) {
327 VOffsetReg = Base;
328 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: SOffset).getReg(Idx: 0);
329 InstOffsetVal = ImmOffset;
330 return 0;
331 }
332 // If we have SGPR base, we can use it for soffset.
333 if (SOffset == 0) {
334 VOffsetReg = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
335 SOffsetReg = Base;
336 InstOffsetVal = ImmOffset;
337 return 0;
338 }
339 }
340 // Handle the variable sgpr + vgpr case.
341 MachineInstr *Add = getOpcodeDef(Opcode: AMDGPU::G_ADD, Reg: CombinedOffset, MRI);
342 if (Add && static_cast<int32_t>(Offset) >= 0 &&
343 (!CheckNUW || Add->getFlag(Flag: MachineInstr::NoUWrap))) {
344 Register Src0 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 1).getReg(), MRI);
345 Register Src1 = getSrcRegIgnoringCopies(Reg: Add->getOperand(i: 2).getReg(), MRI);
346 const RegisterBank *Src0Bank = MRI.getRegBank(Reg: Src0);
347 const RegisterBank *Src1Bank = MRI.getRegBank(Reg: Src1);
348 if (Src0Bank == VgprRB && Src1Bank == SgprRB) {
349 VOffsetReg = Src0;
350 SOffsetReg = Src1;
351 return 0;
352 }
353 if (Src0Bank == SgprRB && Src1Bank == VgprRB) {
354 VOffsetReg = Src1;
355 SOffsetReg = Src0;
356 return 0;
357 }
358 }
359 // Ensure we have a VGPR for the combined offset. This could be an issue if we
360 // have an SGPR offset and a VGPR resource.
361 if (MRI.getRegBank(Reg: CombinedOffset) == VgprRB) {
362 VOffsetReg = CombinedOffset;
363 } else {
364 VOffsetReg = B.buildCopy(Res: VgprRB_I32, Op: CombinedOffset).getReg(Idx: 0);
365 }
366 SOffsetReg = B.buildConstant(Res: SgprRB_I32, Val: 0).getReg(Idx: 0);
367 return 0;
368}
369
370bool RegBankLegalizeHelper::splitLoad(MachineInstr &MI,
371 ArrayRef<LLT> LLTBreakdown, LLT MergeTy) {
372 MachineFunction &MF = B.getMF();
373 assert(MI.getNumMemOperands() == 1);
374 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
375 Register Dst = MI.getOperand(i: 0).getReg();
376 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
377 Register Base = MI.getOperand(i: 1).getReg();
378 LLT PtrTy = MRI.getType(Reg: Base);
379 const RegisterBank *PtrRB = MRI.getRegBankOrNull(Reg: Base);
380 LLT OffsetTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
381 SmallVector<Register, 4> LoadPartRegs;
382
383 unsigned ByteOffset = 0;
384 for (LLT PartTy : LLTBreakdown) {
385 Register BasePlusOffset;
386 if (ByteOffset == 0) {
387 BasePlusOffset = Base;
388 } else {
389 auto Offset = B.buildConstant(Res: {PtrRB, OffsetTy}, Val: ByteOffset);
390 BasePlusOffset =
391 B.buildObjectPtrOffset(Res: {PtrRB, PtrTy}, Op0: Base, Op1: Offset).getReg(Idx: 0);
392 }
393 auto *OffsetMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: ByteOffset, Ty: PartTy);
394 auto LoadPart = B.buildLoad(Res: {DstRB, PartTy}, Addr: BasePlusOffset, MMO&: *OffsetMMO);
395 LoadPartRegs.push_back(Elt: LoadPart.getReg(Idx: 0));
396 ByteOffset += PartTy.getSizeInBytes();
397 }
398
399 if (!MergeTy.isValid()) {
400 // Loads are of same size, concat or merge them together.
401 B.buildMergeLikeInstr(Res: Dst, Ops: LoadPartRegs);
402 } else {
403 // Loads are not all of same size, need to unmerge them to smaller pieces
404 // of MergeTy type, then merge pieces to Dst.
405 SmallVector<Register, 4> MergeTyParts;
406 for (Register Reg : LoadPartRegs) {
407 if (MRI.getType(Reg) == MergeTy) {
408 MergeTyParts.push_back(Elt: Reg);
409 } else {
410 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: Reg);
411 for (unsigned i = 0; i < Unmerge->getNumOperands() - 1; ++i)
412 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
413 }
414 }
415 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
416 }
417 MI.eraseFromParent();
418 return true;
419}
420
421bool RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
422 LLT MergeTy) {
423 MachineFunction &MF = B.getMF();
424 assert(MI.getNumMemOperands() == 1);
425 MachineMemOperand &BaseMMO = **MI.memoperands_begin();
426 Register Dst = MI.getOperand(i: 0).getReg();
427 const RegisterBank *DstRB = MRI.getRegBankOrNull(Reg: Dst);
428 Register Base = MI.getOperand(i: 1).getReg();
429
430 MachineMemOperand *WideMMO = MF.getMachineMemOperand(MMO: &BaseMMO, Offset: 0, Ty: WideTy);
431 auto WideLoad = B.buildLoad(Res: {DstRB, WideTy}, Addr: Base, MMO&: *WideMMO);
432
433 if (WideTy.isScalar()) {
434 B.buildTrunc(Res: Dst, Op: WideLoad);
435 } else {
436 SmallVector<Register, 4> MergeTyParts;
437 auto Unmerge = B.buildUnmerge(Attrs: {.RCOrRB: DstRB, .Ty: MergeTy}, Op: WideLoad);
438
439 LLT DstTy = MRI.getType(Reg: Dst);
440 unsigned NumElts = DstTy.getSizeInBits() / MergeTy.getSizeInBits();
441 for (unsigned i = 0; i < NumElts; ++i) {
442 MergeTyParts.push_back(Elt: Unmerge.getReg(Idx: i));
443 }
444 B.buildMergeLikeInstr(Res: Dst, Ops: MergeTyParts);
445 }
446 MI.eraseFromParent();
447 return true;
448}
449
450bool RegBankLegalizeHelper::widenMMOToS32(GAnyLoad &MI) const {
451 Register Dst = MI.getDstReg();
452 Register Ptr = MI.getPointerReg();
453 MachineMemOperand &MMO = MI.getMMO();
454 unsigned MemSize = 8 * MMO.getSize().getValue();
455
456 MachineMemOperand *WideMMO = B.getMF().getMachineMemOperand(MMO: &MMO, Offset: 0, Ty: S32);
457
458 if (MI.getOpcode() == G_LOAD) {
459 B.buildLoad(Res: Dst, Addr: Ptr, MMO&: *WideMMO);
460 } else {
461 auto Load = B.buildLoad(Res: SgprRB_I32, Addr: Ptr, MMO&: *WideMMO);
462
463 if (MI.getOpcode() == G_ZEXTLOAD) {
464 APInt Mask = APInt::getLowBitsSet(numBits: S32.getSizeInBits(), loBitsSet: MemSize);
465 auto MaskCst = B.buildConstant(Res: SgprRB_I32, Val: Mask);
466 B.buildAnd(Dst, Src0: Load, Src1: MaskCst);
467 } else {
468 assert(MI.getOpcode() == G_SEXTLOAD);
469 B.buildSExtInReg(Res: Dst, Op: Load, ImmOp: MemSize);
470 }
471 }
472
473 MI.eraseFromParent();
474 return true;
475}
476
477bool RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
478 Register Dst = MI.getOperand(i: 0).getReg();
479 LLT Ty = MRI.getType(Reg: Dst);
480 Register Src = MI.getOperand(i: 1).getReg();
481 unsigned Opc = MI.getOpcode();
482 int TrueExtCst = Opc == G_SEXT ? -1 : 1;
483 if (Ty == S32 || Ty == S16) {
484 auto True = B.buildConstant(Res: {VgprRB, Ty}, Val: TrueExtCst);
485 auto False = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
486 B.buildSelect(Res: Dst, Tst: Src, Op0: True, Op1: False);
487 } else if (Ty == S64) {
488 auto True = B.buildConstant(Res: {VgprRB_I32}, Val: TrueExtCst);
489 auto False = B.buildConstant(Res: {VgprRB_I32}, Val: 0);
490 auto Lo = B.buildSelect(Res: {VgprRB_I32}, Tst: Src, Op0: True, Op1: False);
491 MachineInstrBuilder Hi;
492 switch (Opc) {
493 case G_SEXT:
494 Hi = Lo;
495 break;
496 case G_ZEXT:
497 Hi = False;
498 break;
499 case G_ANYEXT:
500 Hi = B.buildUndef(Res: {VgprRB_I32});
501 break;
502 default:
503 reportGISelFailure(
504 MF, MORE, DEBUG_TYPE,
505 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Opcode not supported", MI);
506 return false;
507 }
508
509 B.buildMergeValues(Res: Dst, Ops: {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)});
510 } else {
511 reportGISelFailure(
512 MF, MORE, DEBUG_TYPE,
513 Msg: "AMDGPU RegBankLegalize: lowerVccExtToSel, Type not supported", MI);
514 return false;
515 }
516
517 MI.eraseFromParent();
518 return true;
519}
520
521std::pair<Register, Register> RegBankLegalizeHelper::unpackZExt(Register Reg) {
522 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
523 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: 0x0000ffff);
524 auto Lo = B.buildAnd(Dst: SgprRB_I32, Src0: PackedI32, Src1: Mask);
525 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
526 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
527}
528
529std::pair<Register, Register> RegBankLegalizeHelper::unpackSExt(Register Reg) {
530 auto PackedI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg);
531 auto Lo = B.buildSExtInReg(Res: SgprRB_I32, Op: PackedI32, ImmOp: 16);
532 auto Hi = B.buildAShr(Dst: SgprRB_I32, Src0: PackedI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
533 return {Lo.getReg(Idx: 0), Hi.getReg(Idx: 0)};
534}
535
536std::pair<Register, Register> RegBankLegalizeHelper::unpackAExt(Register Reg) {
537 Register RegI32 = Reg;
538 if (MRI.getType(Reg) != I32)
539 RegI32 = B.buildBitcast(Dst: SgprRB_I32, Src: Reg).getReg(Idx: 0);
540
541 auto Hi = B.buildLShr(Dst: SgprRB_I32, Src0: RegI32, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
542 return {RegI32, Hi.getReg(Idx: 0)};
543}
544
545std::pair<Register, Register>
546RegBankLegalizeHelper::unpackAExtTruncS16(Register Reg) {
547 auto [Lo32, Hi32] = unpackAExt(Reg);
548 LLT EltTy = MRI.getType(Reg).getElementType();
549 return {B.buildTrunc(Res: {SgprRB, EltTy}, Op: Lo32).getReg(Idx: 0),
550 B.buildTrunc(Res: {SgprRB, EltTy}, Op: Hi32).getReg(Idx: 0)};
551}
552
553bool RegBankLegalizeHelper::lowerUnpackBitShift(MachineInstr &MI) {
554 Register Lo, Hi;
555 switch (MI.getOpcode()) {
556 case AMDGPU::G_SHL: {
557 auto [Val0, Val1] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
558 auto [Amt0, Amt1] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
559 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
560 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
561 break;
562 }
563 case AMDGPU::G_LSHR: {
564 auto [Val0, Val1] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
565 auto [Amt0, Amt1] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
566 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0, Amt0}).getReg(Idx: 0);
567 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val1, Amt1}).getReg(Idx: 0);
568 break;
569 }
570 case AMDGPU::G_ASHR: {
571 auto [Val0, Val1] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
572 auto [Amt0, Amt1] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
573 Lo = B.buildAShr(Dst: SgprRB_I32, Src0: Val0, Src1: Amt0).getReg(Idx: 0);
574 Hi = B.buildAShr(Dst: SgprRB_I32, Src0: Val1, Src1: Amt1).getReg(Idx: 0);
575 break;
576 }
577 default:
578 reportGISelFailure(
579 MF, MORE, DEBUG_TYPE,
580 Msg: "AMDGPU RegBankLegalize: lowerUnpackBitShift, case not implemented",
581 MI);
582 return false;
583 }
584 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
585 MI.eraseFromParent();
586 return true;
587}
588
589bool RegBankLegalizeHelper::lowerUnpackMinMax(MachineInstr &MI) {
590 Register Lo, Hi;
591 switch (MI.getOpcode()) {
592 case AMDGPU::G_SMIN:
593 case AMDGPU::G_SMAX: {
594 // For signed operations, use sign extension
595 auto [Val0_Lo, Val0_Hi] = unpackSExt(Reg: MI.getOperand(i: 1).getReg());
596 auto [Val1_Lo, Val1_Hi] = unpackSExt(Reg: MI.getOperand(i: 2).getReg());
597 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
598 .getReg(Idx: 0);
599 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
600 .getReg(Idx: 0);
601 break;
602 }
603 case AMDGPU::G_UMIN:
604 case AMDGPU::G_UMAX: {
605 // For unsigned operations, use zero extension
606 auto [Val0_Lo, Val0_Hi] = unpackZExt(Reg: MI.getOperand(i: 1).getReg());
607 auto [Val1_Lo, Val1_Hi] = unpackZExt(Reg: MI.getOperand(i: 2).getReg());
608 Lo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Lo, Val1_Lo})
609 .getReg(Idx: 0);
610 Hi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Val0_Hi, Val1_Hi})
611 .getReg(Idx: 0);
612 break;
613 }
614 default:
615 reportGISelFailure(
616 MF, MORE, DEBUG_TYPE,
617 Msg: "AMDGPU RegBankLegalize: lowerUnpackMinMax, case not implemented", MI);
618 return false;
619 }
620 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
621 MI.eraseFromParent();
622 return true;
623}
624
625bool RegBankLegalizeHelper::lowerUnpackAExt(MachineInstr &MI) {
626 auto [Op1Lo, Op1Hi] = unpackAExt(Reg: MI.getOperand(i: 1).getReg());
627 auto [Op2Lo, Op2Hi] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
628 auto ResLo = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Lo, Op2Lo});
629 auto ResHi = B.buildInstr(Opc: MI.getOpcode(), DstOps: {SgprRB_I32}, SrcOps: {Op1Hi, Op2Hi});
630 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
631 Ops: {ResLo.getReg(Idx: 0), ResHi.getReg(Idx: 0)});
632 MI.eraseFromParent();
633 return true;
634}
635
636bool RegBankLegalizeHelper::lowerSBufToBuf(MachineInstr &MI,
637 WaterfallInfo &WFI) {
638 Register Dst = MI.getOperand(i: 0).getReg();
639 LLT Ty = MRI.getType(Reg: Dst);
640 const RegisterBank *RSrcBank = MRI.getRegBank(Reg: MI.getOperand(i: 1).getReg());
641 unsigned LoadSize = Ty.getSizeInBits();
642 int NumLoads = 1;
643 SmallVector<Register, 4> LoadParts;
644 if (LoadSize == 256 || LoadSize == 512) {
645 NumLoads = LoadSize / 128;
646 Ty = Ty.divide(Factor: NumLoads);
647 }
648 for (int I = 0; I < NumLoads; ++I)
649 LoadParts.emplace_back(Args: MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty}));
650 MachineMemOperand *OrigMMO = *MI.memoperands_begin();
651 const Align Alignment = OrigMMO->getAlign();
652 MachineFunction &MF = B.getMF();
653 Register SOffset;
654 Register VOffset;
655 int64_t ImmOffset = 0;
656 unsigned MMOOffset = setBufferOffsets(B, CombinedOffset: MI.getOperand(i: 2).getReg(), VOffsetReg&: VOffset,
657 SOffsetReg&: SOffset, InstOffsetVal&: ImmOffset, Alignment);
658 const unsigned MemSize = divideCeil(Numerator: OrigMMO->getSize().getValue(), Denominator: NumLoads);
659 MachineMemOperand *BaseMMO = MF.getMachineMemOperand(MMO: OrigMMO, Offset: 0, Size: MemSize);
660 if (MMOOffset != 0)
661 BaseMMO = MF.getMachineMemOperand(MMO: BaseMMO, Offset: MMOOffset, Size: MemSize);
662 // If only the offset is divergent, emit a MUBUF buffer load
663 // instead. We can assume that the buffer is unswizzled.
664 Register RSrc = MI.getOperand(i: 1).getReg();
665 Register VIndex = B.buildConstant(Res: VgprRB_I32, Val: 0).getReg(Idx: 0);
666 unsigned CachePolicy = MI.getOperand(i: 3).getImm();
667 unsigned Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD;
668 switch (MI.getOpcode()) {
669 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SBYTE:
670 Opc = G_AMDGPU_BUFFER_LOAD_SBYTE;
671 break;
672 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE:
673 Opc = G_AMDGPU_BUFFER_LOAD_UBYTE;
674 break;
675 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_SSHORT:
676 Opc = G_AMDGPU_BUFFER_LOAD_SSHORT;
677 break;
678 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT:
679 Opc = G_AMDGPU_BUFFER_LOAD_USHORT;
680 break;
681 default:
682 break;
683 }
684 for (int I = 0; I < NumLoads; ++I) {
685 B.buildInstr(Opcode: Opc)
686 .addDef(RegNo: LoadParts[I]) // vdata
687 .addUse(RegNo: RSrc) // rsrc
688 .addUse(RegNo: VIndex) // vindex
689 .addUse(RegNo: VOffset) // voffset
690 .addUse(RegNo: SOffset) // soffset
691 .addImm(Val: ImmOffset + 16 * I) // offset(imm)
692 .addImm(Val: CachePolicy) // cachepolicy, swizzled buffer(imm)
693 .addImm(Val: 0) // idxen(imm)
694 .addMemOperand(MMO: MF.getMachineMemOperand(MMO: BaseMMO, Offset: 16 * I, Size: MemSize));
695 }
696 if (NumLoads == 1)
697 B.buildCopy(Res: Dst, Op: LoadParts[0]);
698 else
699 B.buildMergeLikeInstr(Res: Dst, Ops: LoadParts);
700 B.setInstr(*MRI.getVRegDef(Reg: LoadParts[0]));
701 if (RSrcBank != SgprRB) {
702 WFI.SgprWaterfallOperandRegs.insert(V: RSrc);
703 WFI.Start = MRI.getVRegDef(Reg: LoadParts.front());
704 WFI.End = std::next(x: MRI.getVRegDef(Reg: LoadParts.back())->getIterator());
705 }
706 MI.eraseFromParent();
707 return true;
708}
709
710static bool isSignedBFE(MachineInstr &MI) {
711 if (GIntrinsic *GI = dyn_cast<GIntrinsic>(Val: &MI))
712 return (GI->is(ID: Intrinsic::amdgcn_sbfe));
713
714 return MI.getOpcode() == AMDGPU::G_SBFX;
715}
716
717bool RegBankLegalizeHelper::lowerV_BFE(MachineInstr &MI) {
718 Register Dst = MI.getOperand(i: 0).getReg();
719 assert(MRI.getType(Dst) == LLT::scalar(64));
720 bool Signed = isSignedBFE(MI);
721 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
722 // Extract bitfield from Src, LSBit is the least-significant bit for the
723 // extraction (field offset) and Width is size of bitfield.
724 Register Src = MI.getOperand(i: FirstOpnd).getReg();
725 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
726 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
727 // Comments are for signed bitfield extract, similar for unsigned. x is sign
728 // bit. s is sign, l is LSB and y are remaining bits of bitfield to extract.
729
730 // Src >> LSBit Hi|Lo: x?????syyyyyyl??? -> xxxx?????syyyyyyl
731 unsigned SHROpc = Signed ? AMDGPU::G_ASHR : AMDGPU::G_LSHR;
732 auto SHRSrc = B.buildInstr(Opc: SHROpc, DstOps: {VgprRB_I64}, SrcOps: {Src, LSBit});
733
734 auto ConstWidth = getIConstantVRegValWithLookThrough(VReg: Width, MRI);
735
736 // Expand to Src >> LSBit << (64 - Width) >> (64 - Width)
737 // << (64 - Width): Hi|Lo: xxxx?????syyyyyyl -> syyyyyyl000000000
738 // >> (64 - Width): Hi|Lo: syyyyyyl000000000 -> ssssssssssyyyyyyl
739 if (!ConstWidth) {
740 auto Amt = B.buildSub(Dst: VgprRB_I32, Src0: B.buildConstant(Res: SgprRB_I32, Val: 64), Src1: Width);
741 auto SignBit = B.buildShl(Dst: VgprRB_I64, Src0: SHRSrc, Src1: Amt);
742 B.buildInstr(Opc: SHROpc, DstOps: {Dst}, SrcOps: {SignBit, Amt});
743 MI.eraseFromParent();
744 return true;
745 }
746
747 uint64_t WidthImm = ConstWidth->Value.getZExtValue();
748 auto UnmergeSHRSrc = B.buildUnmerge(Attrs: VgprRB_I32, Op: SHRSrc);
749 Register SHRSrcLo = UnmergeSHRSrc.getReg(Idx: 0);
750 Register SHRSrcHi = UnmergeSHRSrc.getReg(Idx: 1);
751 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
752 unsigned BFXOpc = Signed ? AMDGPU::G_SBFX : AMDGPU::G_UBFX;
753
754 if (WidthImm <= 32) {
755 // SHRSrc Hi|Lo: ????????|???syyyl -> ????????|ssssyyyl
756 Register Lo = SHRSrcLo;
757 // V_BFE masks its width to 5 bits, so 32 would extract zero bits.
758 if (WidthImm < 32) {
759 Lo =
760 B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcLo, Zero, Width}).getReg(Idx: 0);
761 }
762 MachineInstrBuilder Hi;
763 if (Signed) {
764 // SHRSrc Hi|Lo: ????????|ssssyyyl -> ssssssss|ssssyyyl
765 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: B.buildConstant(Res: VgprRB_I32, Val: 31));
766 } else {
767 // SHRSrc Hi|Lo: ????????|000syyyl -> 00000000|000syyyl
768 Hi = Zero;
769 }
770 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
771 } else {
772 auto Amt = B.buildConstant(Res: VgprRB_I32, Val: WidthImm - 32);
773 // SHRSrc Hi|Lo: ??????sy|yyyyyyyl -> sssssssy|yyyyyyyl
774 auto Hi = B.buildInstr(Opc: BFXOpc, DstOps: {VgprRB_I32}, SrcOps: {SHRSrcHi, Zero, Amt});
775 B.buildMergeLikeInstr(Res: Dst, Ops: {SHRSrcLo, Hi});
776 }
777
778 MI.eraseFromParent();
779 return true;
780}
781
782bool RegBankLegalizeHelper::lowerS_BFE(MachineInstr &MI) {
783 Register DstReg = MI.getOperand(i: 0).getReg();
784 LLT Ty = MRI.getType(Reg: DstReg);
785 bool Signed = isSignedBFE(MI);
786 unsigned FirstOpnd = isa<GIntrinsic>(Val: MI) ? 2 : 1;
787 Register Src = MI.getOperand(i: FirstOpnd).getReg();
788 Register LSBit = MI.getOperand(i: FirstOpnd + 1).getReg();
789 Register Width = MI.getOperand(i: FirstOpnd + 2).getReg();
790 // For uniform bit field extract there are 4 available instructions, but
791 // LSBit(field offset) and Width(size of bitfield) need to be packed in S32,
792 // field offset in low and size in high 16 bits.
793
794 // Src1 Hi16|Lo16 = Size|FieldOffset
795 auto Mask = B.buildConstant(Res: SgprRB_I32, Val: maskTrailingOnes<unsigned>(N: 6));
796 auto FieldOffset = B.buildAnd(Dst: SgprRB_I32, Src0: LSBit, Src1: Mask);
797 auto Size = B.buildShl(Dst: SgprRB_I32, Src0: Width, Src1: B.buildConstant(Res: SgprRB_I32, Val: 16));
798 auto Src1 = B.buildOr(Dst: SgprRB_I32, Src0: FieldOffset, Src1: Size);
799 unsigned Opc32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
800 unsigned Opc64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
801 unsigned Opc = Ty == S32 ? Opc32 : Opc64;
802
803 // Select machine instruction, because of reg class constraining, insert
804 // copies from reg class to reg bank.
805 auto S_BFE = B.buildInstr(Opc, DstOps: {{SgprRB, Ty}},
806 SrcOps: {B.buildCopy(Res: Ty, Op: Src), B.buildCopy(Res: I32, Op: Src1)});
807 constrainSelectedInstRegOperands(I&: *S_BFE, TII: *ST.getInstrInfo(),
808 TRI: *ST.getRegisterInfo(), RBI);
809
810 B.buildCopy(Res: DstReg, Op: S_BFE->getOperand(i: 0).getReg());
811 MI.eraseFromParent();
812 return true;
813}
814
815bool RegBankLegalizeHelper::lowerSplitTo32(MachineInstr &MI) {
816 Register Dst = MI.getOperand(i: 0).getReg();
817 LLT DstTy = MRI.getType(Reg: Dst);
818 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64);
819 LLT Ty = DstTy.divide(Factor: 2);
820 auto Op1 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 1).getReg());
821 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
822 unsigned Opc = MI.getOpcode();
823 auto Flags = MI.getFlags();
824 auto Lo =
825 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 0), Op2.getReg(Idx: 0)}, Flags);
826 auto Hi =
827 B.buildInstr(Opc, DstOps: {{VgprRB, Ty}}, SrcOps: {Op1.getReg(Idx: 1), Op2.getReg(Idx: 1)}, Flags);
828 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
829 MI.eraseFromParent();
830 return true;
831}
832
833bool RegBankLegalizeHelper::lowerSplitTo32Mul(MachineInstr &MI) {
834 Register Dst = MI.getOperand(i: 0).getReg();
835 assert(MRI.getType(Dst) == S64);
836 auto Op1 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 1).getReg());
837 auto Op2 = B.buildUnmerge(Attrs: {VgprRB_I32}, Op: MI.getOperand(i: 2).getReg());
838
839 // TODO: G_AMDGPU_MAD_* optimizations for G_MUL divergent S64 operation to
840 // match GlobalISel with old regbankselect.
841 auto Lo = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
842 auto Carry = B.buildUMulH(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 0));
843 auto MulLo0Hi1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 0), Src1: Op2.getReg(Idx: 1));
844 auto MulHi0Lo1 = B.buildMul(Dst: VgprRB_I32, Src0: Op1.getReg(Idx: 1), Src1: Op2.getReg(Idx: 0));
845 auto Sum = B.buildAdd(Dst: VgprRB_I32, Src0: MulLo0Hi1, Src1: MulHi0Lo1);
846 auto Hi = B.buildAdd(Dst: VgprRB_I32, Src0: Sum, Src1: Carry);
847
848 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
849 MI.eraseFromParent();
850 return true;
851}
852
853bool RegBankLegalizeHelper::lowerSplitTo16(MachineInstr &MI) {
854 Register Dst = MI.getOperand(i: 0).getReg();
855 assert(MRI.getType(Dst) == V2S16);
856 unsigned Opc = MI.getOpcode();
857 unsigned NumOps = MI.getNumOperands();
858 auto Flags = MI.getFlags();
859
860 auto [Op1Lo, Op1Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 1).getReg());
861 LLT EltTy = MRI.getType(Reg: Dst).getElementType();
862
863 if (NumOps == 2) {
864 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo}, Flags);
865 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi}, Flags);
866 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
867 MI.eraseFromParent();
868 return true;
869 }
870
871 auto [Op2Lo, Op2Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 2).getReg());
872
873 if (NumOps == 3) {
874 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo}, Flags);
875 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi}, Flags);
876 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
877 MI.eraseFromParent();
878 return true;
879 }
880
881 assert(NumOps == 4);
882 auto [Op3Lo, Op3Hi] = unpackAExtTruncS16(Reg: MI.getOperand(i: 3).getReg());
883 auto Lo = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Lo, Op2Lo, Op3Lo}, Flags);
884 auto Hi = B.buildInstr(Opc, DstOps: {{SgprRB, EltTy}}, SrcOps: {Op1Hi, Op2Hi, Op3Hi}, Flags);
885 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
886 MI.eraseFromParent();
887 return true;
888}
889
890bool RegBankLegalizeHelper::lowerUniMAD64(MachineInstr &MI) {
891 Register Dst0 = MI.getOperand(i: 0).getReg();
892 Register Dst1 = MI.getOperand(i: 1).getReg();
893 Register Src0 = MI.getOperand(i: 2).getReg();
894 Register Src1 = MI.getOperand(i: 3).getReg();
895 Register Src2 = MI.getOperand(i: 4).getReg();
896
897 const GCNSubtarget &ST = B.getMF().getSubtarget<GCNSubtarget>();
898
899 // Keep the multiplication on the SALU.
900 Register DstLo = B.buildMul(Dst: SgprRB_I32, Src0, Src1).getReg(Idx: 0);
901 Register DstHi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
902 if (ST.hasScalarMulHiInsts()) {
903 B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {{DstHi}}, SrcOps: {Src0, Src1});
904 } else {
905 auto VSrc0 = B.buildCopy(Res: VgprRB_I32, Op: Src0);
906 auto VSrc1 = B.buildCopy(Res: VgprRB_I32, Op: Src1);
907 auto MulHi = B.buildInstr(Opc: AMDGPU::G_UMULH, DstOps: {VgprRB_I32}, SrcOps: {VSrc0, VSrc1});
908 buildReadAnyLane(B, SgprDst: DstHi, VgprSrc: MulHi.getReg(Idx: 0), RBI);
909 }
910
911 // Accumulate and produce the "carry-out" bit.
912
913 // The "carry-out" is defined as bit 64 of the result when computed as a
914 // big integer. For unsigned multiply-add, this matches the usual
915 // definition of carry-out.
916 if (mi_match(R: Src2, MRI, P: MIPatternMatch::m_ZeroInt())) {
917 // No accumulate: result is just the multiplication, carry is 0.
918 B.buildMergeLikeInstr(Res: Dst0, Ops: {DstLo, DstHi});
919 B.buildConstant(Res: Dst1, Val: 0);
920 } else {
921 // Accumulate: add Src2 to the multiplication result with carry chain.
922 Register Src2Lo = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
923 Register Src2Hi = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
924 B.buildUnmerge(Res: {Src2Lo, Src2Hi}, Op: Src2);
925
926 auto AddLo = B.buildUAddo(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstLo, Op1: Src2Lo);
927 auto AddHi =
928 B.buildUAdde(Res: SgprRB_I32, CarryOut: SgprRB_I32, Op0: DstHi, Op1: Src2Hi, CarryIn: AddLo.getReg(Idx: 1));
929 B.buildMergeLikeInstr(Res: Dst0, Ops: {AddLo.getReg(Idx: 0), AddHi.getReg(Idx: 0)});
930 B.buildCopy(Res: Dst1, Op: AddHi.getReg(Idx: 1));
931 }
932
933 MI.eraseFromParent();
934 return true;
935}
936
937bool RegBankLegalizeHelper::lowerSplitTo32Select(MachineInstr &MI) {
938 Register Dst = MI.getOperand(i: 0).getReg();
939 LLT DstTy = MRI.getType(Reg: Dst);
940 assert(DstTy == V4S16 || DstTy == V2S32 || DstTy == S64 ||
941 (DstTy.isPointer() && DstTy.getSizeInBits() == 64));
942 LLT Ty = DstTy.isFloat() ? LLT::float32() : DstTy.divide(Factor: 2);
943 auto Op2 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 2).getReg());
944 auto Op3 = B.buildUnmerge(Attrs: {.RCOrRB: VgprRB, .Ty: Ty}, Op: MI.getOperand(i: 3).getReg());
945 Register Cond = MI.getOperand(i: 1).getReg();
946 Cond = B.buildFreeze(Dst: VccRB_S1, Src: Cond).getReg(Idx: 0);
947 auto Flags = MI.getFlags();
948 auto Lo =
949 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 0), Op1: Op3.getReg(Idx: 0), Flags);
950 auto Hi =
951 B.buildSelect(Res: {VgprRB, Ty}, Tst: Cond, Op0: Op2.getReg(Idx: 1), Op1: Op3.getReg(Idx: 1), Flags);
952
953 B.buildMergeLikeInstr(Res: Dst, Ops: {Lo, Hi});
954 MI.eraseFromParent();
955 return true;
956}
957
958bool RegBankLegalizeHelper::lowerSplitTo32SExtInReg(MachineInstr &MI) {
959 auto Op1 = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
960 int Amt = MI.getOperand(i: 2).getImm();
961 Register Lo, Hi;
962 // Hi|Lo: s sign bit, ?/x bits changed/not changed by sign-extend
963 if (Amt <= 32) {
964 auto Freeze = B.buildFreeze(Dst: VgprRB_I32, Src: Op1.getReg(Idx: 0));
965 if (Amt == 32) {
966 // Hi|Lo: ????????|sxxxxxxx -> ssssssss|sxxxxxxx
967 Lo = Freeze.getReg(Idx: 0);
968 } else {
969 // Hi|Lo: ????????|???sxxxx -> ssssssss|ssssxxxx
970 Lo = B.buildSExtInReg(Res: VgprRB_I32, Op: Freeze, ImmOp: Amt).getReg(Idx: 0);
971 }
972
973 auto SignExtCst = B.buildConstant(Res: SgprRB_I32, Val: 31);
974 Hi = B.buildAShr(Dst: VgprRB_I32, Src0: Lo, Src1: SignExtCst).getReg(Idx: 0);
975 } else {
976 // Hi|Lo: ?????sxx|xxxxxxxx -> ssssssxx|xxxxxxxx
977 Lo = Op1.getReg(Idx: 0);
978 Hi = B.buildSExtInReg(Res: VgprRB_I32, Op: Op1.getReg(Idx: 1), ImmOp: Amt - 32).getReg(Idx: 0);
979 }
980
981 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(), Ops: {Lo, Hi});
982 MI.eraseFromParent();
983 return true;
984}
985
986bool RegBankLegalizeHelper::lowerSplitBitCount64To32(MachineInstr &MI) {
987 // Split 64-bit find-first-bit operations into 32-bit halves:
988 // (ffbh hi:lo) -> umin(ffbh(hi), uaddsat(ffbh(lo), 32))
989 // (ffbl hi:lo) -> umin(ffbl(lo), uaddsat(ffbl(hi), 32))
990 // (ctlz_zero_poison hi:lo) -> umin(ffbh(hi), add(ffbh(lo), 32))
991 // (cttz_zero_poison hi:lo) -> umin(ffbl(lo), add(ffbl(hi), 32))
992 unsigned Opc = MI.getOpcode();
993
994 // FFBH/FFBL return 0xFFFFFFFF on zero input, using uaddsat to avoid
995 // wrapping. CTLZ/CTTZ guarantee non-zero input (zero_poison), so plain add
996 // is fine.
997 unsigned FFBOpc;
998 unsigned AddOpc;
999 bool SearchFromMSB;
1000 switch (Opc) {
1001 case AMDGPU::G_AMDGPU_FFBH_U32:
1002 FFBOpc = Opc;
1003 AddOpc = AMDGPU::G_UADDSAT;
1004 SearchFromMSB = true;
1005 break;
1006 case AMDGPU::G_AMDGPU_FFBL_B32:
1007 FFBOpc = Opc;
1008 AddOpc = AMDGPU::G_UADDSAT;
1009 SearchFromMSB = false;
1010 break;
1011 case AMDGPU::G_CTLZ_ZERO_POISON:
1012 FFBOpc = AMDGPU::G_AMDGPU_FFBH_U32;
1013 AddOpc = AMDGPU::G_ADD;
1014 SearchFromMSB = true;
1015 break;
1016 case AMDGPU::G_CTTZ_ZERO_POISON:
1017 FFBOpc = AMDGPU::G_AMDGPU_FFBL_B32;
1018 AddOpc = AMDGPU::G_ADD;
1019 SearchFromMSB = false;
1020 break;
1021 default:
1022 llvm_unreachable("unexpected opcode in lowerSplitBitCount64To32");
1023 }
1024
1025 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1026 Register Lo = Unmerge.getReg(Idx: 0);
1027 Register Hi = Unmerge.getReg(Idx: 1);
1028
1029 // MSB-first (FFBH/CTLZ) searches hi first; LSB-first (FFBL/CTTZ) searches
1030 // lo first. The secondary half adds 32 to account for the primary half's
1031 // width.
1032 auto Primary = B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Hi : Lo});
1033 auto Secondary =
1034 B.buildInstr(Opc: FFBOpc, DstOps: {VgprRB_I32}, SrcOps: {SearchFromMSB ? Lo : Hi});
1035
1036 auto Adjusted = B.buildInstr(Opc: AddOpc, DstOps: {VgprRB_I32},
1037 SrcOps: {Secondary, B.buildConstant(Res: VgprRB_I32, Val: 32)});
1038 B.buildUMin(Dst: MI.getOperand(i: 0).getReg(), Src0: Primary, Src1: Adjusted);
1039
1040 MI.eraseFromParent();
1041 return true;
1042}
1043
1044bool RegBankLegalizeHelper::lowerExtrVecEltToSel(MachineInstr &MI) {
1045 // Lower extract vector element to a compare-select chain:
1046 // result = elt[0]
1047 // for i in 1..N-1:
1048 // result = (idx == i) ? elt[i] : result
1049 //
1050 // When the index is divergent, each lane may want a different element, so
1051 // we must check every element per lane.
1052 Register Dst = MI.getOperand(i: 0).getReg();
1053 Register Src = MI.getOperand(i: 1).getReg();
1054 Register Idx = MI.getOperand(i: 2).getReg();
1055
1056 LLT VecTy = MRI.getType(Reg: Src);
1057 LLT ScalarTy = VecTy.getScalarType();
1058 unsigned NumElts = VecTy.getNumElements();
1059 MachineRegisterInfo::VRegAttrs VgprRB_EltTy = {.RCOrRB: VgprRB, .Ty: ScalarTy};
1060
1061 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_EltTy, Op: Src);
1062
1063 if (ScalarTy.getSizeInBits() == 32) {
1064 Register PrevSelect = Unmerge.getReg(Idx: 0);
1065 for (unsigned I = 1; I < NumElts; ++I) {
1066 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1067 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1068 PrevSelect =
1069 B.buildSelect(Res: VgprRB_EltTy, Tst: Cmp, Op0: Unmerge.getReg(Idx: I), Op1: PrevSelect)
1070 .getReg(Idx: 0);
1071 }
1072 B.buildCopy(Res: Dst, Op: PrevSelect);
1073 } else if (ScalarTy.getSizeInBits() == 64) {
1074 auto InitUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: 0));
1075 Register PrevLo = InitUnmerge.getReg(Idx: 0);
1076 Register PrevHi = InitUnmerge.getReg(Idx: 1);
1077 for (unsigned I = 1; I < NumElts; ++I) {
1078 auto IdxConst = B.buildConstant(Res: {SgprRB, MRI.getType(Reg: Idx)}, Val: I);
1079 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1080 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Unmerge.getReg(Idx: I));
1081 PrevLo = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 0), Op1: PrevLo)
1082 .getReg(Idx: 0);
1083 PrevHi = B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltUnmerge.getReg(Idx: 1), Op1: PrevHi)
1084 .getReg(Idx: 0);
1085 }
1086 B.buildMergeLikeInstr(Res: Dst, Ops: {PrevLo, PrevHi});
1087 } else {
1088 reportGISelFailure(
1089 MF, MORE, DEBUG_TYPE,
1090 Msg: "AMDGPU RegBankLegalize: ExtrVecEltToSel unsupported element type", MI);
1091 return false;
1092 }
1093
1094 MI.eraseFromParent();
1095 return true;
1096}
1097
1098bool RegBankLegalizeHelper::lowerExtrVecEltTo32(MachineInstr &MI) {
1099 // Reduce a 64-bit element extract to two 32-bit extracts:
1100 // vec32 = bitcast <N x s64> to <2N x s32>
1101 // lo = vec32[idx * 2]
1102 // hi = vec32[idx * 2 + 1]
1103 // result = merge(lo, hi)
1104 //
1105 // When the index is uniform, all lanes extract the same element, so we can
1106 // just split the s64 extract into two s32 extracts which lower to MOVREL.
1107 Register Dst = MI.getOperand(i: 0).getReg();
1108 Register Src = MI.getOperand(i: 1).getReg();
1109 Register Idx = MI.getOperand(i: 2).getReg();
1110
1111 LLT SrcTy = MRI.getType(Reg: Src);
1112 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1113
1114 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1115 "expected VGPR src and SGPR idx");
1116
1117 auto CastSrc = B.buildBitcast(Dst: {VgprRB, Vec32Ty}, Src);
1118
1119 // Calculate new Lo and Hi indices
1120 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1121 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1122 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1123
1124 auto ExtLo = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxLo);
1125 auto ExtHi = B.buildExtractVectorElement(Res: VgprRB_I32, Val: CastSrc, Idx: IdxHi);
1126
1127 B.buildMergeLikeInstr(Res: Dst, Ops: {ExtLo.getReg(Idx: 0), ExtHi.getReg(Idx: 0)});
1128
1129 MI.eraseFromParent();
1130 return true;
1131}
1132
1133bool RegBankLegalizeHelper::lowerInsVecEltToSel(MachineInstr &MI) {
1134 // Lower insert vector element to a compare-select chain:
1135 // for i in 0..N-1:
1136 // result[i] = (idx == i) ? elt : srcVec[i]
1137 // dst = merge(result[0..N-1])
1138 //
1139 // VGPR B64 requires splitting to lo/hi s32 pairs since there is no
1140 // v_cndmask_b64. SGPR B64/B32 and VGPR B32 can be handled natively.
1141 Register Dst = MI.getOperand(i: 0).getReg();
1142 Register Src = MI.getOperand(i: 1).getReg();
1143 Register Elt = MI.getOperand(i: 2).getReg();
1144 Register Idx = MI.getOperand(i: 3).getReg();
1145
1146 LLT VecTy = MRI.getType(Reg: Src);
1147 LLT ScalarTy = VecTy.getScalarType();
1148 unsigned NumElts = VecTy.getNumElements();
1149 const RegisterBank *SrcRB = MRI.getRegBank(Reg: Src);
1150 bool IsSGPR = (SrcRB == SgprRB);
1151 SmallVector<Register, 16> Selects;
1152
1153 if (!IsSGPR && ScalarTy.getSizeInBits() == 64) {
1154 // VGPR B64: split to 32-bit lo/hi since there is no v_cndmask_b64.
1155 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1156 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1157 Register EltLo = EltUnmerge.getReg(Idx: 0);
1158 Register EltHi = EltUnmerge.getReg(Idx: 1);
1159 for (unsigned I = 0; I < NumElts; ++I) {
1160 auto IdxConst = B.buildConstant(Res: VgprRB_I32, Val: I);
1161 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: VccRB_S1, Op0: Idx, Op1: IdxConst);
1162 Selects.push_back(
1163 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltLo, Op1: Unmerge.getReg(Idx: 2 * I))
1164 .getReg(Idx: 0));
1165 Selects.push_back(
1166 Elt: B.buildSelect(Res: VgprRB_I32, Tst: Cmp, Op0: EltHi, Op1: Unmerge.getReg(Idx: 2 * I + 1))
1167 .getReg(Idx: 0));
1168 }
1169 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * NumElts, ScalarSizeInBits: 32);
1170 auto Vec32 = B.buildBuildVector(Res: {VgprRB, Vec32Ty}, Ops: Selects);
1171 B.buildBitcast(Dst, Src: Vec32);
1172 } else if (ScalarTy.getSizeInBits() == 32 || ScalarTy.getSizeInBits() == 64) {
1173 // B32 (any bank) and SGPR B64: element-wise select at native width.
1174 MachineRegisterInfo::VRegAttrs SrcRB_EltTy = {.RCOrRB: SrcRB, .Ty: ScalarTy};
1175 MachineRegisterInfo::VRegAttrs CmpTy = IsSGPR ? SgprRB_I32 : VccRB_S1;
1176 auto Unmerge = B.buildUnmerge(Attrs: SrcRB_EltTy, Op: Src);
1177 for (unsigned I = 0; I < NumElts; ++I) {
1178 auto IdxConst = B.buildConstant(Res: SgprRB_I32, Val: I);
1179 auto Cmp = B.buildICmp(Pred: CmpInst::ICMP_EQ, Res: CmpTy, Op0: Idx, Op1: IdxConst);
1180 Selects.push_back(
1181 Elt: B.buildSelect(Res: SrcRB_EltTy, Tst: Cmp, Op0: Elt, Op1: Unmerge.getReg(Idx: I)).getReg(Idx: 0));
1182 }
1183 B.buildMergeLikeInstr(Res: Dst, Ops: Selects);
1184 } else {
1185 reportGISelFailure(
1186 MF, MORE, DEBUG_TYPE,
1187 Msg: "AMDGPU RegBankLegalize: InsVecEltToSel unsupported element type", MI);
1188 return false;
1189 }
1190
1191 MI.eraseFromParent();
1192 return true;
1193}
1194
1195bool RegBankLegalizeHelper::lowerInsVecEltTo32(MachineInstr &MI) {
1196 // Reduce a 64-bit element insert to two 32-bit inserts:
1197 // vec32 = bitcast <N x s64> to <2N x s32>
1198 // lo, hi = unmerge elt
1199 // vec32[idx * 2] = lo
1200 // vec32[idx * 2 + 1] = hi
1201 // dst = bitcast <2N x s32> to <N x s64>
1202 //
1203 // When the index is uniform, all lanes insert at the same position, so we
1204 // can split the s64 insert into two s32 inserts which lower to MOVREL/GPRIDX.
1205 Register Dst = MI.getOperand(i: 0).getReg();
1206 Register Src = MI.getOperand(i: 1).getReg();
1207 Register Elt = MI.getOperand(i: 2).getReg();
1208 Register Idx = MI.getOperand(i: 3).getReg();
1209
1210 LLT SrcTy = MRI.getType(Reg: Src);
1211 LLT Vec32Ty = LLT::fixed_vector(NumElements: 2 * SrcTy.getNumElements(), ScalarSizeInBits: 32);
1212
1213 assert(MRI.getRegBank(Src) == VgprRB && MRI.getRegBank(Idx) == SgprRB &&
1214 "expected VGPR src and SGPR idx");
1215
1216 MachineRegisterInfo::VRegAttrs VgprRB_Vec32Ty = {.RCOrRB: VgprRB, .Ty: Vec32Ty};
1217
1218 auto CastSrc = B.buildBitcast(Dst: VgprRB_Vec32Ty, Src);
1219 auto EltUnmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: Elt);
1220
1221 // Calculate new Lo and Hi indices
1222 auto One = B.buildConstant(Res: SgprRB_I32, Val: 1);
1223 auto IdxLo = B.buildShl(Dst: SgprRB_I32, Src0: Idx, Src1: One);
1224 auto IdxHi = B.buildAdd(Dst: SgprRB_I32, Src0: IdxLo, Src1: One);
1225
1226 auto InsLo = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: CastSrc,
1227 Elt: EltUnmerge.getReg(Idx: 0), Idx: IdxLo);
1228 auto InsHi = B.buildInsertVectorElement(Res: VgprRB_Vec32Ty, Val: InsLo,
1229 Elt: EltUnmerge.getReg(Idx: 1), Idx: IdxHi);
1230
1231 B.buildBitcast(Dst, Src: InsHi);
1232
1233 MI.eraseFromParent();
1234 return true;
1235}
1236
1237bool RegBankLegalizeHelper::lowerAbsToNegMax(MachineInstr &MI) {
1238 // Lower divergent G_ABS to smax(x, 0 - x) in the VGPR bank:
1239 // zero = 0
1240 // neg = G_SUB zero, x
1241 // dst = G_SMAX x, neg
1242 //
1243 // There is no integer v_abs instruction on AMDGPU, so divergent G_ABS is
1244 // expanded to this sub/smax pair.
1245 Register DstReg = MI.getOperand(i: 0).getReg();
1246 Register SrcReg = MI.getOperand(i: 1).getReg();
1247 LLT Ty = MRI.getType(Reg: DstReg);
1248
1249 Register Zero;
1250 if (Ty == V2S16) {
1251 // buildConstant cannot produce a V2S16 directly; pack two S16 zeros.
1252 Register Zero16 = B.buildConstant(Res: {VgprRB, I16}, Val: 0).getReg(Idx: 0);
1253 Zero = B.buildBuildVector(Res: {VgprRB, Ty}, Ops: {Zero16, Zero16}).getReg(Idx: 0);
1254 } else {
1255 assert((Ty == S32 || Ty == S16) && "unexpected type for AbsToNegMax");
1256 Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0).getReg(Idx: 0);
1257 }
1258
1259 auto Neg = B.buildSub(Dst: {VgprRB, Ty}, Src0: Zero, Src1: SrcReg);
1260 B.buildSMax(Dst: DstReg, Src0: SrcReg, Src1: Neg);
1261 MI.eraseFromParent();
1262 return true;
1263}
1264
1265bool RegBankLegalizeHelper::lowerAbsToS32(MachineInstr &MI) {
1266 // Lower uniform V2S16 abs by unpacking the values to two separate SGPR
1267 // registers and re-emitting G_ABS on each:
1268 // packed = bitcast <2 x s16> src to s32
1269 // lo = sext_inreg packed, 16
1270 // hi = ashr packed, 16
1271 // dst = build_vector_trunc G_ABS(lo), G_ABS(hi)
1272 //
1273 // SALU only has s_abs_i32, with no direct uniform V2S16 abs. The
1274 // re-emitted G_ABS(SgprRB, S32) selects to s_abs_i32 on each value.
1275 auto Bitcast = B.buildBitcast(Dst: {SgprRB_I32}, Src: MI.getOperand(i: 1).getReg());
1276 auto SextInReg = B.buildSExtInReg(Res: {SgprRB_I32}, Op: Bitcast, ImmOp: 16);
1277 auto ShiftHi =
1278 B.buildAShr(Dst: {SgprRB_I32}, Src0: Bitcast, Src1: B.buildConstant(Res: {SgprRB_I32}, Val: 16));
1279
1280 auto AbsLo = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {SextInReg});
1281 auto AbsHi = B.buildInstr(Opc: AMDGPU::G_ABS, DstOps: {{SgprRB_I32}}, SrcOps: {ShiftHi});
1282 B.buildBuildVectorTrunc(Res: MI.getOperand(i: 0).getReg(),
1283 Ops: {AbsLo.getReg(Idx: 0), AbsHi.getReg(Idx: 0)});
1284
1285 MI.eraseFromParent();
1286 return true;
1287}
1288
1289// Ported from SITargetLowering::lowerSET_ROUNDING in SIISelLowering.cpp.
1290// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1291bool RegBankLegalizeHelper::lowerSetRounding(MachineInstr &MI) {
1292 Register NewMode = MI.getOperand(i: 0).getReg();
1293
1294 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
1295 // hardware MODE.fp_round values.
1296 if (auto ConstMode = getIConstantVRegValWithLookThrough(VReg: NewMode, MRI)) {
1297 uint32_t ClampedVal = std::min(
1298 a: static_cast<uint32_t>(ConstMode->Value.getZExtValue()),
1299 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
1300 uint32_t DecodedVal = AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal);
1301 NewMode = B.buildConstant(Res: SgprRB_I32, Val: DecodedVal).getReg(Idx: 0);
1302 } else {
1303 // If we know the input can only be one of the supported standard modes in
1304 // the range 0-3, we can use a simplified mapping to hardware values.
1305 KnownBits Known = VT->getKnownBits(R: NewMode);
1306 const bool UseReducedTable = Known.countMinLeadingZeros() >= 30;
1307 // The supported standard values are 0-3. The extended values start at 8. We
1308 // need to offset by 4 if the value is in the extended range.
1309
1310 if (UseReducedTable) {
1311 // Truncate to the low 32-bits.
1312 auto BitTable = B.buildConstant(
1313 Res: SgprRB_I32, Val: AMDGPU::FltRoundToHWConversionTable & 0xffff);
1314
1315 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1316 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: NewMode, Src1: Two);
1317
1318 NewMode =
1319 B.buildLShr(Dst: SgprRB_I32, Src0: BitTable, Src1: RoundModeTimesNumBits).getReg(Idx: 0);
1320
1321 // TODO: A demanded-bits simplification on the setreg source here could
1322 // likely reduce the table extracted bits into inline immediates.
1323 } else {
1324 // table_index = umin(value, value - 4)
1325 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
1326 auto NegFour = B.buildConstant(Res: SgprRB_I32, Val: -4);
1327 auto OffsetEnum = B.buildAdd(Dst: SgprRB_I32, Src0: NewMode, Src1: NegFour);
1328 auto IndexVal = B.buildUMin(Dst: SgprRB_I32, Src0: NewMode, Src1: OffsetEnum);
1329
1330 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1331 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: IndexVal, Src1: Two);
1332
1333 auto BitTable =
1334 B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundToHWConversionTable);
1335 auto TableValue =
1336 B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1337 // No need to mask out the high bits since the setreg will ignore them
1338 // anyway.
1339 NewMode = B.buildTrunc(Res: SgprRB_I32, Op: TableValue).getReg(Idx: 0);
1340 }
1341 }
1342
1343 // N.B. The setreg will be later folded into s_round_mode on supported
1344 // targets.
1345 uint32_t BothRoundHwReg =
1346 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1347 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_setreg, Res: ArrayRef<DstOp>(),
1348 /*HasSideEffects=*/true, /*isConvergent=*/false)
1349 .addImm(Val: static_cast<int16_t>(BothRoundHwReg))
1350 .addReg(RegNo: NewMode);
1351
1352 MI.eraseFromParent();
1353 return true;
1354}
1355
1356// Ported from SITargetLowering::lowerGET_ROUNDING in SIISelLowering.cpp.
1357// Keep the mapping logic and conversion tables aligned with the SDAG lowering.
1358bool RegBankLegalizeHelper::lowerGetRounding(MachineInstr &MI) {
1359 Register Dst = MI.getOperand(i: 0).getReg();
1360
1361 uint32_t BothRoundHwReg =
1362 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
1363 auto GetReg =
1364 B.buildIntrinsic(ID: Intrinsic::amdgcn_s_getreg, Res: {SgprRB_I32},
1365 /*HasSideEffects=*/true, /*isConvergent=*/false)
1366 .addImm(Val: BothRoundHwReg);
1367
1368 // There are two rounding modes, one for f32 and one for f64/f16. We only
1369 // report in the standard value range if both are the same.
1370 //
1371 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
1372 // ties away from zero is not supported, and the other values are rotated by
1373 // 1.
1374 //
1375 // If the two rounding modes are not the same, report a target defined value.
1376
1377 // Mode register rounding mode fields:
1378 //
1379 // [1:0] Single-precision round mode.
1380 // [3:2] Double/Half-precision round mode.
1381 //
1382 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
1383 //
1384 // Hardware Spec
1385 // Toward-0 3 0
1386 // Nearest Even 0 1
1387 // +Inf 1 2
1388 // -Inf 2 3
1389 // NearestAway0 N/A 4
1390 //
1391 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
1392 // table we can index by the raw hardware mode.
1393 //
1394 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
1395 auto BitTable = B.buildConstant(Res: SgprRB_I64, Val: AMDGPU::FltRoundConversionTable);
1396
1397 auto Two = B.buildConstant(Res: SgprRB_I32, Val: 2);
1398 auto RoundModeTimesNumBits = B.buildShl(Dst: SgprRB_I32, Src0: GetReg, Src1: Two);
1399
1400 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
1401 // knew only one mode was demanded.
1402 auto TableValue = B.buildLShr(Dst: SgprRB_I64, Src0: BitTable, Src1: RoundModeTimesNumBits);
1403 auto TruncTable = B.buildTrunc(Res: SgprRB_I32, Op: TableValue);
1404
1405 auto EntryMask = B.buildConstant(Res: SgprRB_I32, Val: 0xf);
1406 auto TableEntry = B.buildAnd(Dst: SgprRB_I32, Src0: TruncTable, Src1: EntryMask);
1407
1408 // There's a gap in the 4-bit encoded table and actual enum values, so offset
1409 // if it's an extended value.
1410 auto Four = B.buildConstant(Res: SgprRB_I32, Val: 4);
1411 auto EnumOffset = B.buildAdd(Dst: SgprRB_I32, Src0: TableEntry, Src1: Four);
1412 auto IsStandardMode =
1413 B.buildICmp(Pred: CmpInst::ICMP_ULT, Res: SgprRB_I32, Op0: TableEntry, Op1: Four);
1414 B.buildSelect(Res: Dst, Tst: IsStandardMode, Op0: TableEntry, Op1: EnumOffset);
1415
1416 MI.eraseFromParent();
1417 return true;
1418}
1419
1420bool RegBankLegalizeHelper::lower(MachineInstr &MI,
1421 const RegBankLLTMapping &Mapping,
1422 WaterfallInfo &WFI) {
1423
1424 switch (Mapping.LoweringMethod) {
1425 case DoNotLower:
1426 break;
1427 case VccExtToSel:
1428 return lowerVccExtToSel(MI);
1429 case UniExtToSel: {
1430 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1431 auto True = B.buildConstant(Res: {SgprRB, Ty},
1432 Val: MI.getOpcode() == AMDGPU::G_SEXT ? -1 : 1);
1433 auto False = B.buildConstant(Res: {SgprRB, Ty}, Val: 0);
1434 // Input to G_{Z|S}EXT is 'Legalizer legal' S1. Most common case is compare.
1435 // We are making select here. S1 cond was already 'any-extended to S32' +
1436 // 'AND with 1 to clean high bits' by Sgpr32AExtBoolInReg.
1437 B.buildSelect(Res: MI.getOperand(i: 0).getReg(), Tst: MI.getOperand(i: 1).getReg(), Op0: True,
1438 Op1: False);
1439 MI.eraseFromParent();
1440 return true;
1441 }
1442 case UnpackBitShift:
1443 return lowerUnpackBitShift(MI);
1444 case UnpackMinMax:
1445 return lowerUnpackMinMax(MI);
1446 case ScalarizeToS16:
1447 return lowerSplitTo16(MI);
1448 case Ext32To64: {
1449 const RegisterBank *RB = MRI.getRegBank(Reg: MI.getOperand(i: 0).getReg());
1450 MachineInstrBuilder Hi;
1451 switch (MI.getOpcode()) {
1452 case AMDGPU::G_ZEXT: {
1453 Hi = B.buildConstant(Res: {RB, I32}, Val: 0);
1454 break;
1455 }
1456 case AMDGPU::G_SEXT: {
1457 // Replicate sign bit from 32-bit extended part.
1458 auto ShiftAmt = B.buildConstant(Res: {RB, I32}, Val: 31);
1459 Hi = B.buildAShr(Dst: {RB, MRI.getType(Reg: MI.getOperand(i: 1).getReg())},
1460 Src0: MI.getOperand(i: 1).getReg(), Src1: ShiftAmt);
1461 break;
1462 }
1463 case AMDGPU::G_ANYEXT: {
1464 Hi = B.buildUndef(Res: {RB, I32});
1465 break;
1466 }
1467 default:
1468 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1469 Msg: "AMDGPU RegBankLegalize: Ext32To64, unsuported opcode",
1470 MI);
1471 return false;
1472 }
1473
1474 B.buildMergeLikeInstr(Res: MI.getOperand(i: 0).getReg(),
1475 Ops: {MI.getOperand(i: 1).getReg(), Hi});
1476 MI.eraseFromParent();
1477 return true;
1478 }
1479 case UniCstExt: {
1480 uint64_t ConstVal = MI.getOperand(i: 1).getCImm()->getZExtValue();
1481 B.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: ConstVal);
1482
1483 MI.eraseFromParent();
1484 return true;
1485 }
1486 case VgprToVccCopy: {
1487 Register Src = MI.getOperand(i: 1).getReg();
1488 LLT Ty = MRI.getType(Reg: Src);
1489 // Take lowest bit from each lane and put it in lane mask.
1490 // Lowering via compare, but we need to clean high bits first as compare
1491 // compares all bits in register.
1492 Register BoolSrc = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
1493 if (Ty == S64) {
1494 auto Src64 = B.buildUnmerge(Attrs: VgprRB_I32, Op: Src);
1495 auto One = B.buildConstant(Res: VgprRB_I32, Val: 1);
1496 auto AndLo = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 0), Src1: One);
1497 auto Zero = B.buildConstant(Res: VgprRB_I32, Val: 0);
1498 auto AndHi = B.buildAnd(Dst: VgprRB_I32, Src0: Src64.getReg(Idx: 1), Src1: Zero);
1499 B.buildMergeLikeInstr(Res: BoolSrc, Ops: {AndLo, AndHi});
1500 } else {
1501 assert(Ty == S32 || Ty == S16);
1502 auto One = B.buildConstant(Res: {VgprRB, Ty}, Val: 1);
1503 B.buildAnd(Dst: BoolSrc, Src0: Src, Src1: One);
1504 }
1505 auto Zero = B.buildConstant(Res: {VgprRB, Ty}, Val: 0);
1506 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: MI.getOperand(i: 0).getReg(), Op0: BoolSrc, Op1: Zero);
1507 MI.eraseFromParent();
1508 return true;
1509 }
1510 case V_BFE:
1511 return lowerV_BFE(MI);
1512 case S_BFE:
1513 return lowerS_BFE(MI);
1514 case UniMAD64:
1515 return lowerUniMAD64(MI);
1516 case UniMul64: {
1517 B.buildMul(Dst: MI.getOperand(i: 0), Src0: MI.getOperand(i: 1), Src1: MI.getOperand(i: 2));
1518 MI.eraseFromParent();
1519 return true;
1520 }
1521 case DivSMulToMAD: {
1522 auto Op1 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 1));
1523 auto Op2 = B.buildTrunc(Res: VgprRB_I32, Op: MI.getOperand(i: 2));
1524 auto Zero = B.buildConstant(Res: VgprRB_I64, Val: 0);
1525
1526 unsigned NewOpc = MI.getOpcode() == AMDGPU::G_AMDGPU_S_MUL_U64_U32
1527 ? AMDGPU::G_AMDGPU_MAD_U64_U32
1528 : AMDGPU::G_AMDGPU_MAD_I64_I32;
1529
1530 B.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0).getReg(), SgprRB_I32},
1531 SrcOps: {Op1, Op2, Zero});
1532 MI.eraseFromParent();
1533 return true;
1534 }
1535 case SplitTo32:
1536 return lowerSplitTo32(MI);
1537 case SplitTo32Mul:
1538 return lowerSplitTo32Mul(MI);
1539 case SplitTo32Select:
1540 return lowerSplitTo32Select(MI);
1541 case SplitTo32SExtInReg:
1542 return lowerSplitTo32SExtInReg(MI);
1543 case CtPop64To32: {
1544 auto Unmerge = B.buildUnmerge(Attrs: VgprRB_I32, Op: MI.getOperand(i: 1).getReg());
1545 auto LoPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 0));
1546 auto HiPopCnt = B.buildCTPOP(Dst: VgprRB_I32, Src0: Unmerge.getReg(Idx: 1));
1547 // Max popcount of two 32-bit values is 64, so this add cannot overflow.
1548 B.buildAdd(Dst: MI.getOperand(i: 0).getReg(), Src0: LoPopCnt, Src1: HiPopCnt,
1549 Flags: MachineInstr::NoSWrap | MachineInstr::NoUWrap);
1550
1551 MI.eraseFromParent();
1552 break;
1553 }
1554 case S_BUF_to_BUF:
1555 return lowerSBufToBuf(MI, WFI);
1556 case SplitLoad: {
1557 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1558 unsigned Size = DstTy.getSizeInBits();
1559 // Even split to 128-bit loads
1560 if (Size > 128) {
1561 LLT B128;
1562 if (DstTy.isVector()) {
1563 LLT EltTy = DstTy.getElementType();
1564 B128 = LLT::fixed_vector(NumElements: 128 / EltTy.getSizeInBits(), ScalarTy: EltTy);
1565 } else {
1566 B128 = LLT::integer(SizeInBits: 128);
1567 }
1568 if (Size / 128 == 2)
1569 splitLoad(MI, LLTBreakdown: {B128, B128});
1570 else if (Size / 128 == 4)
1571 splitLoad(MI, LLTBreakdown: {B128, B128, B128, B128});
1572 else {
1573 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1574 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1575 MI);
1576 return false;
1577 }
1578 }
1579 // 64 and 32 bit load
1580 else if (DstTy == S96)
1581 splitLoad(MI, LLTBreakdown: {S64, S32}, MergeTy: S32);
1582 else if (DstTy == V3S32)
1583 splitLoad(MI, LLTBreakdown: {V2S32, S32}, MergeTy: S32);
1584 else if (DstTy == V6S16)
1585 splitLoad(MI, LLTBreakdown: {V4S16, V2S16}, MergeTy: V2S16);
1586 else {
1587 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1588 Msg: "AMDGPU RegBankLegalize: SplitLoad, unsuported type",
1589 MI);
1590 return false;
1591 }
1592 return true;
1593 }
1594 case DynStackAlloc: {
1595 const auto &TFI = *ST.getFrameLowering();
1596 // Guard in case the stack growth direction ever changes with scratch
1597 // instructions.
1598 assert(TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
1599 "Stack grows upwards for AMDGPU");
1600
1601 Register Dst = MI.getOperand(i: 0).getReg();
1602 Register AllocSize = MI.getOperand(i: 1).getReg();
1603 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
1604
1605 // Erase before building new instrs to avoid hitting multiple Dst assert
1606 // with CSE.
1607 B.setInsertPt(MBB&: *MI.getParent(), II: std::next(x: MI.getIterator()));
1608 MI.eraseFromParent();
1609
1610 if (MRI.getRegBank(Reg: AllocSize) != SgprRB) {
1611 auto WaveReduction =
1612 B.buildIntrinsic(ID: Intrinsic::amdgcn_wave_reduce_umax, Res: {SgprRB_I32})
1613 .addUse(RegNo: AllocSize)
1614 .addImm(Val: 0);
1615 AllocSize = WaveReduction.getReg(Idx: 0);
1616 }
1617
1618 LLT PtrTy = MRI.getType(Reg: Dst);
1619 assert(PtrTy.getSizeInBits() == 32 &&
1620 "Expected 32-bit pointer for stack allocation");
1621 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1622 Register SPReg = Info->getStackPtrOffsetReg();
1623
1624 // When using flat-scratch, the stack offset is unscaled.
1625 const bool HasFlatScratch = ST.hasFlatScratchEnabled();
1626 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
1627
1628 Register AdjustedSize = AllocSize;
1629 if (!HasFlatScratch) {
1630 auto WaveSize = B.buildConstant(Res: SgprRB_I32, Val: WavefrontSizeLog2);
1631 AdjustedSize = B.buildShl(Dst: SgprRB_I32, Src0: AllocSize, Src1: WaveSize).getReg(Idx: 0);
1632 }
1633 if (Alignment > TFI.getStackAlign()) {
1634 const uint64_t EffectiveAlignment =
1635 Alignment.value() << (HasFlatScratch ? 0 : WavefrontSizeLog2);
1636 auto OldSP = B.buildCopy(Res: {SgprRB, PtrTy}, Op: SPReg);
1637 auto Tmp1 =
1638 B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: OldSP,
1639 Op1: B.buildConstant(Res: SgprRB_I32, Val: EffectiveAlignment - 1));
1640 uint64_t Mask = maskTrailingZeros<uint64_t>(N: Log2_64(Value: EffectiveAlignment));
1641 B.buildPtrMask(Res: Dst, Op0: Tmp1, Op1: B.buildConstant(Res: SgprRB_I32, Val: Mask));
1642 } else {
1643 B.buildCopy(Res: Dst, Op: SPReg);
1644 }
1645 auto PtrAdd = B.buildPtrAdd(Res: {SgprRB, PtrTy}, Op0: Dst, Op1: AdjustedSize);
1646 B.buildCopy(Res: SPReg, Op: PtrAdd);
1647 return true;
1648 }
1649 case WidenLoad: {
1650 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1651 if (DstTy == S96)
1652 widenLoad(MI, WideTy: S128);
1653 else if (DstTy == V3S32)
1654 widenLoad(MI, WideTy: V4S32, MergeTy: S32);
1655 else if (DstTy == V6S16)
1656 widenLoad(MI, WideTy: V8S16, MergeTy: V2S16);
1657 else {
1658 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1659 Msg: "AMDGPU RegBankLegalize: WidenLoad, unsuported type",
1660 MI);
1661 return false;
1662 }
1663 return true;
1664 }
1665 case UnpackAExt:
1666 return lowerUnpackAExt(MI);
1667 case WidenMMOToS32:
1668 return widenMMOToS32(MI&: cast<GAnyLoad>(Val&: MI));
1669 case VerifyAllSgpr: {
1670 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1671 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1672 }));
1673 return true;
1674 }
1675 case ApplyAllVgpr: {
1676 assert(llvm::all_of(MI.defs(), [&](const MachineOperand &Op) {
1677 return MRI.getRegBankOrNull(Op.getReg()) == VgprRB;
1678 }));
1679 B.setInstrAndDebugLoc(MI);
1680 for (unsigned i = MI.getNumDefs(); i < MI.getNumOperands(); ++i) {
1681 MachineOperand &Op = MI.getOperand(i);
1682 if (!Op.isReg())
1683 continue;
1684 Register Reg = Op.getReg();
1685 if (MRI.getRegBank(Reg) != VgprRB) {
1686 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
1687 Op.setReg(Copy.getReg(Idx: 0));
1688 }
1689 }
1690 return true;
1691 }
1692 case UnmergeToShiftTrunc: {
1693 GUnmerge *Unmerge = dyn_cast<GUnmerge>(Val: &MI);
1694 LLT Ty = MRI.getType(Reg: Unmerge->getSourceReg());
1695 if (Ty.getSizeInBits() % 32 != 0) {
1696 reportGISelFailure(MF, MORE, DEBUG_TYPE,
1697 Msg: "AMDGPU RegBankLegalize: unmerge not multiple of 32",
1698 MI);
1699 return false;
1700 }
1701
1702 B.setInstrAndDebugLoc(MI);
1703 if (Ty.getSizeInBits() > 32) {
1704 auto UnmergeV2S16 =
1705 B.buildUnmerge(Attrs: {.RCOrRB: SgprRB, .Ty: V2S16}, Op: Unmerge->getSourceReg());
1706 for (unsigned i = 0; i < UnmergeV2S16->getNumDefs(); ++i) {
1707 auto [Dst0I32, Dst1I32] =
1708 unpackAExt(Reg: UnmergeV2S16->getOperand(i).getReg());
1709 B.buildTrunc(Res: MI.getOperand(i: i * 2).getReg(), Op: Dst0I32);
1710 B.buildTrunc(Res: MI.getOperand(i: i * 2 + 1).getReg(), Op: Dst1I32);
1711 }
1712 } else {
1713 auto [Dst0I32, Dst1I32] = unpackAExt(Reg: MI.getOperand(i: 2).getReg());
1714 B.buildTrunc(Res: MI.getOperand(i: 0).getReg(), Op: Dst0I32);
1715 B.buildTrunc(Res: MI.getOperand(i: 1).getReg(), Op: Dst1I32);
1716 }
1717
1718 MI.eraseFromParent();
1719 return true;
1720 }
1721 case AextToS32InIncomingBlockGPHI: {
1722 Register Dst = MI.getOperand(i: 0).getReg();
1723 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
1724 B.setInsertPt(MBB&: *MI.getParent(), II: MI.getParent()->getFirstNonPHI());
1725 MI.getOperand(i: 0).setReg(NewDst);
1726 B.buildTrunc(Res: Dst, Op: NewDst);
1727
1728 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1729 Register UseReg = MI.getOperand(i).getReg();
1730
1731 auto DefMI = MRI.getVRegDef(Reg: UseReg)->getIterator();
1732 MachineBasicBlock *DefMBB = DefMI->getParent();
1733
1734 B.setInsertPt(MBB&: *DefMBB, II: DefMBB->SkipPHIsAndLabels(I: std::next(x: DefMI)));
1735
1736 auto NewUse = B.buildAnyExt(Res: SgprRB_I32, Op: UseReg);
1737 MI.getOperand(i).setReg(NewUse.getReg(Idx: 0));
1738 }
1739 break;
1740 }
1741 case VerifyAllSgprGPHI: {
1742 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1743 if (Op.isMBB())
1744 return true;
1745 return MRI.getRegBankOrNull(Op.getReg()) == SgprRB;
1746 }));
1747 return true;
1748 }
1749 case VerifyAllSgprOrVgprGPHI: {
1750 assert(MRI.getRegBankOrNull(MI.getOperand(0).getReg()) == VgprRB);
1751 assert(llvm::all_of(MI.operands(), [&](const MachineOperand &Op) {
1752 if (Op.isMBB())
1753 return true;
1754 const RegisterBank *RB = MRI.getRegBankOrNull(Op.getReg());
1755 return RB == VgprRB || RB == SgprRB;
1756 }));
1757 return true;
1758 }
1759 case ApplyINTRIN_IMAGE: {
1760 const AMDGPU::RsrcIntrinsic *RSrcIntrin =
1761 AMDGPU::lookupRsrcIntrinsic(Intr: AMDGPU::getIntrinsicID(I: MI));
1762 assert(RSrcIntrin && RSrcIntrin->IsImage);
1763 // The reported argument index is relative to the IR intrinsic call
1764 // arguments, so shift by the number of defs and the intrinsic ID.
1765 unsigned RsrcIdx = RSrcIntrin->RsrcArg + MI.getNumExplicitDefs() + 1;
1766 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1767 }
1768 case ApplyBVH_INTERSECT_RAY: {
1769 // Rsrc is the last register operand. Base BVH trails an A16 immediate
1770 // after rsrc; dual/BVH8 do not. Scan backwards for the last virtual
1771 // register.
1772 unsigned RsrcIdx = MI.getNumOperands();
1773 while (RsrcIdx-- > MI.getNumExplicitDefs()) {
1774 const MachineOperand &Op = MI.getOperand(i: RsrcIdx);
1775 if (Op.isReg() && Op.getReg().isVirtual())
1776 break;
1777 }
1778 return applyRegisterBanksVgprWithSgprRsrc(MI, RsrcIdx);
1779 }
1780 case SplitBitCount64To32:
1781 return lowerSplitBitCount64To32(MI);
1782 case ExtrVecEltToSel:
1783 return lowerExtrVecEltToSel(MI);
1784 case ExtrVecEltTo32:
1785 return lowerExtrVecEltTo32(MI);
1786 case InsVecEltToSel:
1787 return lowerInsVecEltToSel(MI);
1788 case InsVecEltTo32:
1789 return lowerInsVecEltTo32(MI);
1790 case AbsToNegMax:
1791 return lowerAbsToNegMax(MI);
1792 case AbsToS32:
1793 return lowerAbsToS32(MI);
1794 case DeletePrefetch:
1795 MI.eraseFromParent();
1796 return true;
1797 case LowerSetRounding:
1798 return lowerSetRounding(MI);
1799 case LowerGetRounding:
1800 return lowerGetRounding(MI);
1801 }
1802
1803 return true;
1804}
1805
1806LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
1807 switch (ID) {
1808 case Vcc:
1809 case UniInVcc:
1810 return LLT::scalar(SizeInBits: 1);
1811 case Sgpr16:
1812 case Vgpr16:
1813 case UniInVgprS16:
1814 return LLT::scalar(SizeInBits: 16);
1815 case Sgpr32:
1816 case Sgpr32_WF:
1817 case Sgpr32Trunc:
1818 case Sgpr32AExt:
1819 case Sgpr32AExtBoolInReg:
1820 case Sgpr32SExt:
1821 case Sgpr32ZExt:
1822 case UniInVgprS32:
1823 case Sgpr32ToVgprDst:
1824 case Vgpr32:
1825 case Vgpr32AExt:
1826 case Vgpr32SExt:
1827 case Vgpr32ZExt:
1828 return LLT::scalar(SizeInBits: 32);
1829 case Sgpr64:
1830 case Vgpr64:
1831 case UniInVgprS64:
1832 case Sgpr64ToVgprDst:
1833 return LLT::scalar(SizeInBits: 64);
1834 case Sgpr128:
1835 case Vgpr128:
1836 return LLT::scalar(SizeInBits: 128);
1837 case SgprP0:
1838 case SgprP0Call_WF:
1839 case VgprP0:
1840 return LLT::pointer(AddressSpace: 0, SizeInBits: 64);
1841 case SgprP1:
1842 case VgprP1:
1843 return LLT::pointer(AddressSpace: 1, SizeInBits: 64);
1844 case SgprP2:
1845 case VgprP2:
1846 return LLT::pointer(AddressSpace: 2, SizeInBits: 32);
1847 case SgprP3:
1848 case VgprP3:
1849 return LLT::pointer(AddressSpace: 3, SizeInBits: 32);
1850 case SgprP4:
1851 case SgprP4Call_WF:
1852 case VgprP4:
1853 return LLT::pointer(AddressSpace: 4, SizeInBits: 64);
1854 case SgprP5:
1855 case VgprP5:
1856 return LLT::pointer(AddressSpace: 5, SizeInBits: 32);
1857 case SgprP6:
1858 return LLT::pointer(AddressSpace: 6, SizeInBits: 32);
1859 case SgprP8:
1860 return LLT::pointer(AddressSpace: 8, SizeInBits: 128);
1861 case SgprV2S16:
1862 case VgprV2S16:
1863 case UniInVgprV2S16:
1864 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
1865 case SgprV2S32:
1866 case VgprV2S32:
1867 case UniInVgprV2S32:
1868 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
1869 case VgprV3S32:
1870 case UniInVgprV3S32:
1871 return LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
1872 case VgprV4S16:
1873 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
1874 case VgprV8S16:
1875 case UniInVgprV8S16:
1876 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
1877 case VgprV16S16:
1878 case UniInVgprV16S16:
1879 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
1880 case SgprV4S32:
1881 case SgprV4S32_WF:
1882 case SgprV4S32_ReadFirstLane:
1883 case VgprV4S32:
1884 case UniInVgprV4S32:
1885 return LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
1886 case VgprV8S32:
1887 case UniInVgprV8S32:
1888 case SgprV8S32_ReadFirstLane:
1889 return LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
1890 case VgprV2S64:
1891 case UniInVgprV2S64:
1892 return LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1893 case VgprV6S32:
1894 case UniInVgprV6S32:
1895 return LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32);
1896 case VgprV16S32:
1897 case UniInVgprV16S32:
1898 return LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
1899 case VgprV32S16:
1900 case UniInVgprV32S16:
1901 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
1902 case VgprV32S32:
1903 case UniInVgprV32S32:
1904 return LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32);
1905 default:
1906 return LLT();
1907 }
1908}
1909
1910LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
1911 switch (ID) {
1912 case SgprB32:
1913 case VgprB32:
1914 case SgprB32_M0:
1915 case SgprB32_ReadFirstLane:
1916 case UniInVgprB32:
1917 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
1918 isAnyPtr(Ty, Width: 32))
1919 return Ty;
1920 return LLT();
1921 case SgprPtr32:
1922 case VgprPtr32:
1923 return isAnyPtr(Ty, Width: 32) ? Ty : LLT();
1924 case SgprPtr64:
1925 case VgprPtr64:
1926 return isAnyPtr(Ty, Width: 64) ? Ty : LLT();
1927 case SgprPtr128:
1928 case VgprPtr128:
1929 return isAnyPtr(Ty, Width: 128) ? Ty : LLT();
1930 case SgprB64:
1931 case VgprB64:
1932 case SgprB64_ReadFirstLane:
1933 case UniInVgprB64:
1934 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
1935 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
1936 return Ty;
1937 return LLT();
1938 case SgprB96:
1939 case VgprB96:
1940 case UniInVgprB96:
1941 if (Ty == LLT::scalar(SizeInBits: 96) || Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) ||
1942 Ty == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 16))
1943 return Ty;
1944 return LLT();
1945 case SgprB128:
1946 case VgprB128:
1947 case UniInVgprB128:
1948 if (Ty.getSizeInBits() == 128)
1949 return Ty;
1950 return LLT();
1951 case VgprB160:
1952 case UniInVgprB160:
1953 if (Ty.getSizeInBits() == 160)
1954 return Ty;
1955 return LLT();
1956 case SgprB256:
1957 case VgprB256:
1958 case UniInVgprB256:
1959 if (Ty.getSizeInBits() == 256)
1960 return Ty;
1961 return LLT();
1962 case SgprB512:
1963 case VgprB512:
1964 case UniInVgprB512:
1965 if (Ty.getSizeInBits() == 512)
1966 return Ty;
1967 return LLT();
1968 case SgprBRC: {
1969 const SIRegisterInfo *TRI =
1970 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1971 unsigned LLTSize = Ty.getSizeInBits();
1972 if (LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize))
1973 return Ty;
1974 return LLT();
1975 }
1976 case VgprBRC: {
1977 const SIRegisterInfo *TRI =
1978 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
1979 if (TRI->getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()))
1980 return Ty;
1981 return LLT();
1982 }
1983 default:
1984 return LLT();
1985 }
1986}
1987
1988const RegisterBank *
1989RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
1990 switch (ID) {
1991 case Vcc:
1992 return VccRB;
1993 case Sgpr16:
1994 case Sgpr32:
1995 case Sgpr32_WF:
1996 case Sgpr64:
1997 case Sgpr128:
1998 case SgprP0:
1999 case SgprP0Call_WF:
2000 case SgprP1:
2001 case SgprP2:
2002 case SgprP3:
2003 case SgprP4:
2004 case SgprP4Call_WF:
2005 case SgprP5:
2006 case SgprP6:
2007 case SgprP8:
2008 case SgprPtr32:
2009 case SgprPtr64:
2010 case SgprPtr128:
2011 case SgprV2S16:
2012 case SgprV2S32:
2013 case SgprV4S32:
2014 case SgprV4S32_WF:
2015 case SgprV4S32_ReadFirstLane:
2016 case SgprV8S32_ReadFirstLane:
2017 case SgprB32:
2018 case SgprB64:
2019 case SgprB96:
2020 case SgprB128:
2021 case SgprB256:
2022 case SgprB512:
2023 case SgprBRC:
2024 case UniInVcc:
2025 case UniInVgprS16:
2026 case UniInVgprS32:
2027 case UniInVgprS64:
2028 case UniInVgprV2S16:
2029 case UniInVgprV2S32:
2030 case UniInVgprV3S32:
2031 case UniInVgprV4S32:
2032 case UniInVgprV2S64:
2033 case UniInVgprV6S32:
2034 case UniInVgprV8S16:
2035 case UniInVgprV8S32:
2036 case UniInVgprV16S16:
2037 case UniInVgprV16S32:
2038 case UniInVgprV32S16:
2039 case UniInVgprV32S32:
2040 case UniInVgprB32:
2041 case UniInVgprB64:
2042 case UniInVgprB96:
2043 case UniInVgprB128:
2044 case UniInVgprB160:
2045 case UniInVgprB256:
2046 case UniInVgprB512:
2047 case Sgpr32Trunc:
2048 case Sgpr32AExt:
2049 case Sgpr32AExtBoolInReg:
2050 case Sgpr32SExt:
2051 case Sgpr32ZExt:
2052 return SgprRB;
2053 case AgprAnyTy:
2054 return AgprRB;
2055 case Vgpr16:
2056 case Vgpr32:
2057 case Vgpr64:
2058 case Vgpr128:
2059 case VgprP0:
2060 case VgprP1:
2061 case VgprP2:
2062 case VgprP3:
2063 case VgprP4:
2064 case VgprP5:
2065 case VgprPtr32:
2066 case VgprPtr64:
2067 case VgprPtr128:
2068 case VgprV2S16:
2069 case VgprV2S32:
2070 case VgprV2S64:
2071 case VgprV3S32:
2072 case VgprV4S16:
2073 case VgprV8S16:
2074 case VgprV16S16:
2075 case VgprV4S32:
2076 case VgprV6S32:
2077 case VgprV8S32:
2078 case VgprV16S32:
2079 case VgprV32S16:
2080 case VgprV32S32:
2081 case VgprB32:
2082 case VgprB64:
2083 case VgprB96:
2084 case VgprB128:
2085 case VgprB160:
2086 case VgprB256:
2087 case VgprB512:
2088 case VgprBRC:
2089 case VgprAnyTy:
2090 case Vgpr32AExt:
2091 case Vgpr32SExt:
2092 case Vgpr32ZExt:
2093 case Sgpr32ToVgprDst:
2094 case Sgpr64ToVgprDst:
2095 return VgprRB;
2096 default:
2097 return nullptr;
2098 }
2099}
2100
2101bool RegBankLegalizeHelper::applyMappingDst(
2102 MachineInstr &MI, unsigned &OpIdx,
2103 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs) {
2104 // Defs start from operand 0
2105 for (; OpIdx < MethodIDs.size(); ++OpIdx) {
2106 if (MethodIDs[OpIdx] == None)
2107 continue;
2108 MachineOperand &Op = MI.getOperand(i: OpIdx);
2109 Register Reg = Op.getReg();
2110 LLT Ty = MRI.getType(Reg);
2111 [[maybe_unused]] const RegisterBank *RB = MRI.getRegBank(Reg);
2112
2113 switch (MethodIDs[OpIdx]) {
2114 // vcc, sgpr and vgpr scalars, pointers and vectors
2115 case Vcc:
2116 case Sgpr16:
2117 case Sgpr32:
2118 case Sgpr64:
2119 case Sgpr128:
2120 case SgprP0:
2121 case SgprP1:
2122 case SgprP3:
2123 case SgprP4:
2124 case SgprP5:
2125 case SgprP6:
2126 case SgprP8:
2127 case SgprV2S16:
2128 case SgprV2S32:
2129 case SgprV4S32:
2130 case Vgpr16:
2131 case Vgpr32:
2132 case Vgpr64:
2133 case Vgpr128:
2134 case VgprP0:
2135 case VgprP1:
2136 case VgprP2:
2137 case VgprP3:
2138 case VgprP4:
2139 case VgprP5:
2140 case VgprV2S16:
2141 case VgprV2S32:
2142 case VgprV2S64:
2143 case VgprV3S32:
2144 case VgprV4S16:
2145 case VgprV8S16:
2146 case VgprV16S16:
2147 case VgprV4S32:
2148 case VgprV6S32:
2149 case VgprV8S32:
2150 case VgprV16S32:
2151 case VgprV32S16:
2152 case VgprV32S32: {
2153 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2154 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2155 break;
2156 }
2157 // sgpr and vgpr B-types
2158 case SgprB32:
2159 case SgprB64:
2160 case SgprB96:
2161 case SgprB128:
2162 case SgprB256:
2163 case SgprB512:
2164 case SgprBRC:
2165 case SgprPtr32:
2166 case SgprPtr64:
2167 case SgprPtr128:
2168 case VgprB32:
2169 case VgprB64:
2170 case VgprB96:
2171 case VgprB128:
2172 case VgprB160:
2173 case VgprB256:
2174 case VgprB512:
2175 case VgprBRC:
2176 case VgprPtr32:
2177 case VgprPtr64:
2178 case VgprPtr128: {
2179 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2180 assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
2181 break;
2182 }
2183 case VgprAnyTy: {
2184 assert(RB == VgprRB);
2185 break;
2186 }
2187 case AgprAnyTy: {
2188 if (RB == AgprRB)
2189 break;
2190 Register NewAgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: AgprRB, .Ty: Ty});
2191 Op.setReg(NewAgprDst);
2192 if (!MRI.use_nodbg_empty(RegNo: Reg))
2193 B.buildCopy(Res: Reg, Op: NewAgprDst);
2194 break;
2195 }
2196 case VgprOrAgprAnyTy: {
2197 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2198 const RegisterBank *DstRB =
2199 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2200 if (RB == DstRB)
2201 break;
2202 Register NewDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: DstRB, .Ty: Ty});
2203 Op.setReg(NewDst);
2204 if (!MRI.use_nodbg_empty(RegNo: Reg))
2205 B.buildCopy(Res: Reg, Op: NewDst);
2206 break;
2207 }
2208 // uniform in vcc/vgpr: scalars, vectors and B-types
2209 case UniInVcc: {
2210 assert(Ty == S1);
2211 assert(RB == SgprRB);
2212 Register NewDst = MRI.createVirtualRegister(RegAttr: VccRB_S1);
2213 Op.setReg(NewDst);
2214 if (!MRI.use_empty(RegNo: Reg)) {
2215 auto CopyS32_Vcc =
2216 B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_SCC_VCC, DstOps: {SgprRB_I32}, SrcOps: {NewDst});
2217 B.buildTrunc(Res: Reg, Op: CopyS32_Vcc);
2218 }
2219 break;
2220 }
2221 case UniInVgprS16: {
2222 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2223 assert(RB == SgprRB);
2224 Register NewVgprDst16 = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2225 Register NewVgprDstI32 = MRI.createVirtualRegister(RegAttr: VgprRB_I32);
2226 Register NewSgprDstI32 = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2227 Op.setReg(NewVgprDst16);
2228 B.buildAnyExt(Res: NewVgprDstI32, Op: NewVgprDst16);
2229 buildReadAnyLane(B, SgprDst: NewSgprDstI32, VgprSrc: NewVgprDstI32, RBI);
2230 B.buildTrunc(Res: Reg, Op: NewSgprDstI32);
2231 break;
2232 }
2233 case UniInVgprS32:
2234 case UniInVgprS64:
2235 case UniInVgprV2S16:
2236 case UniInVgprV2S32:
2237 case UniInVgprV3S32:
2238 case UniInVgprV4S32:
2239 case UniInVgprV2S64:
2240 case UniInVgprV6S32:
2241 case UniInVgprV8S16:
2242 case UniInVgprV8S32:
2243 case UniInVgprV16S16:
2244 case UniInVgprV16S32:
2245 case UniInVgprV32S16:
2246 case UniInVgprV32S32: {
2247 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2248 assert(RB == SgprRB);
2249 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2250 Op.setReg(NewVgprDst);
2251 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2252 break;
2253 }
2254 case UniInVgprB32:
2255 case UniInVgprB64:
2256 case UniInVgprB96:
2257 case UniInVgprB128:
2258 case UniInVgprB160:
2259 case UniInVgprB256:
2260 case UniInVgprB512: {
2261 assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
2262 assert(RB == SgprRB);
2263 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: Ty});
2264 Op.setReg(NewVgprDst);
2265 AMDGPU::buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2266 break;
2267 }
2268 // sgpr trunc
2269 case Sgpr32Trunc: {
2270 assert(Ty.getSizeInBits() < 32);
2271 assert(RB == SgprRB);
2272 Register NewDst = MRI.createVirtualRegister(RegAttr: SgprRB_I32);
2273 Op.setReg(NewDst);
2274 if (!MRI.use_empty(RegNo: Reg))
2275 B.buildTrunc(Res: Reg, Op: NewDst);
2276 break;
2277 }
2278 case Sgpr32ToVgprDst:
2279 case Sgpr64ToVgprDst: {
2280 assert(Ty == getTyFromID(MethodIDs[OpIdx]));
2281 assert(RB == VgprRB);
2282 Op.setReg(MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty}));
2283 B.buildCopy(Res: Reg, Op: Op.getReg());
2284 break;
2285 }
2286 case InvalidMapping: {
2287 reportGISelFailure(
2288 MF, MORE, DEBUG_TYPE,
2289 Msg: "AMDGPU RegBankLegalize: missing fast rule ('Div' or 'Uni') for", MI);
2290 return false;
2291 }
2292 default:
2293 reportGISelFailure(
2294 MF, MORE, DEBUG_TYPE,
2295 Msg: "AMDGPU RegBankLegalize: applyMappingDst, ID not supported", MI);
2296 return false;
2297 }
2298 }
2299
2300 return true;
2301}
2302
2303bool RegBankLegalizeHelper::applyMappingSrc(
2304 MachineInstr &MI, unsigned &OpIdx,
2305 const SmallVectorImpl<RegBankLLTMappingApplyID> &MethodIDs,
2306 WaterfallInfo &WFI) {
2307 for (unsigned i = 0; i < MethodIDs.size(); ++OpIdx, ++i) {
2308 if (MethodIDs[i] == None || MethodIDs[i] == IntrId || MethodIDs[i] == Imm)
2309 continue;
2310
2311 MachineOperand &Op = MI.getOperand(i: OpIdx);
2312 Register Reg = Op.getReg();
2313 LLT Ty = MRI.getType(Reg);
2314 const RegisterBank *RB = MRI.getRegBank(Reg);
2315
2316 switch (MethodIDs[i]) {
2317 case Vcc: {
2318 assert(Ty == S1);
2319 assert(RB == VccRB || RB == SgprRB);
2320 if (RB == SgprRB) {
2321 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2322 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2323 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2324 auto CopyVcc_Scc = B.buildInstr(Opc: AMDGPU::G_AMDGPU_COPY_VCC_SCC,
2325 DstOps: {VccRB_S1}, SrcOps: {BoolInReg});
2326 Op.setReg(CopyVcc_Scc.getReg(Idx: 0));
2327 }
2328 break;
2329 }
2330 // sgpr scalars, pointers and vectors
2331 case Sgpr16:
2332 case Sgpr32:
2333 case Sgpr64:
2334 case Sgpr128:
2335 case SgprP0:
2336 case SgprP1:
2337 case SgprP3:
2338 case SgprP4:
2339 case SgprP5:
2340 case SgprP6:
2341 case SgprP8:
2342 case SgprV2S16:
2343 case SgprV2S32:
2344 case SgprV4S32: {
2345 assert(Ty == getTyFromID(MethodIDs[i]));
2346 assert(RB == getRegBankFromID(MethodIDs[i]));
2347 break;
2348 }
2349 // sgpr B-types
2350 case SgprB32:
2351 case SgprB64:
2352 case SgprB96:
2353 case SgprB128:
2354 case SgprB256:
2355 case SgprB512:
2356 case SgprBRC:
2357 case SgprPtr32:
2358 case SgprPtr64:
2359 case SgprPtr128: {
2360 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2361 assert(RB == getRegBankFromID(MethodIDs[i]));
2362 break;
2363 }
2364 // vgpr scalars, pointers and vectors
2365 case Vgpr16:
2366 case Vgpr32:
2367 case Vgpr64:
2368 case Vgpr128:
2369 case VgprP0:
2370 case VgprP1:
2371 case VgprP2:
2372 case VgprP3:
2373 case VgprP4:
2374 case VgprP5:
2375 case VgprV2S16:
2376 case VgprV2S32:
2377 case VgprV2S64:
2378 case VgprV3S32:
2379 case VgprV4S16:
2380 case VgprV8S16:
2381 case VgprV16S16:
2382 case VgprV4S32:
2383 case VgprV6S32:
2384 case VgprV8S32:
2385 case VgprV16S32:
2386 case VgprV32S16:
2387 case VgprV32S32: {
2388 assert(Ty == getTyFromID(MethodIDs[i]));
2389 if (RB != VgprRB) {
2390 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2391 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2392 }
2393 break;
2394 }
2395 // vgpr B-types
2396 case VgprB32:
2397 case VgprB64:
2398 case VgprB96:
2399 case VgprB128:
2400 case VgprB160:
2401 case VgprB256:
2402 case VgprB512:
2403 case VgprBRC:
2404 case VgprPtr32:
2405 case VgprPtr64:
2406 case VgprPtr128: {
2407 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2408 if (RB != VgprRB) {
2409 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2410 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2411 }
2412 break;
2413 }
2414 case VgprAnyTy: {
2415 if (RB != VgprRB) {
2416 auto CopyToVgpr = B.buildCopy(Res: {VgprRB, Ty}, Op: Reg);
2417 Op.setReg(CopyToVgpr.getReg(Idx: 0));
2418 }
2419 break;
2420 }
2421 case AgprAnyTy: {
2422 if (RB != AgprRB) {
2423 auto CopyToAgpr = B.buildCopy(Res: {AgprRB, Ty}, Op: Reg);
2424 Op.setReg(CopyToAgpr.getReg(Idx: 0));
2425 }
2426 break;
2427 }
2428 case VgprOrAgprAnyTy: {
2429 const unsigned NumRegs = Ty.getSizeInBits() / 32;
2430 const RegisterBank *SrcRB =
2431 MFI->selectAGPRFormMFMA(NumRegs) ? AgprRB : VgprRB;
2432 if (RB != SrcRB)
2433 Op.setReg(B.buildCopy(Res: {SrcRB, Ty}, Op: Reg).getReg(Idx: 0));
2434 break;
2435 }
2436 // sgpr waterfall, scalars, and vectors
2437 case Sgpr32_WF:
2438 case SgprV4S32_WF: {
2439 assert(Ty == getTyFromID(MethodIDs[i]));
2440 if (RB != SgprRB) {
2441 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2442 if (!WFI.Start.isValid()) {
2443 WFI.Start = MI.getIterator();
2444 WFI.End = std::next(x: MI.getIterator());
2445 }
2446 }
2447 break;
2448 }
2449 case SgprP0Call_WF:
2450 case SgprP4Call_WF: {
2451 assert(Ty == getTyFromID(MethodIDs[i]));
2452 if (RB != SgprRB) {
2453 WFI.SgprWaterfallOperandRegs.insert(V: Reg);
2454
2455 // Find the ADJCALLSTACKUP before the call.
2456 MachineBasicBlock::iterator Start = MI.getIterator();
2457 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
2458 --Start;
2459
2460 // Find the ADJCALLSTACKDOWN after the call (include it in range).
2461 MachineBasicBlock::iterator End = MI.getIterator();
2462 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
2463 ++End;
2464 ++End;
2465
2466 WFI.Start = Start;
2467 WFI.End = End;
2468 }
2469 break;
2470 }
2471 case SgprB32_M0:
2472 case SgprB32_ReadFirstLane:
2473 case SgprB64_ReadFirstLane: {
2474 assert(Ty == getBTyFromID(MethodIDs[i], Ty));
2475 if (RB == SgprRB)
2476 break;
2477 assert(RB == VgprRB);
2478 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2479 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2480 Op.setReg(NewSGPR);
2481 break;
2482 }
2483 case SgprV4S32_ReadFirstLane:
2484 case SgprV8S32_ReadFirstLane: {
2485 assert(Ty == getTyFromID(MethodIDs[i]));
2486 if (RB == SgprRB)
2487 break;
2488 assert(RB == VgprRB);
2489 Register NewSGPR = MRI.createVirtualRegister(RegAttr: {.RCOrRB: SgprRB, .Ty: Ty});
2490 buildReadFirstLane(B, SgprDst: NewSGPR, VgprSrc: Op.getReg(), RBI);
2491 Op.setReg(NewSGPR);
2492 break;
2493 }
2494 // sgpr and vgpr scalars with extend
2495 case Sgpr32AExt: {
2496 // Note: this ext allows S1, and it is meant to be combined away.
2497 assert(Ty.getSizeInBits() < 32);
2498 assert(RB == SgprRB);
2499 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2500 Op.setReg(Aext.getReg(Idx: 0));
2501 break;
2502 }
2503 case Sgpr32AExtBoolInReg: {
2504 // Note: this ext allows S1, and it is meant to be combined away.
2505 assert(Ty.getSizeInBits() == 1);
2506 assert(RB == SgprRB);
2507 auto Aext = B.buildAnyExt(Res: SgprRB_I32, Op: Reg);
2508 // Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
2509 // most of times meant to be combined away in AMDGPURegBankCombiner.
2510 auto Cst1 = B.buildConstant(Res: SgprRB_I32, Val: 1);
2511 auto BoolInReg = B.buildAnd(Dst: SgprRB_I32, Src0: Aext, Src1: Cst1);
2512 Op.setReg(BoolInReg.getReg(Idx: 0));
2513 break;
2514 }
2515 case Sgpr32SExt: {
2516 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2517 assert(RB == SgprRB);
2518 auto Sext = B.buildSExt(Res: SgprRB_I32, Op: Reg);
2519 Op.setReg(Sext.getReg(Idx: 0));
2520 break;
2521 }
2522 case Sgpr32ZExt: {
2523 assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
2524 assert(RB == SgprRB);
2525 auto Zext = B.buildZExt(Res: SgprRB_I32, Op: Reg);
2526 Op.setReg(Zext.getReg(Idx: 0));
2527 break;
2528 }
2529 case Vgpr32AExt: {
2530 assert(Ty.getSizeInBits() < 32);
2531 assert(RB == VgprRB);
2532 auto Aext = B.buildAnyExt(Res: VgprRB_I32, Op: Reg);
2533 Op.setReg(Aext.getReg(Idx: 0));
2534 break;
2535 }
2536 case Vgpr32SExt: {
2537 // Note this ext allows S1, and it is meant to be combined away.
2538 assert(Ty.getSizeInBits() < 32);
2539 assert(RB == VgprRB);
2540 auto Sext = B.buildSExt(Res: VgprRB_I32, Op: Reg);
2541 Op.setReg(Sext.getReg(Idx: 0));
2542 break;
2543 }
2544 case Vgpr32ZExt: {
2545 // Note this ext allows S1, and it is meant to be combined away.
2546 assert(Ty.getSizeInBits() < 32);
2547 assert(RB == VgprRB);
2548 auto Zext = B.buildZExt(Res: VgprRB_I32, Op: Reg);
2549 Op.setReg(Zext.getReg(Idx: 0));
2550 break;
2551 }
2552 default:
2553 reportGISelFailure(
2554 MF, MORE, DEBUG_TYPE,
2555 Msg: "AMDGPU RegBankLegalize: applyMappingSrc, ID not supported", MI);
2556 return false;
2557 }
2558 }
2559 return true;
2560}
2561
2562[[maybe_unused]] static bool verifyRegBankOnOperands(MachineInstr &MI,
2563 const RegisterBank *RB,
2564 MachineRegisterInfo &MRI,
2565 unsigned StartOpIdx,
2566 unsigned EndOpIdx) {
2567 for (unsigned i = StartOpIdx; i <= EndOpIdx; ++i) {
2568 if (MRI.getRegBankOrNull(Reg: MI.getOperand(i).getReg()) != RB)
2569 return false;
2570 }
2571 return true;
2572}
2573
2574bool RegBankLegalizeHelper::applyRegisterBanksVgprWithSgprRsrc(
2575 MachineInstr &MI, unsigned RsrcIdx) {
2576 const unsigned NumDefs = MI.getNumExplicitDefs();
2577
2578 MachineBasicBlock *MBB = MI.getParent();
2579 B.setInsertPt(MBB&: *MBB, II: MBB->SkipPHIsAndLabels(I: std::next(x: MI.getIterator())));
2580
2581 // Defs are vgpr.
2582 for (unsigned i = 0; i < NumDefs; ++i) {
2583 Register Reg = MI.getOperand(i).getReg();
2584 if (MRI.getRegBank(Reg) == VgprRB)
2585 continue;
2586
2587 Register NewVgprDst = MRI.createVirtualRegister(RegAttr: {.RCOrRB: VgprRB, .Ty: MRI.getType(Reg)});
2588 MI.getOperand(i).setReg(NewVgprDst);
2589 buildReadAnyLane(B, SgprDst: Reg, VgprSrc: NewVgprDst, RBI);
2590 }
2591
2592 B.setInstrAndDebugLoc(MI);
2593
2594 // Register uses before RsrcIdx are vgpr.
2595 for (unsigned i = NumDefs; i < RsrcIdx; ++i) {
2596 MachineOperand &Op = MI.getOperand(i);
2597 if (!Op.isReg())
2598 continue;
2599
2600 Register Reg = Op.getReg();
2601 if (!Reg.isVirtual())
2602 continue;
2603
2604 if (MRI.getRegBank(Reg) == VgprRB)
2605 continue;
2606
2607 auto Copy = B.buildCopy(Res: {VgprRB, MRI.getType(Reg)}, Op: Reg);
2608 Op.setReg(Copy.getReg(Idx: 0));
2609 }
2610
2611 SmallSet<Register, 4> OpsToWaterfall;
2612
2613 // Register use RsrcIdx (and later register operands) is sgpr.
2614 for (unsigned i = RsrcIdx; i < MI.getNumOperands(); ++i) {
2615 MachineOperand &Op = MI.getOperand(i);
2616 if (!Op.isReg())
2617 continue;
2618
2619 Register Reg = Op.getReg();
2620 if (MRI.getRegBank(Reg) != SgprRB)
2621 OpsToWaterfall.insert(V: Reg);
2622 }
2623
2624 if (!OpsToWaterfall.empty()) {
2625 MachineBasicBlock::iterator MII = MI.getIterator();
2626 executeInWaterfallLoop(B, WFI: {.SgprWaterfallOperandRegs: OpsToWaterfall, .Start: MII, .End: std::next(x: MII)});
2627 }
2628
2629 return true;
2630}
2631