1//===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// SI Implementation of TargetInstrInfo.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIInstrInfo.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "GCNHazardRecognizer.h"
19#include "GCNSubtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/LiveIntervals.h"
26#include "llvm/CodeGen/MachineCycleAnalysis.h"
27#include "llvm/CodeGen/MachineDominators.h"
28#include "llvm/CodeGen/MachineFrameInfo.h"
29#include "llvm/CodeGen/MachineScheduler.h"
30#include "llvm/CodeGen/RegisterScavenging.h"
31#include "llvm/CodeGen/ScheduleDAG.h"
32#include "llvm/IR/DiagnosticInfo.h"
33#include "llvm/IR/IntrinsicsAMDGPU.h"
34#include "llvm/MC/MCContext.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Target/TargetMachine.h"
37#include <tuple>
38
39using namespace llvm;
40
41#define DEBUG_TYPE "si-instr-info"
42
43#define GET_INSTRINFO_CTOR_DTOR
44#include "AMDGPUGenInstrInfo.inc"
45
46namespace llvm::AMDGPU {
47#define GET_ImageDimIntrinsicTable_IMPL
48#define GET_RsrcIntrinsics_IMPL
49#define GET_GFX1250BlockingCyclesTable_DECL
50#define GET_GFX1250BlockingCyclesTable_IMPL
51
52struct AMDGPUBlockingCyclesInfo {
53 uint16_t Opcode;
54 uint8_t GFX1250BlockingCycles;
55};
56
57#include "AMDGPUGenSearchableTables.inc"
58} // namespace llvm::AMDGPU
59
60// Must be at least 4 to be able to branch over minimum unconditional branch
61// code. This is only for making it possible to write reasonably small tests for
62// long branches.
63static cl::opt<unsigned>
64BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(Val: 16),
65 cl::desc("Restrict range of branch instructions (DEBUG)"));
66
67static cl::opt<bool> Fix16BitCopies(
68 "amdgpu-fix-16-bit-physreg-copies",
69 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"),
70 cl::init(Val: true),
71 cl::ReallyHidden);
72
73SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
74 : AMDGPUGenInstrInfo(ST, RI, AMDGPU::ADJCALLSTACKUP,
75 AMDGPU::ADJCALLSTACKDOWN),
76 RI(ST), ST(ST) {
77 SchedModel.init(TSInfo: &ST);
78}
79
80//===----------------------------------------------------------------------===//
81// TargetInstrInfo callbacks
82//===----------------------------------------------------------------------===//
83
84static unsigned getNumOperandsNoGlue(SDNode *Node) {
85 unsigned N = Node->getNumOperands();
86 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
87 --N;
88 return N;
89}
90
91/// Returns true if both nodes have the same value for the given
92/// operand \p Op, or if both nodes do not have this operand.
93static bool nodesHaveSameOperandValue(SDNode *N0, SDNode *N1,
94 AMDGPU::OpName OpName) {
95 unsigned Opc0 = N0->getMachineOpcode();
96 unsigned Opc1 = N1->getMachineOpcode();
97
98 int Op0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: OpName);
99 int Op1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: OpName);
100
101 if (Op0Idx == -1 && Op1Idx == -1)
102 return true;
103
104
105 if ((Op0Idx == -1 && Op1Idx != -1) ||
106 (Op1Idx == -1 && Op0Idx != -1))
107 return false;
108
109 // getNamedOperandIdx returns the index for the MachineInstr's operands,
110 // which includes the result as the first operand. We are indexing into the
111 // MachineSDNode's operands, so we need to skip the result operand to get
112 // the real index.
113 --Op0Idx;
114 --Op1Idx;
115
116 return N0->getOperand(Num: Op0Idx) == N1->getOperand(Num: Op1Idx);
117}
118
119static bool canRemat(const MachineInstr &MI) {
120
121 if (SIInstrInfo::isVOP1(MI) || SIInstrInfo::isVOP2(MI) ||
122 SIInstrInfo::isVOP3(MI) || SIInstrInfo::isSDWA(MI) ||
123 SIInstrInfo::isSALU(MI))
124 return true;
125
126 if (SIInstrInfo::isSMRD(MI)) {
127 return !MI.memoperands_empty() &&
128 llvm::all_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
129 return MMO->isLoad() && MMO->isInvariant();
130 });
131 }
132
133 return false;
134}
135
136// Split relocation flags for 64-bit global-address materialization into a
137// common base and the hi/lo relocation variants.
138static std::tuple<unsigned, unsigned, unsigned>
139splitGlobalAddressRelocFlags(const GCNSubtarget &ST,
140 const MachineOperand &SrcOp) {
141 const unsigned BaseFlags = SrcOp.getTargetFlags() & ~SIInstrInfo::MO_MASK;
142 const unsigned Reloc = SrcOp.getTargetFlags() & SIInstrInfo::MO_MASK;
143
144 // Infer the relocation type from the existing flags on the global operand.
145 // The relocation type should have been determined earlier in the pipeline.
146 unsigned LoReloc, HiReloc;
147 switch (Reloc) {
148 case SIInstrInfo::MO_REL32_LO:
149 case SIInstrInfo::MO_REL32_HI:
150 case SIInstrInfo::MO_REL64:
151 LoReloc = SIInstrInfo::MO_REL32_LO;
152 HiReloc = SIInstrInfo::MO_REL32_HI;
153 break;
154 case SIInstrInfo::MO_GOTPCREL32_LO:
155 case SIInstrInfo::MO_GOTPCREL32_HI:
156 LoReloc = SIInstrInfo::MO_GOTPCREL32_LO;
157 HiReloc = SIInstrInfo::MO_GOTPCREL32_HI;
158 break;
159 case SIInstrInfo::MO_GOTPCREL:
160 case SIInstrInfo::MO_GOTPCREL64:
161 // For 64-bit GOT-relative, use the 64-bit relocation.
162 LoReloc = SIInstrInfo::MO_GOTPCREL64;
163 HiReloc = SIInstrInfo::MO_GOTPCREL64;
164 break;
165 case SIInstrInfo::MO_ABS32_LO:
166 case SIInstrInfo::MO_ABS32_HI:
167 case SIInstrInfo::MO_ABS64:
168 LoReloc = SIInstrInfo::MO_ABS32_LO;
169 HiReloc = SIInstrInfo::MO_ABS32_HI;
170 break;
171 default:
172 llvm_unreachable("unknown relocation type for global address");
173 break;
174 }
175
176 return {BaseFlags, LoReloc, HiReloc};
177}
178
179bool SIInstrInfo::isReMaterializableImpl(
180 const MachineInstr &MI) const {
181
182 if (canRemat(MI)) {
183 // Normally VALU use of exec would block the rematerialization, but that
184 // is OK in this case to have an implicit exec read as all VALU do.
185 // We really want all of the generic logic for this except for this.
186
187 // Another potential implicit use is mode register. The core logic of
188 // the RA will not attempt rematerialization if mode is set anywhere
189 // in the function, otherwise it is safe since mode is not changed.
190
191 // There is difference to generic method which does not allow
192 // rematerialization if there are virtual register uses. We allow this,
193 // therefore this method includes SOP instructions as well.
194 if (!MI.hasImplicitDef() &&
195 MI.getNumImplicitOperands() == MI.getDesc().implicit_uses().size() &&
196 !MI.mayRaiseFPException())
197 return true;
198 }
199
200 // Everything below copied from TargetInstrInfo::isReMaterializableImpl. The
201 // only difference is that we allow operations that perform read-modify-write
202 // on sub-registers.
203
204 // Remat clients assume operand 0 is the defined register.
205 if (!MI.getNumOperands() || !MI.getOperand(i: 0).isReg())
206 return false;
207 Register DefReg = MI.getOperand(i: 0).getReg();
208
209 const MachineFunction &MF = *MI.getMF();
210
211 // A load from a fixed stack slot can be rematerialized. This may be
212 // redundant with subsequent checks, but it's target-independent,
213 // simple, and a common case.
214 int FrameIdx = 0;
215 if (isLoadFromStackSlot(MI, FrameIndex&: FrameIdx) &&
216 MF.getFrameInfo().isImmutableObjectIndex(ObjectIdx: FrameIdx))
217 return true;
218
219 // Avoid instructions obviously unsafe for remat.
220 if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() ||
221 MI.hasUnmodeledSideEffects())
222 return false;
223
224 // Don't remat inline asm. We have no idea how expensive it is
225 // even if it's side effect free.
226 if (MI.isInlineAsm())
227 return false;
228
229 // Avoid instructions which load from potentially varying memory.
230 if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad())
231 return false;
232
233 const MachineRegisterInfo &MRI = MF.getRegInfo();
234
235 // If any of the registers accessed are non-constant, conservatively assume
236 // the instruction is not rematerializable.
237 for (const MachineOperand &MO : MI.operands()) {
238 if (!MO.isReg())
239 continue;
240 Register Reg = MO.getReg();
241 if (Reg == 0)
242 continue;
243
244 // Check for a well-behaved physical register.
245 if (Reg.isPhysical()) {
246 if (MO.isUse()) {
247 // If the physreg has no defs anywhere, it's just an ambient register
248 // and we can freely move its uses. Alternatively, if it's allocatable,
249 // it could get allocated to something with a def during allocation.
250 if (!MRI.isConstantPhysReg(PhysReg: Reg))
251 return false;
252 } else {
253 // A physreg def. We can't remat it.
254 return false;
255 }
256 continue;
257 }
258
259 // Only allow one virtual-register def. There may be multiple defs of the
260 // same virtual register, though.
261 if (MO.isDef() && Reg != DefReg)
262 return false;
263 }
264
265 return true;
266}
267
268bool SIInstrInfo::isSrc1DPPRevOpcode(const GCNSubtarget &ST, uint32_t Opcode) {
269 switch (Opcode) {
270 // v_subrev_u16 (gfx9)
271 case AMDGPU::V_SUBREV_U16_e32:
272 case AMDGPU::V_SUBREV_U16_e64:
273 // v_subrev_u32 (gfx9) / v_subrev_nc_u32 (gfx10+)
274 case AMDGPU::V_SUBREV_U32_e32:
275 case AMDGPU::V_SUBREV_U32_e64:
276 // v_subrev_co_u32
277 case AMDGPU::V_SUBREV_CO_U32_e32:
278 case AMDGPU::V_SUBREV_CO_U32_e64:
279 // v_subbrev_u32 (gfx9) / v_subrev_co_ci_u32 (gfx10+)
280 case AMDGPU::V_SUBBREV_U32_e32:
281 case AMDGPU::V_SUBBREV_U32_e64:
282 return true;
283 // REV shift opcodes worked this way before GFX11, verified on hardware
284 case AMDGPU::V_ASHRREV_I16_e32:
285 case AMDGPU::V_ASHRREV_I16_e64:
286 case AMDGPU::V_ASHRREV_I32_e32:
287 case AMDGPU::V_ASHRREV_I32_e64:
288 case AMDGPU::V_ASHRREV_I64_e64:
289 case AMDGPU::V_LSHLREV_B16_e32:
290 case AMDGPU::V_LSHLREV_B16_e64:
291 case AMDGPU::V_LSHLREV_B32_e32:
292 case AMDGPU::V_LSHLREV_B32_e64:
293 case AMDGPU::V_LSHLREV_B64_e64:
294 case AMDGPU::V_LSHRREV_B16_e32:
295 case AMDGPU::V_LSHRREV_B16_e64:
296 case AMDGPU::V_LSHRREV_B32_e32:
297 case AMDGPU::V_LSHRREV_B32_e64:
298 case AMDGPU::V_LSHRREV_B64_e64:
299 return !ST.hasGFX11Insts();
300 default:
301 return false;
302 }
303}
304
305// Returns true if the result of a VALU instruction depends on exec.
306bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
307 assert(isVALU(MI, /*AllowLDSDMA=*/true));
308
309 // If it is convergent it depends on EXEC.
310 if (MI.isConvergent())
311 return true;
312
313 // If it defines an SGPR it depends on EXEC, unless it's dead.
314 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
315 for (const MachineOperand &Def : MI.defs()) {
316 if (Def.isDead())
317 continue;
318
319 Register Reg = Def.getReg();
320 if (Reg && RI.isSGPRReg(MRI, Reg))
321 return true;
322 }
323
324 return false;
325}
326
327bool SIInstrInfo::isIgnorableUse(const MachineInstr &MI, unsigned OpIdx) const {
328 const MachineOperand &MO = MI.getOperand(i: OpIdx);
329 // Any implicit use of exec by VALU is not a real register read.
330 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
331 isVALU(MI, /*AllowLDSDMA=*/true) && !resultDependsOnExec(MI);
332}
333
334bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
335 MachineBasicBlock *SuccToSinkTo,
336 MachineCycleInfo *CI) const {
337 // Allow sinking if MI edits lane mask (divergent i1 in sgpr).
338 if (MI.getOpcode() == AMDGPU::SI_IF_BREAK)
339 return true;
340
341 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
342 // Check if sinking of MI would create temporal divergent use.
343 for (auto Op : MI.uses()) {
344 if (Op.isReg() && Op.getReg().isVirtual() &&
345 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Op.getReg()))) {
346 MachineInstr *SgprDef = MRI.getVRegDef(Reg: Op.getReg());
347 if (!SgprDef)
348 continue;
349
350 // SgprDef defined inside cycle
351 CycleRef FromCycle = CI->getCycle(Block: SgprDef->getParent());
352 if (!FromCycle)
353 continue;
354
355 CycleRef ToCycle = CI->getCycle(Block: SuccToSinkTo);
356 // Check if there is a FromCycle that contains SgprDef's basic block but
357 // does not contain SuccToSinkTo and also has divergent exit condition.
358 while (FromCycle && !(ToCycle && CI->contains(Outer: FromCycle, Inner: ToCycle))) {
359 SmallVector<MachineBasicBlock *, 1> ExitingBlocks;
360 CI->getExitingBlocks(C: FromCycle, TmpStorage&: ExitingBlocks);
361
362 // FromCycle has divergent exit condition.
363 for (MachineBasicBlock *ExitingBlock : ExitingBlocks) {
364 if (hasDivergentBranch(MBB: ExitingBlock))
365 return false;
366 }
367
368 FromCycle = CI->getParentCycle(C: FromCycle);
369 }
370 }
371 }
372
373 return true;
374}
375
376bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
377 int64_t &Offset0,
378 int64_t &Offset1) const {
379 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
380 return false;
381
382 unsigned Opc0 = Load0->getMachineOpcode();
383 unsigned Opc1 = Load1->getMachineOpcode();
384
385 // Make sure both are actually loads.
386 if (!get(Opcode: Opc0).mayLoad() || !get(Opcode: Opc1).mayLoad())
387 return false;
388
389 // A mayLoad instruction without a def is not a load. Likely a prefetch.
390 if (!get(Opcode: Opc0).getNumDefs() || !get(Opcode: Opc1).getNumDefs())
391 return false;
392
393 if (isDS(Opcode: Opc0) && isDS(Opcode: Opc1)) {
394
395 // FIXME: Handle this case:
396 if (getNumOperandsNoGlue(Node: Load0) != getNumOperandsNoGlue(Node: Load1))
397 return false;
398
399 // Check base reg.
400 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
401 return false;
402
403 // Skip read2 / write2 variants for simplicity.
404 // TODO: We should report true if the used offsets are adjacent (excluded
405 // st64 versions).
406 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
407 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
408 if (Offset0Idx == -1 || Offset1Idx == -1)
409 return false;
410
411 // XXX - be careful of dataless loads
412 // getNamedOperandIdx returns the index for MachineInstrs. Since they
413 // include the output in the operand list, but SDNodes don't, we need to
414 // subtract the index by one.
415 Offset0Idx -= get(Opcode: Opc0).NumDefs;
416 Offset1Idx -= get(Opcode: Opc1).NumDefs;
417 Offset0 = Load0->getConstantOperandVal(Num: Offset0Idx);
418 Offset1 = Load1->getConstantOperandVal(Num: Offset1Idx);
419 return true;
420 }
421
422 if (isSMRD(Opcode: Opc0) && isSMRD(Opcode: Opc1)) {
423 // Skip time and cache invalidation instructions.
424 if (!AMDGPU::hasNamedOperand(Opcode: Opc0, NamedIdx: AMDGPU::OpName::sbase) ||
425 !AMDGPU::hasNamedOperand(Opcode: Opc1, NamedIdx: AMDGPU::OpName::sbase))
426 return false;
427
428 unsigned NumOps = getNumOperandsNoGlue(Node: Load0);
429 if (NumOps != getNumOperandsNoGlue(Node: Load1))
430 return false;
431
432 // Check base reg.
433 if (Load0->getOperand(Num: 0) != Load1->getOperand(Num: 0))
434 return false;
435
436 // Match register offsets, if both register and immediate offsets present.
437 assert(NumOps == 4 || NumOps == 5);
438 if (NumOps == 5 && Load0->getOperand(Num: 1) != Load1->getOperand(Num: 1))
439 return false;
440
441 const ConstantSDNode *Load0Offset =
442 dyn_cast<ConstantSDNode>(Val: Load0->getOperand(Num: NumOps - 3));
443 const ConstantSDNode *Load1Offset =
444 dyn_cast<ConstantSDNode>(Val: Load1->getOperand(Num: NumOps - 3));
445
446 if (!Load0Offset || !Load1Offset)
447 return false;
448
449 Offset0 = Load0Offset->getZExtValue();
450 Offset1 = Load1Offset->getZExtValue();
451 return true;
452 }
453
454 // MUBUF and MTBUF can access the same addresses.
455 if ((isMUBUF(Opcode: Opc0) || isMTBUF(Opcode: Opc0)) && (isMUBUF(Opcode: Opc1) || isMTBUF(Opcode: Opc1))) {
456
457 // MUBUF and MTBUF have vaddr at different indices.
458 if (!nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::soffset) ||
459 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::vaddr) ||
460 !nodesHaveSameOperandValue(N0: Load0, N1: Load1, OpName: AMDGPU::OpName::srsrc))
461 return false;
462
463 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opcode: Opc0, Name: AMDGPU::OpName::offset);
464 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opcode: Opc1, Name: AMDGPU::OpName::offset);
465
466 if (OffIdx0 == -1 || OffIdx1 == -1)
467 return false;
468
469 // getNamedOperandIdx returns the index for MachineInstrs. Since they
470 // include the output in the operand list, but SDNodes don't, we need to
471 // subtract the index by one.
472 OffIdx0 -= get(Opcode: Opc0).NumDefs;
473 OffIdx1 -= get(Opcode: Opc1).NumDefs;
474
475 SDValue Off0 = Load0->getOperand(Num: OffIdx0);
476 SDValue Off1 = Load1->getOperand(Num: OffIdx1);
477
478 // The offset might be a FrameIndexSDNode.
479 if (!isa<ConstantSDNode>(Val: Off0) || !isa<ConstantSDNode>(Val: Off1))
480 return false;
481
482 Offset0 = Off0->getAsZExtVal();
483 Offset1 = Off1->getAsZExtVal();
484 return true;
485 }
486
487 return false;
488}
489
490static bool isStride64(unsigned Opc) {
491 switch (Opc) {
492 case AMDGPU::DS_READ2ST64_B32:
493 case AMDGPU::DS_READ2ST64_B64:
494 case AMDGPU::DS_WRITE2ST64_B32:
495 case AMDGPU::DS_WRITE2ST64_B64:
496 return true;
497 default:
498 return false;
499 }
500}
501
502bool SIInstrInfo::getMemOperandsWithOffsetWidth(
503 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
504 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
505 if (!LdSt.mayLoadOrStore())
506 return false;
507
508 unsigned Opc = LdSt.getOpcode();
509 OffsetIsScalable = false;
510 const MachineOperand *BaseOp, *OffsetOp;
511 int DataOpIdx;
512
513 if (isDS(MI: LdSt)) {
514 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::addr);
515 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
516 if (OffsetOp) {
517 // Normal, single offset LDS instruction.
518 if (!BaseOp) {
519 // DS_CONSUME/DS_APPEND use M0 for the base address.
520 // TODO: find the implicit use operand for M0 and use that as BaseOp?
521 return false;
522 }
523 BaseOps.push_back(Elt: BaseOp);
524 Offset = OffsetOp->getImm();
525 // Get appropriate operand, and compute width accordingly.
526 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
527 if (DataOpIdx == -1)
528 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
529 if (Opc == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
530 Width = LocationSize::precise(Value: 64);
531 else
532 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
533 } else {
534 // The 2 offset instructions use offset0 and offset1 instead. We can treat
535 // these as a load with a single offset if the 2 offsets are consecutive.
536 // We will use this for some partially aligned loads.
537 const MachineOperand *Offset0Op =
538 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset0);
539 const MachineOperand *Offset1Op =
540 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset1);
541
542 unsigned Offset0 = Offset0Op->getImm() & 0xff;
543 unsigned Offset1 = Offset1Op->getImm() & 0xff;
544 if (Offset0 + 1 != Offset1)
545 return false;
546
547 // Each of these offsets is in element sized units, so we need to convert
548 // to bytes of the individual reads.
549
550 unsigned EltSize;
551 if (LdSt.mayLoad())
552 EltSize = RI.getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: 0)) / 16;
553 else {
554 assert(LdSt.mayStore());
555 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
556 EltSize = RI.getRegSizeInBits(RC: *getOpRegClass(MI: LdSt, OpNo: Data0Idx)) / 8;
557 }
558
559 if (isStride64(Opc))
560 EltSize *= 64;
561
562 BaseOps.push_back(Elt: BaseOp);
563 Offset = EltSize * Offset0;
564 // Get appropriate operand(s), and compute width accordingly.
565 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
566 if (DataOpIdx == -1) {
567 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
568 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
569 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
570 Width = LocationSize::precise(
571 Value: Width.getValue() + TypeSize::getFixed(ExactSize: getOpSize(MI: LdSt, OpNo: DataOpIdx)));
572 } else {
573 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
574 }
575 }
576 return true;
577 }
578
579 if (isMUBUF(MI: LdSt) || isMTBUF(MI: LdSt)) {
580 const MachineOperand *RSrc = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::srsrc);
581 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL
582 return false;
583 BaseOps.push_back(Elt: RSrc);
584 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
585 if (BaseOp && !BaseOp->isFI())
586 BaseOps.push_back(Elt: BaseOp);
587 const MachineOperand *OffsetImm =
588 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
589 Offset = OffsetImm->getImm();
590 const MachineOperand *SOffset =
591 getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::soffset);
592 if (SOffset) {
593 if (SOffset->isReg())
594 BaseOps.push_back(Elt: SOffset);
595 else
596 Offset += SOffset->getImm();
597 }
598 // Get appropriate operand, and compute width accordingly.
599 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
600 if (DataOpIdx == -1)
601 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
602 if (DataOpIdx == -1) // LDS DMA
603 return false;
604 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
605 return true;
606 }
607
608 if (isImage(MI: LdSt)) {
609 auto RsrcOpName =
610 isMIMG(MI: LdSt) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
611 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RsrcOpName);
612 BaseOps.push_back(Elt: &LdSt.getOperand(i: SRsrcIdx));
613 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
614 if (VAddr0Idx >= 0) {
615 // GFX10 possible NSA encoding.
616 for (int I = VAddr0Idx; I < SRsrcIdx; ++I)
617 BaseOps.push_back(Elt: &LdSt.getOperand(i: I));
618 } else {
619 BaseOps.push_back(Elt: getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr));
620 }
621 Offset = 0;
622 // Get appropriate operand, and compute width accordingly.
623 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
624 if (DataOpIdx == -1)
625 return false; // no return sampler
626 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
627 return true;
628 }
629
630 if (isSMRD(MI: LdSt)) {
631 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::sbase);
632 if (!BaseOp) // e.g. S_MEMTIME
633 return false;
634 BaseOps.push_back(Elt: BaseOp);
635 OffsetOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset);
636 Offset = OffsetOp ? OffsetOp->getImm() : 0;
637 // Get appropriate operand, and compute width accordingly.
638 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
639 if (DataOpIdx == -1)
640 return false;
641 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
642 return true;
643 }
644
645 if (isFLAT(MI: LdSt)) {
646 // Instructions have either vaddr or saddr or both or none.
647 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::vaddr);
648 if (BaseOp)
649 BaseOps.push_back(Elt: BaseOp);
650 BaseOp = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::saddr);
651 if (BaseOp)
652 BaseOps.push_back(Elt: BaseOp);
653 Offset = getNamedOperand(MI: LdSt, OperandName: AMDGPU::OpName::offset)->getImm();
654 // Get appropriate operand, and compute width accordingly.
655 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
656 if (DataOpIdx == -1)
657 DataOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
658 if (DataOpIdx == -1) // LDS DMA
659 return false;
660 Width = LocationSize::precise(Value: getOpSize(MI: LdSt, OpNo: DataOpIdx));
661 return true;
662 }
663
664 return false;
665}
666
667static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
668 ArrayRef<const MachineOperand *> BaseOps1,
669 const MachineInstr &MI2,
670 ArrayRef<const MachineOperand *> BaseOps2) {
671 // Only examine the first "base" operand of each instruction, on the
672 // assumption that it represents the real base address of the memory access.
673 // Other operands are typically offsets or indices from this base address.
674 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
675 return true;
676
677 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
678 return false;
679
680 auto *MO1 = *MI1.memoperands_begin();
681 auto *MO2 = *MI2.memoperands_begin();
682 if (MO1->getAddrSpace() != MO2->getAddrSpace())
683 return false;
684
685 const auto *Base1 = MO1->getValue();
686 const auto *Base2 = MO2->getValue();
687 if (!Base1 || !Base2)
688 return false;
689 Base1 = getUnderlyingObject(V: Base1);
690 Base2 = getUnderlyingObject(V: Base2);
691
692 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
693 return false;
694
695 return Base1 == Base2;
696}
697
698bool SIInstrInfo::memOpsHaveSameBase(
699 ArrayRef<const MachineOperand *> BaseOps1,
700 ArrayRef<const MachineOperand *> BaseOps2) const {
701 // Accesses without base operands (e.g. scratch addressed only by an
702 // immediate offset) share the same implicit base.
703 if (BaseOps1.empty() || BaseOps2.empty())
704 return BaseOps1.empty() && BaseOps2.empty();
705 return memOpsHaveSameBasePtr(MI1: *BaseOps1.front()->getParent(), BaseOps1,
706 MI2: *BaseOps2.front()->getParent(), BaseOps2);
707}
708
709bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
710 int64_t Offset1, bool OffsetIsScalable1,
711 ArrayRef<const MachineOperand *> BaseOps2,
712 int64_t Offset2, bool OffsetIsScalable2,
713 unsigned ClusterSize,
714 unsigned NumBytes) const {
715 // If the mem ops (to be clustered) do not have the same base ptr, then they
716 // should not be clustered
717 if (!memOpsHaveSameBase(BaseOps1, BaseOps2))
718 return false;
719
720 // Both lists are empty or neither is, so BaseOps1 alone decides whether
721 // there is an instruction to read the budget from.
722 assert(BaseOps1.empty() == BaseOps2.empty());
723 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
724 if (!BaseOps1.empty()) {
725 const MachineFunction *MF = BaseOps1.front()->getParent()->getMF();
726 MaxMemoryClusterDWords =
727 MF->getInfo<SIMachineFunctionInfo>()->getMaxMemoryClusterDWords();
728 }
729
730 // In order to avoid register pressure, on an average, the number of DWORDS
731 // loaded together by all clustered mem ops should not exceed
732 // MaxMemoryClusterDWords. This is an empirical value based on certain
733 // observations and performance related experiments.
734 // The good thing about this heuristic is - it avoids clustering of too many
735 // sub-word loads, and also avoids clustering of wide loads. Below is the
736 // brief summary of how the heuristic behaves for various `LoadSize` when
737 // MaxMemoryClusterDWords is 8.
738 //
739 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops
740 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops
741 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops
742 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops
743 // (5) LoadSize >= 17: do not cluster
744 const unsigned LoadSize = NumBytes / ClusterSize;
745 const unsigned NumDWords = ((LoadSize + 3) / 4) * ClusterSize;
746 return NumDWords <= MaxMemoryClusterDWords;
747}
748
749// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
750// the first 16 loads will be interleaved with the stores, and the next 16 will
751// be clustered as expected. It should really split into 2 16 store batches.
752//
753// Loads are clustered until this returns false, rather than trying to schedule
754// groups of stores. This also means we have to deal with saying different
755// address space loads should be clustered, and ones which might cause bank
756// conflicts.
757//
758// This might be deprecated so it might not be worth that much effort to fix.
759bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
760 int64_t Offset0, int64_t Offset1,
761 unsigned NumLoads) const {
762 assert(Offset1 > Offset0 &&
763 "Second offset should be larger than first offset!");
764 // If we have less than 16 loads in a row, and the offsets are within 64
765 // bytes, then schedule together.
766
767 // A cacheline is 64 bytes (for global memory).
768 return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
769}
770
771static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
772 MachineBasicBlock::iterator MI,
773 const DebugLoc &DL, MCRegister DestReg,
774 MCRegister SrcReg, bool KillSrc,
775 const char *Msg = "illegal VGPR to SGPR copy") {
776 MachineFunction *MF = MBB.getParent();
777
778 LLVMContext &C = MF->getFunction().getContext();
779 C.diagnose(DI: DiagnosticInfoUnsupported(MF->getFunction(), Msg, DL, DS_Error));
780
781 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_ILLEGAL_COPY), DestReg)
782 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
783}
784
785/// Handle copying from SGPR to AGPR, or from AGPR to AGPR on GFX908. It is not
786/// possible to have a direct copy in these cases on GFX908, so an intermediate
787/// VGPR copy is required.
788static void indirectCopyToAGPR(const SIInstrInfo &TII, MachineBasicBlock &MBB,
789 MachineBasicBlock::iterator MI,
790 const DebugLoc &DL, MCRegister DestReg,
791 MCRegister SrcReg, bool KillSrc,
792 RegScavenger &RS, bool RegsOverlap,
793 Register ImpUseSuperReg = Register()) {
794 assert((TII.getSubtarget().hasMAIInsts() &&
795 !TII.getSubtarget().hasGFX90AInsts()) &&
796 "Expected GFX908 subtarget.");
797
798 assert((AMDGPU::SReg_32RegClass.contains(SrcReg) ||
799 AMDGPU::AGPR_32RegClass.contains(SrcReg)) &&
800 "Source register of the copy should be either an SGPR or an AGPR.");
801
802 assert(AMDGPU::AGPR_32RegClass.contains(DestReg) &&
803 "Destination register of the copy should be an AGPR.");
804
805 const SIRegisterInfo &RI = TII.getRegisterInfo();
806
807 // First try to find defining accvgpr_write to avoid temporary registers.
808 // In the case of copies of overlapping AGPRs, we conservatively do not
809 // reuse previous accvgpr_writes. Otherwise, we may incorrectly pick up
810 // an accvgpr_write used for this same copy due to implicit-defs
811 if (!RegsOverlap) {
812 for (auto Def = MI, E = MBB.begin(); Def != E; ) {
813 --Def;
814
815 if (!Def->modifiesRegister(Reg: SrcReg, TRI: &RI))
816 continue;
817
818 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
819 Def->getOperand(i: 0).getReg() != SrcReg)
820 break;
821
822 MachineOperand &DefOp = Def->getOperand(i: 1);
823 assert(DefOp.isReg() || DefOp.isImm());
824
825 if (DefOp.isReg()) {
826 bool SafeToPropagate = true;
827 // Check that register source operand is not clobbered before MI.
828 // Immediate operands are always safe to propagate.
829 for (auto I = Def; I != MI && SafeToPropagate; ++I)
830 if (I->modifiesRegister(Reg: DefOp.getReg(), TRI: &RI))
831 SafeToPropagate = false;
832
833 if (!SafeToPropagate)
834 break;
835
836 for (auto I = Def; I != MI; ++I)
837 I->clearRegisterKills(Reg: DefOp.getReg(), RegInfo: &RI);
838 }
839
840 MachineInstrBuilder Builder =
841 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64),
842 DestReg)
843 .add(MO: DefOp);
844
845 if (ImpUseSuperReg) {
846 Builder.addReg(RegNo: ImpUseSuperReg,
847 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
848 }
849
850 return;
851 }
852 }
853
854 RS.enterBasicBlockEnd(MBB);
855 RS.backward(I: std::next(x: MI));
856
857 // Ideally we want to have three registers for a long reg_sequence copy
858 // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
859 unsigned MaxVGPRs = RI.getRegPressureLimit(RC: &AMDGPU::VGPR_32RegClass,
860 MF&: *MBB.getParent());
861
862 // Registers in the sequence are allocated contiguously so we can just
863 // use register number to pick one of three round-robin temps.
864 unsigned RegNo = (DestReg - AMDGPU::AGPR0) % 3;
865 Register Tmp =
866 MBB.getParent()->getInfo<SIMachineFunctionInfo>()->getVGPRForAGPRCopy();
867 assert(MBB.getParent()->getRegInfo().isReserved(Tmp) &&
868 "VGPR used for an intermediate copy should have been reserved.");
869
870 // Only loop through if there are any free registers left. We don't want to
871 // spill.
872 while (RegNo--) {
873 Register Tmp2 = RS.scavengeRegisterBackwards(RC: AMDGPU::VGPR_32RegClass, To: MI,
874 /* RestoreAfter */ false, SPAdj: 0,
875 /* AllowSpill */ false);
876 if (!Tmp2 || RI.getHWRegIndex(Reg: Tmp2) >= MaxVGPRs)
877 break;
878 Tmp = Tmp2;
879 RS.setRegUsed(Reg: Tmp);
880 }
881
882 // Insert copy to temporary VGPR.
883 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32;
884 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg)) {
885 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64;
886 } else {
887 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
888 }
889
890 MachineInstrBuilder UseBuilder = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: TmpCopyOp), DestReg: Tmp)
891 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
892 if (ImpUseSuperReg) {
893 UseBuilder.addReg(RegNo: ImpUseSuperReg,
894 Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
895 }
896
897 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
898 .addReg(RegNo: Tmp, Flags: RegState::Kill);
899}
900
901static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB,
902 MachineBasicBlock::iterator MI, const DebugLoc &DL,
903 MCRegister DestReg, MCRegister SrcReg, bool KillSrc,
904 const TargetRegisterClass *RC, bool Forward) {
905 const SIRegisterInfo &RI = TII.getRegisterInfo();
906 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, EltSize: 4);
907 MachineBasicBlock::iterator I = MI;
908 MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
909
910 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) {
911 int16_t SubIdx = BaseIndices[Idx];
912 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
913 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
914 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
915 unsigned Opcode = AMDGPU::S_MOV_B32;
916
917 // Is SGPR aligned? If so try to combine with next.
918 bool AlignedDest = ((DestSubReg - AMDGPU::SGPR0) % 2) == 0;
919 bool AlignedSrc = ((SrcSubReg - AMDGPU::SGPR0) % 2) == 0;
920 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) {
921 // Can use SGPR64 copy
922 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
923 SubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
924 DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
925 SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
926 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
927 Opcode = AMDGPU::S_MOV_B64;
928 Idx++;
929 }
930
931 LastMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DestSubReg)
932 .addReg(RegNo: SrcSubReg)
933 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
934
935 if (!FirstMI)
936 FirstMI = LastMI;
937
938 if (!Forward)
939 I--;
940 }
941
942 assert(FirstMI && LastMI);
943 if (!Forward)
944 std::swap(a&: FirstMI, b&: LastMI);
945
946 if (KillSrc)
947 LastMI->addRegisterKilled(IncomingReg: SrcReg, RegInfo: &RI);
948}
949
950void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
951 MachineBasicBlock::iterator MI,
952 const DebugLoc &DL, Register DestReg,
953 Register SrcReg, bool KillSrc, bool RenamableDest,
954 bool RenamableSrc) const {
955 const TargetRegisterClass *RC = RI.getPhysRegBaseClass(Reg: DestReg);
956 unsigned Size = RI.getRegSizeInBits(RC: *RC);
957 const TargetRegisterClass *SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
958 unsigned SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
959
960 // The rest of copyPhysReg assumes Src and Dst size are the same size.
961 // TODO-GFX11_16BIT If all true 16 bit instruction patterns are completed can
962 // we remove Fix16BitCopies and this code block?
963 if (Fix16BitCopies) {
964 if (((Size == 16) != (SrcSize == 16))) {
965 // Non-VGPR Src and Dst will later be expanded back to 32 bits.
966 assert(ST.useRealTrue16Insts());
967 Register &RegToFix = (Size == 32) ? DestReg : SrcReg;
968 MCRegister SubReg = RI.getSubReg(Reg: RegToFix, Idx: AMDGPU::lo16);
969 RegToFix = SubReg;
970
971 if (DestReg == SrcReg) {
972 // Identity copy. Insert empty bundle since ExpandPostRA expects an
973 // instruction here.
974 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::BUNDLE));
975 return;
976 }
977 RC = RI.getPhysRegBaseClass(Reg: DestReg);
978 Size = RI.getRegSizeInBits(RC: *RC);
979 SrcRC = RI.getPhysRegBaseClass(Reg: SrcReg);
980 SrcSize = RI.getRegSizeInBits(RC: *SrcRC);
981 }
982 }
983
984 if (RC == &AMDGPU::VGPR_32RegClass) {
985 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
986 AMDGPU::SReg_32RegClass.contains(SrcReg) ||
987 AMDGPU::AGPR_32RegClass.contains(SrcReg));
988 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) ?
989 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32;
990 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
991 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
992 return;
993 }
994
995 if (RC == &AMDGPU::SReg_32_XM0RegClass ||
996 RC == &AMDGPU::SReg_32RegClass) {
997 if (SrcReg == AMDGPU::SCC) {
998 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg)
999 .addImm(Val: 1)
1000 .addImm(Val: 0);
1001 return;
1002 }
1003
1004 if (!AMDGPU::SReg_32RegClass.contains(Reg: SrcReg)) {
1005 if (DestReg == AMDGPU::VCC_LO) {
1006 // FIXME: Hack until VReg_1 removed.
1007 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
1008 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
1009 .addImm(Val: 0)
1010 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1011 return;
1012 }
1013
1014 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1015 return;
1016 }
1017
1018 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg)
1019 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1020 return;
1021 }
1022
1023 if (RC == &AMDGPU::SReg_64RegClass) {
1024 if (SrcReg == AMDGPU::SCC) {
1025 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg)
1026 .addImm(Val: 1)
1027 .addImm(Val: 0);
1028 return;
1029 }
1030
1031 if (!AMDGPU::SReg_64_EncodableRegClass.contains(Reg: SrcReg)) {
1032 if (DestReg == AMDGPU::VCC) {
1033 // FIXME: Hack until VReg_1 removed.
1034 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
1035 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CMP_NE_U32_e32))
1036 .addImm(Val: 0)
1037 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1038 return;
1039 }
1040
1041 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1042 return;
1043 }
1044
1045 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B64), DestReg)
1046 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1047 return;
1048 }
1049
1050 if (DestReg == AMDGPU::SCC) {
1051 // Copying 64-bit or 32-bit sources to SCC barely makes sense,
1052 // but SelectionDAG emits such copies for i1 sources.
1053 if (AMDGPU::SReg_64RegClass.contains(Reg: SrcReg)) {
1054 // This copy can only be produced by patterns
1055 // with explicit SCC, which are known to be enabled
1056 // only for subtargets with S_CMP_LG_U64 present.
1057 assert(ST.hasScalarCompareEq64());
1058 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U64))
1059 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1060 .addImm(Val: 0);
1061 } else {
1062 assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
1063 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32))
1064 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
1065 .addImm(Val: 0);
1066 }
1067
1068 return;
1069 }
1070
1071 if (RC == &AMDGPU::AGPR_32RegClass) {
1072 if (AMDGPU::VGPR_32RegClass.contains(Reg: SrcReg) ||
1073 (ST.hasGFX90AInsts() && AMDGPU::SReg_32RegClass.contains(Reg: SrcReg))) {
1074 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg)
1075 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1076 return;
1077 }
1078
1079 if (AMDGPU::AGPR_32RegClass.contains(Reg: SrcReg) && ST.hasGFX90AInsts()) {
1080 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_MOV_B32), DestReg)
1081 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1082 return;
1083 }
1084
1085 // FIXME: Pass should maintain scavenger to avoid scan through the block on
1086 // every AGPR spill.
1087 RegScavenger RS;
1088 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1089 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS, RegsOverlap: Overlap);
1090 return;
1091 }
1092
1093 if (Size == 16) {
1094 assert(AMDGPU::VGPR_16RegClass.contains(SrcReg) ||
1095 AMDGPU::SReg_LO16RegClass.contains(SrcReg) ||
1096 AMDGPU::AGPR_LO16RegClass.contains(SrcReg));
1097
1098 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(Reg: DestReg);
1099 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(Reg: SrcReg);
1100 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(Reg: DestReg);
1101 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(Reg: SrcReg);
1102 bool DstLow = !AMDGPU::isHi16Reg(Reg: DestReg, MRI: RI);
1103 bool SrcLow = !AMDGPU::isHi16Reg(Reg: SrcReg, MRI: RI);
1104 MCRegister NewDestReg = RI.get32BitRegister(Reg: DestReg);
1105 MCRegister NewSrcReg = RI.get32BitRegister(Reg: SrcReg);
1106
1107 if (IsSGPRDst) {
1108 if (!IsSGPRSrc) {
1109 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1110 return;
1111 }
1112
1113 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: NewDestReg)
1114 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1115 return;
1116 }
1117
1118 if (IsAGPRDst || IsAGPRSrc) {
1119 if (!DstLow || !SrcLow) {
1120 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1121 Msg: "Cannot use hi16 subreg with an AGPR!");
1122 }
1123
1124 copyPhysReg(MBB, MI, DL, DestReg: NewDestReg, SrcReg: NewSrcReg, KillSrc);
1125 return;
1126 }
1127
1128 if (ST.useRealTrue16Insts()) {
1129 if (IsSGPRSrc) {
1130 assert(SrcLow);
1131 SrcReg = NewSrcReg;
1132 }
1133 // Use the smaller instruction encoding if possible.
1134 if (AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: DestReg) &&
1135 (IsSGPRSrc || AMDGPU::VGPR_16_Lo128RegClass.contains(Reg: SrcReg))) {
1136 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e32), DestReg)
1137 .addReg(RegNo: SrcReg);
1138 } else {
1139 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B16_t16_e64), DestReg)
1140 .addImm(Val: 0) // src0_modifiers
1141 .addReg(RegNo: SrcReg)
1142 .addImm(Val: 0); // op_sel
1143 }
1144 return;
1145 }
1146
1147 if (IsSGPRSrc && !ST.hasSDWAScalar()) {
1148 if (!DstLow || !SrcLow) {
1149 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc,
1150 Msg: "Cannot use hi16 subreg on VI!");
1151 }
1152
1153 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: NewDestReg)
1154 .addReg(RegNo: NewSrcReg, Flags: getKillRegState(B: KillSrc));
1155 return;
1156 }
1157
1158 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: NewDestReg)
1159 .addImm(Val: 0) // src0_modifiers
1160 .addReg(RegNo: NewSrcReg)
1161 .addImm(Val: 0) // clamp
1162 .addImm(Val: DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1163 : AMDGPU::SDWA::SdwaSel::WORD_1)
1164 .addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE)
1165 .addImm(Val: SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0
1166 : AMDGPU::SDWA::SdwaSel::WORD_1)
1167 .addReg(RegNo: NewDestReg, Flags: RegState::Implicit | RegState::Undef);
1168 // First implicit operand is $exec.
1169 MIB->tieOperands(DefIdx: 0, UseIdx: MIB->getNumOperands() - 1);
1170 return;
1171 }
1172
1173 // Returns true if Dst and Src are in Opc's HwMode-resolved destination and
1174 // source operand classes.
1175 auto CanCopyWith = [&](unsigned Opc, MCRegister Dst, MCRegister Src,
1176 unsigned SrcOp = 1) {
1177 const MCInstrDesc &Desc = get(Opcode: Opc);
1178 const TargetRegisterClass *DstOpRC = getRegClass(MCID: Desc, OpNum: 0);
1179 const TargetRegisterClass *SrcOpRC = getRegClass(MCID: Desc, OpNum: SrcOp);
1180 return DstOpRC && SrcOpRC && DstOpRC->contains(Reg: Dst) &&
1181 SrcOpRC->contains(Reg: Src);
1182 };
1183
1184 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(RC: SrcRC))) {
1185 if (ST.hasVMovB64Inst() &&
1186 CanCopyWith(AMDGPU::V_MOV_B64_e32, DestReg, SrcReg)) {
1187 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B64_e32), DestReg)
1188 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
1189 return;
1190 }
1191 if (ST.hasPkMovB32() &&
1192 CanCopyWith(AMDGPU::V_PK_MOV_B32, DestReg, SrcReg, /*SrcOp=*/2)) {
1193 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg)
1194 .addImm(Val: SISrcMods::OP_SEL_1)
1195 .addReg(RegNo: SrcReg)
1196 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1197 .addReg(RegNo: SrcReg)
1198 .addImm(Val: 0) // op_sel_lo
1199 .addImm(Val: 0) // op_sel_hi
1200 .addImm(Val: 0) // neg_lo
1201 .addImm(Val: 0) // neg_hi
1202 .addImm(Val: 0) // clamp
1203 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc) | RegState::Implicit);
1204 return;
1205 }
1206 }
1207
1208 const bool Forward = RI.getHWRegIndex(Reg: DestReg) <= RI.getHWRegIndex(Reg: SrcReg);
1209 if (RI.isSGPRClass(RC)) {
1210 if (!RI.isSGPRClass(RC: SrcRC)) {
1211 reportIllegalCopy(TII: this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
1212 return;
1213 }
1214 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1215 expandSGPRCopy(TII: *this, MBB, MI, DL, DestReg, SrcReg, KillSrc: CanKillSuperReg, RC,
1216 Forward);
1217 return;
1218 }
1219
1220 unsigned Opcode = AMDGPU::V_MOV_B32_e32;
1221 unsigned WideOpcode = AMDGPU::INSTRUCTION_LIST_END;
1222 if (RI.isAGPRClass(RC)) {
1223 if (ST.hasGFX90AInsts() && RI.isAGPRClass(RC: SrcRC))
1224 Opcode = AMDGPU::V_ACCVGPR_MOV_B32;
1225 else if (RI.hasVGPRs(RC: SrcRC) ||
1226 (ST.hasGFX90AInsts() && RI.isSGPRClass(RC: SrcRC)))
1227 Opcode = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
1228 else
1229 Opcode = AMDGPU::INSTRUCTION_LIST_END;
1230 } else if (RI.hasVGPRs(RC) && RI.isAGPRClass(RC: SrcRC)) {
1231 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64;
1232 } else if (RI.isVGPRClass(RC)) {
1233 if (ST.hasVMovB64Inst())
1234 WideOpcode = AMDGPU::V_MOV_B64_e32;
1235 else if (ST.hasPkMovB32())
1236 WideOpcode = AMDGPU::V_PK_MOV_B32;
1237 }
1238
1239 const TargetRegisterClass *WideDstRC{}, *WideSrcRC{};
1240 if (WideOpcode != AMDGPU::INSTRUCTION_LIST_END) {
1241 const MCInstrDesc &Desc = get(Opcode: WideOpcode);
1242 unsigned SrcOp = WideOpcode == AMDGPU::V_PK_MOV_B32 ? 2 : 1;
1243 WideDstRC = getRegClass(MCID: Desc, OpNum: 0);
1244 WideSrcRC = getRegClass(MCID: Desc, OpNum: SrcOp);
1245 }
1246
1247 // If there is an overlap, we can't kill the super-register on the last
1248 // instruction, since it will also kill the components made live by this def.
1249 const bool Overlap = RI.regsOverlap(RegA: SrcReg, RegB: DestReg);
1250 const bool CanKillSuperReg = KillSrc && !Overlap;
1251
1252 // For the cases where we need an intermediate instruction/temporary register
1253 // (destination is an AGPR), we need a scavenger.
1254 //
1255 // FIXME: The pass should maintain this for us so we don't have to re-scan the
1256 // whole block for every handled copy.
1257 std::unique_ptr<RegScavenger> RS;
1258 if (Opcode == AMDGPU::INSTRUCTION_LIST_END)
1259 RS = std::make_unique<RegScavenger>();
1260
1261 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize: 4);
1262
1263 for (unsigned Idx{}; Idx < SubIndices.size();) {
1264 unsigned NumRegs = 1;
1265 unsigned ThisOpcode = Opcode;
1266 unsigned SubIdx =
1267 Forward ? SubIndices[Idx] : SubIndices[SubIndices.size() - Idx - 1];
1268
1269 if (WideDstRC && WideSrcRC && Idx + 1 < SubIndices.size()) {
1270 unsigned Channel = RI.getChannelFromSubReg(SubReg: SubIdx);
1271 if (!Forward)
1272 --Channel;
1273
1274 unsigned WideSubIdx = RI.getSubRegFromChannel(Channel, NumRegs: 2);
1275 Register WideDst = RI.getSubReg(Reg: DestReg, Idx: WideSubIdx);
1276 Register WideSrc = RI.getSubReg(Reg: SrcReg, Idx: WideSubIdx);
1277
1278 if (WideDst && WideSrc && WideDstRC->contains(Reg: WideDst) &&
1279 WideSrcRC->contains(Reg: WideSrc)) {
1280 SubIdx = WideSubIdx;
1281 NumRegs = 2;
1282 ThisOpcode = WideOpcode;
1283 }
1284 }
1285
1286 Register DestSubReg = RI.getSubReg(Reg: DestReg, Idx: SubIdx);
1287 Register SrcSubReg = RI.getSubReg(Reg: SrcReg, Idx: SubIdx);
1288 assert(DestSubReg && SrcSubReg && "Failed to find subregs!");
1289
1290 Idx += NumRegs;
1291 bool UseKill = CanKillSuperReg && Idx == SubIndices.size();
1292
1293 if (ThisOpcode == AMDGPU::INSTRUCTION_LIST_END) {
1294 Register ImpUseSuper = SrcReg;
1295 indirectCopyToAGPR(TII: *this, MBB, MI, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc: UseKill,
1296 RS&: *RS, RegsOverlap: Overlap, ImpUseSuperReg: ImpUseSuper);
1297 } else if (ThisOpcode == AMDGPU::V_PK_MOV_B32) {
1298 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: DestSubReg)
1299 .addImm(Val: SISrcMods::OP_SEL_1)
1300 .addReg(RegNo: SrcSubReg)
1301 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)
1302 .addReg(RegNo: SrcSubReg)
1303 .addImm(Val: 0) // op_sel_lo
1304 .addImm(Val: 0) // op_sel_hi
1305 .addImm(Val: 0) // neg_lo
1306 .addImm(Val: 0) // neg_hi
1307 .addImm(Val: 0) // clamp
1308 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1309 } else {
1310 MachineInstrBuilder Builder =
1311 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ThisOpcode), DestReg: DestSubReg).addReg(RegNo: SrcSubReg);
1312
1313 Builder.addReg(RegNo: SrcReg, Flags: getKillRegState(B: UseKill) | RegState::Implicit);
1314 }
1315 }
1316}
1317
1318int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
1319 int32_t NewOpc;
1320
1321 // Try to map original to commuted opcode
1322 NewOpc = AMDGPU::getCommuteRev(Opcode);
1323 if (NewOpc != -1)
1324 // Check if the commuted (REV) opcode exists on the target.
1325 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1326
1327 // Try to map commuted to original opcode
1328 NewOpc = AMDGPU::getCommuteOrig(Opcode);
1329 if (NewOpc != -1)
1330 // Check if the original (non-REV) opcode exists on the target.
1331 return pseudoToMCOpcode(Opcode: NewOpc) != -1 ? NewOpc : -1;
1332
1333 return Opcode;
1334}
1335
1336bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
1337 const Register Reg,
1338 int64_t &ImmVal) const {
1339 switch (MI.getOpcode()) {
1340 case AMDGPU::V_MOV_B32_e32:
1341 case AMDGPU::S_MOV_B32:
1342 case AMDGPU::S_MOVK_I32:
1343 case AMDGPU::S_MOV_B64:
1344 case AMDGPU::V_MOV_B64_e32:
1345 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
1346 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
1347 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
1348 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
1349 case AMDGPU::V_MOV_B64_PSEUDO:
1350 case AMDGPU::V_MOV_B16_t16_e32: {
1351 const MachineOperand &Src0 = MI.getOperand(i: 1);
1352 if (Src0.isImm()) {
1353 ImmVal = Src0.getImm();
1354 return MI.getOperand(i: 0).getReg() == Reg;
1355 }
1356
1357 return false;
1358 }
1359 case AMDGPU::V_MOV_B16_t16_e64: {
1360 const MachineOperand &Src0 = MI.getOperand(i: 2);
1361 if (Src0.isImm() && !MI.getOperand(i: 1).getImm()) {
1362 ImmVal = Src0.getImm();
1363 return MI.getOperand(i: 0).getReg() == Reg;
1364 }
1365
1366 return false;
1367 }
1368 case AMDGPU::S_BREV_B32:
1369 case AMDGPU::V_BFREV_B32_e32:
1370 case AMDGPU::V_BFREV_B32_e64: {
1371 const MachineOperand &Src0 = MI.getOperand(i: 1);
1372 if (Src0.isImm()) {
1373 ImmVal = static_cast<int64_t>(reverseBits<int32_t>(Val: Src0.getImm()));
1374 return MI.getOperand(i: 0).getReg() == Reg;
1375 }
1376
1377 return false;
1378 }
1379 case AMDGPU::S_NOT_B32:
1380 case AMDGPU::V_NOT_B32_e32:
1381 case AMDGPU::V_NOT_B32_e64: {
1382 const MachineOperand &Src0 = MI.getOperand(i: 1);
1383 if (Src0.isImm()) {
1384 ImmVal = static_cast<int64_t>(~static_cast<int32_t>(Src0.getImm()));
1385 return MI.getOperand(i: 0).getReg() == Reg;
1386 }
1387
1388 return false;
1389 }
1390 default:
1391 return false;
1392 }
1393}
1394
1395std::optional<int64_t>
1396SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1397 const MachineOperand &Op,
1398 MachineInstr **DefMI) const {
1399 if (DefMI)
1400 *DefMI = nullptr;
1401
1402 if (Op.isImm())
1403 return Op.getImm();
1404
1405 if (!Op.isReg() || !Op.getReg().isVirtual())
1406 return std::nullopt;
1407 MachineInstr *Def = MRI.getUniqueVRegDef(Reg: Op.getReg());
1408 if (Def && Def->isMoveImmediate()) {
1409 const MachineOperand &ImmSrc = Def->getOperand(i: 1);
1410 if (ImmSrc.isImm()) {
1411 if (DefMI)
1412 *DefMI = Def;
1413 return extractSubregFromImm(ImmVal: ImmSrc.getImm(), SubRegIndex: Op.getSubReg());
1414 }
1415 }
1416
1417 return std::nullopt;
1418}
1419
1420std::optional<int64_t>
1421SIInstrInfo::getImmOrMaterializedImm(const MachineRegisterInfo &MRI,
1422 Register Reg, MachineInstr **DefMI) const {
1423 MachineOperand Op = MachineOperand::CreateReg(Reg, isDef: false);
1424 return getImmOrMaterializedImm(MRI, Op, DefMI);
1425}
1426
1427unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
1428
1429 if (RI.isAGPRClass(RC: DstRC))
1430 return AMDGPU::COPY;
1431 if (RI.getRegSizeInBits(RC: *DstRC) == 16) {
1432 // Assume hi bits are unneeded. Only _e64 true16 instructions are legal
1433 // before RA.
1434 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::COPY : AMDGPU::V_MOV_B16_t16_e64;
1435 }
1436 if (RI.getRegSizeInBits(RC: *DstRC) == 32)
1437 return RI.isSGPRClass(RC: DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1438 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && RI.isSGPRClass(RC: DstRC))
1439 return AMDGPU::S_MOV_B64;
1440 if (RI.getRegSizeInBits(RC: *DstRC) == 64 && !RI.isSGPRClass(RC: DstRC))
1441 return AMDGPU::V_MOV_B64_PSEUDO;
1442 return AMDGPU::COPY;
1443}
1444
1445const MCInstrDesc &
1446SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
1447 bool IsIndirectSrc) const {
1448 if (IsIndirectSrc) {
1449 if (VecSize <= 32) // 4 bytes
1450 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
1451 if (VecSize <= 64) // 8 bytes
1452 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
1453 if (VecSize <= 96) // 12 bytes
1454 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
1455 if (VecSize <= 128) // 16 bytes
1456 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
1457 if (VecSize <= 160) // 20 bytes
1458 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
1459 if (VecSize <= 192) // 24 bytes
1460 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6);
1461 if (VecSize <= 224) // 28 bytes
1462 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7);
1463 if (VecSize <= 256) // 32 bytes
1464 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
1465 if (VecSize <= 288) // 36 bytes
1466 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9);
1467 if (VecSize <= 320) // 40 bytes
1468 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10);
1469 if (VecSize <= 352) // 44 bytes
1470 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11);
1471 if (VecSize <= 384) // 48 bytes
1472 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12);
1473 if (VecSize <= 512) // 64 bytes
1474 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
1475 if (VecSize <= 1024) // 128 bytes
1476 return get(Opcode: AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
1477
1478 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
1479 }
1480
1481 if (VecSize <= 32) // 4 bytes
1482 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
1483 if (VecSize <= 64) // 8 bytes
1484 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
1485 if (VecSize <= 96) // 12 bytes
1486 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
1487 if (VecSize <= 128) // 16 bytes
1488 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
1489 if (VecSize <= 160) // 20 bytes
1490 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
1491 if (VecSize <= 192) // 24 bytes
1492 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6);
1493 if (VecSize <= 224) // 28 bytes
1494 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7);
1495 if (VecSize <= 256) // 32 bytes
1496 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
1497 if (VecSize <= 288) // 36 bytes
1498 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9);
1499 if (VecSize <= 320) // 40 bytes
1500 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10);
1501 if (VecSize <= 352) // 44 bytes
1502 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11);
1503 if (VecSize <= 384) // 48 bytes
1504 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12);
1505 if (VecSize <= 512) // 64 bytes
1506 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
1507 if (VecSize <= 1024) // 128 bytes
1508 return get(Opcode: AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
1509
1510 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
1511}
1512
1513static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
1514 if (VecSize <= 32) // 4 bytes
1515 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1516 if (VecSize <= 64) // 8 bytes
1517 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1518 if (VecSize <= 96) // 12 bytes
1519 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1520 if (VecSize <= 128) // 16 bytes
1521 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1522 if (VecSize <= 160) // 20 bytes
1523 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1524 if (VecSize <= 192) // 24 bytes
1525 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1526 if (VecSize <= 224) // 28 bytes
1527 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1528 if (VecSize <= 256) // 32 bytes
1529 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1530 if (VecSize <= 288) // 36 bytes
1531 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1532 if (VecSize <= 320) // 40 bytes
1533 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1534 if (VecSize <= 352) // 44 bytes
1535 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1536 if (VecSize <= 384) // 48 bytes
1537 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1538 if (VecSize <= 512) // 64 bytes
1539 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1540 if (VecSize <= 1024) // 128 bytes
1541 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1542
1543 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1544}
1545
1546static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
1547 if (VecSize <= 32) // 4 bytes
1548 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
1549 if (VecSize <= 64) // 8 bytes
1550 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
1551 if (VecSize <= 96) // 12 bytes
1552 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
1553 if (VecSize <= 128) // 16 bytes
1554 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
1555 if (VecSize <= 160) // 20 bytes
1556 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
1557 if (VecSize <= 192) // 24 bytes
1558 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6;
1559 if (VecSize <= 224) // 28 bytes
1560 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7;
1561 if (VecSize <= 256) // 32 bytes
1562 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
1563 if (VecSize <= 288) // 36 bytes
1564 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9;
1565 if (VecSize <= 320) // 40 bytes
1566 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10;
1567 if (VecSize <= 352) // 44 bytes
1568 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11;
1569 if (VecSize <= 384) // 48 bytes
1570 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12;
1571 if (VecSize <= 512) // 64 bytes
1572 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
1573 if (VecSize <= 1024) // 128 bytes
1574 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
1575
1576 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1577}
1578
1579static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
1580 if (VecSize <= 64) // 8 bytes
1581 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
1582 if (VecSize <= 128) // 16 bytes
1583 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
1584 if (VecSize <= 256) // 32 bytes
1585 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
1586 if (VecSize <= 512) // 64 bytes
1587 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
1588 if (VecSize <= 1024) // 128 bytes
1589 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
1590
1591 llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
1592}
1593
1594const MCInstrDesc &
1595SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
1596 bool IsSGPR) const {
1597 if (IsSGPR) {
1598 switch (EltSize) {
1599 case 32:
1600 return get(Opcode: getIndirectSGPRWriteMovRelPseudo32(VecSize));
1601 case 64:
1602 return get(Opcode: getIndirectSGPRWriteMovRelPseudo64(VecSize));
1603 default:
1604 llvm_unreachable("invalid reg indexing elt size");
1605 }
1606 }
1607
1608 assert(EltSize == 32 && "invalid reg indexing elt size");
1609 return get(Opcode: getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
1610}
1611
1612static unsigned getSGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1613 switch (Size) {
1614 case 4:
1615 return NeedsCFI ? AMDGPU::SI_SPILL_S32_CFI_SAVE : AMDGPU::SI_SPILL_S32_SAVE;
1616 case 8:
1617 return NeedsCFI ? AMDGPU::SI_SPILL_S64_CFI_SAVE : AMDGPU::SI_SPILL_S64_SAVE;
1618 case 12:
1619 return NeedsCFI ? AMDGPU::SI_SPILL_S96_CFI_SAVE : AMDGPU::SI_SPILL_S96_SAVE;
1620 case 16:
1621 return NeedsCFI ? AMDGPU::SI_SPILL_S128_CFI_SAVE
1622 : AMDGPU::SI_SPILL_S128_SAVE;
1623 case 20:
1624 return NeedsCFI ? AMDGPU::SI_SPILL_S160_CFI_SAVE
1625 : AMDGPU::SI_SPILL_S160_SAVE;
1626 case 24:
1627 return NeedsCFI ? AMDGPU::SI_SPILL_S192_CFI_SAVE
1628 : AMDGPU::SI_SPILL_S192_SAVE;
1629 case 28:
1630 return NeedsCFI ? AMDGPU::SI_SPILL_S224_CFI_SAVE
1631 : AMDGPU::SI_SPILL_S224_SAVE;
1632 case 32:
1633 return AMDGPU::SI_SPILL_S256_SAVE;
1634 case 36:
1635 return AMDGPU::SI_SPILL_S288_SAVE;
1636 case 40:
1637 return AMDGPU::SI_SPILL_S320_SAVE;
1638 case 44:
1639 return AMDGPU::SI_SPILL_S352_SAVE;
1640 case 48:
1641 return AMDGPU::SI_SPILL_S384_SAVE;
1642 case 64:
1643 return NeedsCFI ? AMDGPU::SI_SPILL_S512_CFI_SAVE
1644 : AMDGPU::SI_SPILL_S512_SAVE;
1645 case 128:
1646 return NeedsCFI ? AMDGPU::SI_SPILL_S1024_CFI_SAVE
1647 : AMDGPU::SI_SPILL_S1024_SAVE;
1648 default:
1649 llvm_unreachable("unknown register size");
1650 }
1651}
1652
1653static unsigned getVGPRSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1654 switch (Size) {
1655 case 2:
1656 return AMDGPU::SI_SPILL_V16_SAVE;
1657 case 4:
1658 return NeedsCFI ? AMDGPU::SI_SPILL_V32_CFI_SAVE : AMDGPU::SI_SPILL_V32_SAVE;
1659 case 8:
1660 return NeedsCFI ? AMDGPU::SI_SPILL_V64_CFI_SAVE : AMDGPU::SI_SPILL_V64_SAVE;
1661 case 12:
1662 return NeedsCFI ? AMDGPU::SI_SPILL_V96_CFI_SAVE : AMDGPU::SI_SPILL_V96_SAVE;
1663 case 16:
1664 return NeedsCFI ? AMDGPU::SI_SPILL_V128_CFI_SAVE
1665 : AMDGPU::SI_SPILL_V128_SAVE;
1666 case 20:
1667 return NeedsCFI ? AMDGPU::SI_SPILL_V160_CFI_SAVE
1668 : AMDGPU::SI_SPILL_V160_SAVE;
1669 case 24:
1670 return NeedsCFI ? AMDGPU::SI_SPILL_V192_CFI_SAVE
1671 : AMDGPU::SI_SPILL_V192_SAVE;
1672 case 28:
1673 return NeedsCFI ? AMDGPU::SI_SPILL_V224_CFI_SAVE
1674 : AMDGPU::SI_SPILL_V224_SAVE;
1675 case 32:
1676 return NeedsCFI ? AMDGPU::SI_SPILL_V256_CFI_SAVE
1677 : AMDGPU::SI_SPILL_V256_SAVE;
1678 case 36:
1679 return NeedsCFI ? AMDGPU::SI_SPILL_V288_CFI_SAVE
1680 : AMDGPU::SI_SPILL_V288_SAVE;
1681 case 40:
1682 return NeedsCFI ? AMDGPU::SI_SPILL_V320_CFI_SAVE
1683 : AMDGPU::SI_SPILL_V320_SAVE;
1684 case 44:
1685 return NeedsCFI ? AMDGPU::SI_SPILL_V352_CFI_SAVE
1686 : AMDGPU::SI_SPILL_V352_SAVE;
1687 case 48:
1688 return NeedsCFI ? AMDGPU::SI_SPILL_V384_CFI_SAVE
1689 : AMDGPU::SI_SPILL_V384_SAVE;
1690 case 64:
1691 return NeedsCFI ? AMDGPU::SI_SPILL_V512_CFI_SAVE
1692 : AMDGPU::SI_SPILL_V512_SAVE;
1693 case 128:
1694 return NeedsCFI ? AMDGPU::SI_SPILL_V1024_CFI_SAVE
1695 : AMDGPU::SI_SPILL_V1024_SAVE;
1696 default:
1697 llvm_unreachable("unknown register size");
1698 }
1699}
1700
1701static unsigned getAVSpillSaveOpcode(unsigned Size, bool NeedsCFI) {
1702 switch (Size) {
1703 case 4:
1704 return NeedsCFI ? AMDGPU::SI_SPILL_AV32_CFI_SAVE
1705 : AMDGPU::SI_SPILL_AV32_SAVE;
1706 case 8:
1707 return NeedsCFI ? AMDGPU::SI_SPILL_AV64_CFI_SAVE
1708 : AMDGPU::SI_SPILL_AV64_SAVE;
1709 case 12:
1710 return NeedsCFI ? AMDGPU::SI_SPILL_AV96_CFI_SAVE
1711 : AMDGPU::SI_SPILL_AV96_SAVE;
1712 case 16:
1713 return NeedsCFI ? AMDGPU::SI_SPILL_AV128_CFI_SAVE
1714 : AMDGPU::SI_SPILL_AV128_SAVE;
1715 case 20:
1716 return NeedsCFI ? AMDGPU::SI_SPILL_AV160_CFI_SAVE
1717 : AMDGPU::SI_SPILL_AV160_SAVE;
1718 case 24:
1719 return NeedsCFI ? AMDGPU::SI_SPILL_AV192_CFI_SAVE
1720 : AMDGPU::SI_SPILL_AV192_SAVE;
1721 case 28:
1722 return NeedsCFI ? AMDGPU::SI_SPILL_AV224_CFI_SAVE
1723 : AMDGPU::SI_SPILL_AV224_SAVE;
1724 case 32:
1725 return NeedsCFI ? AMDGPU::SI_SPILL_AV256_CFI_SAVE
1726 : AMDGPU::SI_SPILL_AV256_SAVE;
1727 case 36:
1728 return AMDGPU::SI_SPILL_AV288_SAVE;
1729 case 40:
1730 return AMDGPU::SI_SPILL_AV320_SAVE;
1731 case 44:
1732 return AMDGPU::SI_SPILL_AV352_SAVE;
1733 case 48:
1734 return AMDGPU::SI_SPILL_AV384_SAVE;
1735 case 64:
1736 return NeedsCFI ? AMDGPU::SI_SPILL_AV512_CFI_SAVE
1737 : AMDGPU::SI_SPILL_AV512_SAVE;
1738 case 128:
1739 return NeedsCFI ? AMDGPU::SI_SPILL_AV1024_CFI_SAVE
1740 : AMDGPU::SI_SPILL_AV1024_SAVE;
1741 default:
1742 llvm_unreachable("unknown register size");
1743 }
1744}
1745
1746static unsigned getWWMRegSpillSaveOpcode(unsigned Size,
1747 bool IsVectorSuperClass) {
1748 // Currently, there is only 32-bit WWM register spills needed.
1749 if (Size != 4)
1750 llvm_unreachable("unknown wwm register spill size");
1751
1752 if (IsVectorSuperClass)
1753 return AMDGPU::SI_SPILL_WWM_AV32_SAVE;
1754
1755 return AMDGPU::SI_SPILL_WWM_V32_SAVE;
1756}
1757
1758unsigned SIInstrInfo::getVectorRegSpillSaveOpcode(
1759 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1760 const SIMachineFunctionInfo &MFI, bool NeedsCFI) const {
1761 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1762
1763 // Choose the right opcode if spilling a WWM register.
1764 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1765 return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass);
1766
1767 // TODO: Check if AGPRs are available
1768 if (ST.hasMAIInsts())
1769 return getAVSpillSaveOpcode(Size, NeedsCFI);
1770
1771 return getVGPRSpillSaveOpcode(Size, NeedsCFI);
1772}
1773
1774void SIInstrInfo::storeRegToStackSlotImpl(
1775 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1776 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1777 MachineInstr::MIFlag Flags, bool NeedsCFI) const {
1778 MachineFunction *MF = MBB.getParent();
1779 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1780 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1781 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1782
1783 MachinePointerInfo PtrInfo
1784 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1785 MachineMemOperand *MMO = MF->getMachineMemOperand(
1786 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
1787 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
1788 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1789
1790 MachineRegisterInfo &MRI = MF->getRegInfo();
1791 if (RI.isSGPRClass(RC)) {
1792 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
1793 MFI->setHasSpilledSGPRs();
1794 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled");
1795 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI &&
1796 SrcReg != AMDGPU::EXEC && "exec should not be spilled");
1797
1798 // We are only allowed to create one new instruction when spilling
1799 // registers, so we need to use pseudo instruction for spilling SGPRs.
1800 const MCInstrDesc &OpDesc =
1801 get(Opcode: getSGPRSpillSaveOpcode(Size: SpillSize, NeedsCFI));
1802
1803 // The SGPR spill/restore instructions only work on number sgprs, so we need
1804 // to make sure we are using the correct register class.
1805 if (SrcReg.isVirtual() && SpillSize == 4) {
1806 MRI.constrainRegClass(Reg: SrcReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
1807 }
1808
1809 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc)
1810 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1811 .addFrameIndex(Idx: FrameIndex) // addr
1812 .addMemOperand(MMO)
1813 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
1814
1815 return;
1816 }
1817
1818 unsigned Opcode = getVectorRegSpillSaveOpcode(Reg: VReg ? VReg : SrcReg, RC,
1819 Size: SpillSize, MFI: *MFI, NeedsCFI);
1820 MFI->setHasSpilledVGPRs();
1821
1822 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode))
1823 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill)) // data
1824 .addFrameIndex(Idx: FrameIndex) // addr
1825 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
1826 .addImm(Val: 0) // offset
1827 .addMemOperand(MMO);
1828}
1829
1830void SIInstrInfo::storeRegToStackSlot(
1831 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
1832 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
1833 MachineInstr::MIFlag Flags) const {
1834 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg, Flags,
1835 NeedsCFI: false);
1836}
1837
1838void SIInstrInfo::storeRegToStackSlotCFI(MachineBasicBlock &MBB,
1839 MachineBasicBlock::iterator MI,
1840 Register SrcReg, bool isKill,
1841 int FrameIndex,
1842 const TargetRegisterClass *RC) const {
1843 storeRegToStackSlotImpl(MBB, MI, SrcReg, isKill, FrameIndex, RC, VReg: Register(),
1844 Flags: MachineInstr::NoFlags, NeedsCFI: true);
1845}
1846
1847static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1848 switch (Size) {
1849 case 4:
1850 return AMDGPU::SI_SPILL_S32_RESTORE;
1851 case 8:
1852 return AMDGPU::SI_SPILL_S64_RESTORE;
1853 case 12:
1854 return AMDGPU::SI_SPILL_S96_RESTORE;
1855 case 16:
1856 return AMDGPU::SI_SPILL_S128_RESTORE;
1857 case 20:
1858 return AMDGPU::SI_SPILL_S160_RESTORE;
1859 case 24:
1860 return AMDGPU::SI_SPILL_S192_RESTORE;
1861 case 28:
1862 return AMDGPU::SI_SPILL_S224_RESTORE;
1863 case 32:
1864 return AMDGPU::SI_SPILL_S256_RESTORE;
1865 case 36:
1866 return AMDGPU::SI_SPILL_S288_RESTORE;
1867 case 40:
1868 return AMDGPU::SI_SPILL_S320_RESTORE;
1869 case 44:
1870 return AMDGPU::SI_SPILL_S352_RESTORE;
1871 case 48:
1872 return AMDGPU::SI_SPILL_S384_RESTORE;
1873 case 64:
1874 return AMDGPU::SI_SPILL_S512_RESTORE;
1875 case 128:
1876 return AMDGPU::SI_SPILL_S1024_RESTORE;
1877 default:
1878 llvm_unreachable("unknown register size");
1879 }
1880}
1881
1882static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1883 switch (Size) {
1884 case 2:
1885 return AMDGPU::SI_SPILL_V16_RESTORE;
1886 case 4:
1887 return AMDGPU::SI_SPILL_V32_RESTORE;
1888 case 8:
1889 return AMDGPU::SI_SPILL_V64_RESTORE;
1890 case 12:
1891 return AMDGPU::SI_SPILL_V96_RESTORE;
1892 case 16:
1893 return AMDGPU::SI_SPILL_V128_RESTORE;
1894 case 20:
1895 return AMDGPU::SI_SPILL_V160_RESTORE;
1896 case 24:
1897 return AMDGPU::SI_SPILL_V192_RESTORE;
1898 case 28:
1899 return AMDGPU::SI_SPILL_V224_RESTORE;
1900 case 32:
1901 return AMDGPU::SI_SPILL_V256_RESTORE;
1902 case 36:
1903 return AMDGPU::SI_SPILL_V288_RESTORE;
1904 case 40:
1905 return AMDGPU::SI_SPILL_V320_RESTORE;
1906 case 44:
1907 return AMDGPU::SI_SPILL_V352_RESTORE;
1908 case 48:
1909 return AMDGPU::SI_SPILL_V384_RESTORE;
1910 case 64:
1911 return AMDGPU::SI_SPILL_V512_RESTORE;
1912 case 128:
1913 return AMDGPU::SI_SPILL_V1024_RESTORE;
1914 default:
1915 llvm_unreachable("unknown register size");
1916 }
1917}
1918
1919static unsigned getAVSpillRestoreOpcode(unsigned Size) {
1920 switch (Size) {
1921 case 4:
1922 return AMDGPU::SI_SPILL_AV32_RESTORE;
1923 case 8:
1924 return AMDGPU::SI_SPILL_AV64_RESTORE;
1925 case 12:
1926 return AMDGPU::SI_SPILL_AV96_RESTORE;
1927 case 16:
1928 return AMDGPU::SI_SPILL_AV128_RESTORE;
1929 case 20:
1930 return AMDGPU::SI_SPILL_AV160_RESTORE;
1931 case 24:
1932 return AMDGPU::SI_SPILL_AV192_RESTORE;
1933 case 28:
1934 return AMDGPU::SI_SPILL_AV224_RESTORE;
1935 case 32:
1936 return AMDGPU::SI_SPILL_AV256_RESTORE;
1937 case 36:
1938 return AMDGPU::SI_SPILL_AV288_RESTORE;
1939 case 40:
1940 return AMDGPU::SI_SPILL_AV320_RESTORE;
1941 case 44:
1942 return AMDGPU::SI_SPILL_AV352_RESTORE;
1943 case 48:
1944 return AMDGPU::SI_SPILL_AV384_RESTORE;
1945 case 64:
1946 return AMDGPU::SI_SPILL_AV512_RESTORE;
1947 case 128:
1948 return AMDGPU::SI_SPILL_AV1024_RESTORE;
1949 default:
1950 llvm_unreachable("unknown register size");
1951 }
1952}
1953
1954static unsigned getWWMRegSpillRestoreOpcode(unsigned Size,
1955 bool IsVectorSuperClass) {
1956 // Currently, there is only 32-bit WWM register spills needed.
1957 if (Size != 4)
1958 llvm_unreachable("unknown wwm register spill size");
1959
1960 if (IsVectorSuperClass) // TODO: Always use this if there are AGPRs
1961 return AMDGPU::SI_SPILL_WWM_AV32_RESTORE;
1962
1963 return AMDGPU::SI_SPILL_WWM_V32_RESTORE;
1964}
1965
1966unsigned SIInstrInfo::getVectorRegSpillRestoreOpcode(
1967 Register Reg, const TargetRegisterClass *RC, unsigned Size,
1968 const SIMachineFunctionInfo &MFI) const {
1969 bool IsVectorSuperClass = RI.isVectorSuperClass(RC);
1970
1971 // Choose the right opcode if restoring a WWM register.
1972 if (MFI.checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
1973 return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass);
1974
1975 // TODO: Check if AGPRs are available
1976 if (ST.hasMAIInsts())
1977 return getAVSpillRestoreOpcode(Size);
1978
1979 assert(!RI.isAGPRClass(RC));
1980 return getVGPRSpillRestoreOpcode(Size);
1981}
1982
1983void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1984 MachineBasicBlock::iterator MI,
1985 Register DestReg, int FrameIndex,
1986 const TargetRegisterClass *RC,
1987 Register VReg, unsigned SubReg,
1988 MachineInstr::MIFlag Flags) const {
1989 MachineFunction *MF = MBB.getParent();
1990 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1991 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1992 const DebugLoc &DL = MBB.findDebugLoc(MBBI: MI);
1993 unsigned SpillSize = RI.getSpillSize(RC: *RC);
1994
1995 MachinePointerInfo PtrInfo
1996 = MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
1997
1998 MachineMemOperand *MMO = MF->getMachineMemOperand(
1999 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
2000 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
2001
2002 if (RI.isSGPRClass(RC)) {
2003 if (FrameInfo.getStackID(ObjectIdx: FrameIndex) == TargetStackID::SGPRSpill)
2004 MFI->setHasSpilledSGPRs();
2005 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into");
2006 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI &&
2007 DestReg != AMDGPU::EXEC && "exec should not be spilled");
2008
2009 // FIXME: Maybe this should not include a memoperand because it will be
2010 // lowered to non-memory instructions.
2011 const MCInstrDesc &OpDesc = get(Opcode: getSGPRSpillRestoreOpcode(Size: SpillSize));
2012 if (DestReg.isVirtual() && SpillSize == 4) {
2013 MachineRegisterInfo &MRI = MF->getRegInfo();
2014 MRI.constrainRegClass(Reg: DestReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
2015 }
2016
2017 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: OpDesc, DestReg)
2018 .addFrameIndex(Idx: FrameIndex) // addr
2019 .addMemOperand(MMO)
2020 .addReg(RegNo: MFI->getStackPtrOffsetReg(), Flags: RegState::Implicit);
2021
2022 return;
2023 }
2024
2025 unsigned Opcode = getVectorRegSpillRestoreOpcode(Reg: VReg ? VReg : DestReg, RC,
2026 Size: SpillSize, MFI: *MFI);
2027 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode), DestReg)
2028 .addFrameIndex(Idx: FrameIndex) // vaddr
2029 .addReg(RegNo: MFI->getStackPtrOffsetReg()) // scratch_offset
2030 .addImm(Val: 0) // offset
2031 .addMemOperand(MMO);
2032}
2033
2034void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
2035 MachineBasicBlock::iterator MI) const {
2036 insertNoops(MBB, MI, Quantity: 1);
2037}
2038
2039void SIInstrInfo::insertNoops(MachineBasicBlock &MBB,
2040 MachineBasicBlock::iterator MI,
2041 unsigned Quantity) const {
2042 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2043 unsigned MaxSNopCount = 1u << ST.getSNopBits();
2044 while (Quantity > 0) {
2045 unsigned Arg = std::min(a: Quantity, b: MaxSNopCount);
2046 Quantity -= Arg;
2047 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOP)).addImm(Val: Arg - 1);
2048 }
2049}
2050
2051MachineBasicBlock *SIInstrInfo::insertSimulatedTrap(MachineRegisterInfo &MRI,
2052 MachineBasicBlock &MBB,
2053 MachineInstr &MI,
2054 const DebugLoc &DL) const {
2055 MachineFunction *MF = MBB.getParent();
2056 constexpr unsigned DoorbellIDMask = 0x3ff;
2057 constexpr unsigned ECQueueWaveAbort = 0x400;
2058
2059 MachineBasicBlock *TrapBB = &MBB;
2060 MachineBasicBlock *HaltLoopBB = MF->CreateMachineBasicBlock();
2061
2062 if (!MBB.succ_empty() ||
2063 next_nodbg(It: MachineBasicBlock::iterator(MI), End: MBB.end(),
2064 /*SkipPseudoOp=*/false) != MBB.end()) {
2065 MBB.splitAt(SplitInst&: MI, /*UpdateLiveIns=*/false);
2066 TrapBB = MF->CreateMachineBasicBlock();
2067 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CBRANCH_EXECNZ)).addMBB(MBB: TrapBB);
2068 MF->push_back(MBB: TrapBB);
2069 MBB.addSuccessor(Succ: TrapBB);
2070 }
2071 // Start with a `s_trap 2`, if we're in PRIV=1 and we need the workaround this
2072 // will be a nop.
2073 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_TRAP))
2074 .addImm(Val: static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
2075 Register DoorbellReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2076 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG_RTN_B32),
2077 DestReg: DoorbellReg)
2078 .addImm(Val: AMDGPU::SendMsg::ID_RTN_GET_DOORBELL);
2079 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::TTMP2)
2080 .addUse(RegNo: AMDGPU::M0);
2081 Register DoorbellRegMasked =
2082 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2083 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_AND_B32), DestReg: DoorbellRegMasked)
2084 .addUse(RegNo: DoorbellReg)
2085 .addImm(Val: DoorbellIDMask)
2086 .setOperandDead(3); // implicit-def $scc
2087 Register SetWaveAbortBit =
2088 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2089 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_OR_B32), DestReg: SetWaveAbortBit)
2090 .addUse(RegNo: DoorbellRegMasked)
2091 .addImm(Val: ECQueueWaveAbort)
2092 .setOperandDead(3); // implicit-def $scc
2093 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2094 .addUse(RegNo: SetWaveAbortBit);
2095 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SENDMSG))
2096 .addImm(Val: AMDGPU::SendMsg::ID_INTERRUPT);
2097 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2098 .addUse(RegNo: AMDGPU::TTMP2);
2099 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH)).addMBB(MBB: HaltLoopBB);
2100 TrapBB->addSuccessor(Succ: HaltLoopBB);
2101
2102 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETHALT)).addImm(Val: 5);
2103 BuildMI(BB&: *HaltLoopBB, I: HaltLoopBB->end(), MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
2104 .addMBB(MBB: HaltLoopBB);
2105 MF->push_back(MBB: HaltLoopBB);
2106 HaltLoopBB->addSuccessor(Succ: HaltLoopBB);
2107
2108 return MBB.getNextNode();
2109}
2110
2111unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
2112 switch (MI.getOpcode()) {
2113 default:
2114 if (MI.isMetaInstruction())
2115 return 0;
2116 return 1; // FIXME: Do wait states equal cycles?
2117
2118 case AMDGPU::S_NOP:
2119 return MI.getOperand(i: 0).getImm() + 1;
2120 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The
2121 // hazard, even if one exist, won't really be visible. Should we handle it?
2122 }
2123}
2124
2125bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2126 MachineBasicBlock &MBB = *MI.getParent();
2127 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2128 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
2129
2130 switch (MI.getOpcode()) {
2131 default: return TargetInstrInfo::expandPostRAPseudo(MI);
2132 case AMDGPU::S_MOV_B64_term:
2133 // This is only a terminator to get the correct spill code placement during
2134 // register allocation.
2135 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2136 break;
2137
2138 case AMDGPU::S_MOV_B32_term:
2139 // This is only a terminator to get the correct spill code placement during
2140 // register allocation.
2141 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2142 break;
2143
2144 case AMDGPU::S_XOR_B64_term:
2145 // This is only a terminator to get the correct spill code placement during
2146 // register allocation.
2147 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B64));
2148 break;
2149
2150 case AMDGPU::S_XOR_B32_term:
2151 // This is only a terminator to get the correct spill code placement during
2152 // register allocation.
2153 MI.setDesc(get(Opcode: AMDGPU::S_XOR_B32));
2154 break;
2155 case AMDGPU::S_OR_B64_term:
2156 // This is only a terminator to get the correct spill code placement during
2157 // register allocation.
2158 MI.setDesc(get(Opcode: AMDGPU::S_OR_B64));
2159 break;
2160 case AMDGPU::S_OR_B32_term:
2161 // This is only a terminator to get the correct spill code placement during
2162 // register allocation.
2163 MI.setDesc(get(Opcode: AMDGPU::S_OR_B32));
2164 break;
2165
2166 case AMDGPU::S_ANDN2_B64_term:
2167 // This is only a terminator to get the correct spill code placement during
2168 // register allocation.
2169 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B64));
2170 break;
2171
2172 case AMDGPU::S_ANDN2_B32_term:
2173 // This is only a terminator to get the correct spill code placement during
2174 // register allocation.
2175 MI.setDesc(get(Opcode: AMDGPU::S_ANDN2_B32));
2176 break;
2177
2178 case AMDGPU::S_AND_B64_term:
2179 // This is only a terminator to get the correct spill code placement during
2180 // register allocation.
2181 MI.setDesc(get(Opcode: AMDGPU::S_AND_B64));
2182 break;
2183
2184 case AMDGPU::S_AND_B32_term:
2185 // This is only a terminator to get the correct spill code placement during
2186 // register allocation.
2187 MI.setDesc(get(Opcode: AMDGPU::S_AND_B32));
2188 break;
2189
2190 case AMDGPU::S_AND_SAVEEXEC_B64_term:
2191 // This is only a terminator to get the correct spill code placement during
2192 // register allocation.
2193 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B64));
2194 break;
2195
2196 case AMDGPU::S_AND_SAVEEXEC_B32_term:
2197 // This is only a terminator to get the correct spill code placement during
2198 // register allocation.
2199 MI.setDesc(get(Opcode: AMDGPU::S_AND_SAVEEXEC_B32));
2200 break;
2201
2202 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
2203 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32));
2204 break;
2205 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
2206 MI.setDesc(get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32));
2207 break;
2208
2209 case AMDGPU::SI_SPILL_S32_TO_VGPR:
2210 MI.setDesc(get(Opcode: AMDGPU::V_WRITELANE_B32));
2211 break;
2212
2213 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
2214 MI.setDesc(get(Opcode: AMDGPU::V_READLANE_B32));
2215 break;
2216 case AMDGPU::AV_MOV_B32_IMM_PSEUDO: {
2217 Register Dst = MI.getOperand(i: 0).getReg();
2218 bool IsAGPR = SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst));
2219 MI.setDesc(
2220 get(Opcode: IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
2221 break;
2222 }
2223 case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
2224 Register Dst = MI.getOperand(i: 0).getReg();
2225 if (SIRegisterInfo::isAGPRClass(RC: RI.getPhysRegBaseClass(Reg: Dst))) {
2226 int64_t Imm = MI.getOperand(i: 1).getImm();
2227
2228 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2229 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2230 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstLo)
2231 .addImm(Val: SignExtend64<32>(x: Imm));
2232 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: DstHi)
2233 .addImm(Val: SignExtend64<32>(x: Imm >> 32));
2234 MI.eraseFromParent();
2235 break;
2236 }
2237
2238 [[fallthrough]];
2239 }
2240 case AMDGPU::V_MOV_B64_PSEUDO: {
2241 Register Dst = MI.getOperand(i: 0).getReg();
2242 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2243 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2244
2245 const MCInstrDesc &Mov64Desc = get(Opcode: AMDGPU::V_MOV_B64_e32);
2246 const TargetRegisterClass *Mov64RC = getRegClass(MCID: Mov64Desc, /*OpNum=*/0);
2247
2248 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2249 // FIXME: Will this work for 64-bit floating point immediates?
2250 assert(!SrcOp.isFPImm());
2251 if (ST.hasVMovB64Inst() && Mov64RC->contains(Reg: Dst)) {
2252 MI.setDesc(Mov64Desc);
2253 if (SrcOp.isReg() || isInlineConstant(MI, OpIdx: 1) ||
2254 (SrcOp.isImm() &&
2255 (isUInt<32>(x: SrcOp.getImm()) || ST.has64BitLiterals())) ||
2256 (SrcOp.isGlobal() && ST.has64BitLiterals()))
2257 break;
2258 }
2259 if (SrcOp.isGlobal()) {
2260 // The address is unknown until link time, so the PK_MOV inline-constant
2261 // shortcut cannot apply.
2262 const GlobalValue *GV = SrcOp.getGlobal();
2263 int64_t Offset = SrcOp.getOffset();
2264 unsigned BaseFlags, LoReloc, HiReloc;
2265 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2266 splitGlobalAddressRelocFlags(ST, SrcOp);
2267
2268 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2269 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2270 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2271 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2272 } else if (SrcOp.isImm()) {
2273 APInt Imm(64, SrcOp.getImm());
2274 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2275 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2276 const MCInstrDesc &PkMovDesc = get(Opcode: AMDGPU::V_PK_MOV_B32);
2277 const TargetRegisterClass *PkMovRC = getRegClass(MCID: PkMovDesc, /*OpNum=*/0);
2278
2279 if (ST.hasPkMovB32() && Lo == Hi && isInlineConstant(Imm: Lo) &&
2280 PkMovRC->contains(Reg: Dst)) {
2281 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: PkMovDesc, DestReg: Dst)
2282 .addImm(Val: SISrcMods::OP_SEL_1)
2283 .addImm(Val: Lo.getSExtValue())
2284 .addImm(Val: SISrcMods::OP_SEL_1)
2285 .addImm(Val: Lo.getSExtValue())
2286 .addImm(Val: 0) // op_sel_lo
2287 .addImm(Val: 0) // op_sel_hi
2288 .addImm(Val: 0) // neg_lo
2289 .addImm(Val: 0) // neg_hi
2290 .addImm(Val: 0); // clamp
2291 } else {
2292 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2293 .addImm(Val: Lo.getSExtValue());
2294 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2295 .addImm(Val: Hi.getSExtValue());
2296 }
2297 } else {
2298 assert(SrcOp.isReg());
2299 if (ST.hasPkMovB32() &&
2300 !RI.isAGPR(MRI: MBB.getParent()->getRegInfo(), Reg: SrcOp.getReg())) {
2301 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_PK_MOV_B32), DestReg: Dst)
2302 .addImm(Val: SISrcMods::OP_SEL_1) // src0_mod
2303 .addReg(RegNo: SrcOp.getReg())
2304 .addImm(Val: SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod
2305 .addReg(RegNo: SrcOp.getReg())
2306 .addImm(Val: 0) // op_sel_lo
2307 .addImm(Val: 0) // op_sel_hi
2308 .addImm(Val: 0) // neg_lo
2309 .addImm(Val: 0) // neg_hi
2310 .addImm(Val: 0); // clamp
2311 } else {
2312 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstLo)
2313 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub0));
2314 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: DstHi)
2315 .addReg(RegNo: RI.getSubReg(Reg: SrcOp.getReg(), Idx: AMDGPU::sub1));
2316 }
2317 }
2318 MI.eraseFromParent();
2319 break;
2320 }
2321 case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
2322 expandMovDPP64(MI);
2323 break;
2324 }
2325 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2326 const MachineOperand &SrcOp = MI.getOperand(i: 1);
2327 assert(!SrcOp.isFPImm());
2328
2329 if (ST.has64BitLiterals()) {
2330 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2331 break;
2332 }
2333
2334 if (SrcOp.isGlobal()) {
2335 Register Dst = MI.getOperand(i: 0).getReg();
2336 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2337 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2338 const GlobalValue *GV = SrcOp.getGlobal();
2339 int64_t Offset = SrcOp.getOffset();
2340 unsigned BaseFlags, LoReloc, HiReloc;
2341 std::tie(args&: BaseFlags, args&: LoReloc, args&: HiReloc) =
2342 splitGlobalAddressRelocFlags(ST, SrcOp);
2343
2344 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2345 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | LoReloc);
2346 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2347 .addGlobalAddress(GV, Offset, TargetFlags: BaseFlags | HiReloc);
2348 MI.eraseFromParent();
2349 break;
2350 }
2351
2352 // SrcOp is immediate
2353 APInt Imm(64, SrcOp.getImm());
2354 if (Imm.isIntN(N: 32) || isInlineConstant(Imm)) {
2355 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B64));
2356 break;
2357 }
2358
2359 Register Dst = MI.getOperand(i: 0).getReg();
2360 Register DstLo = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub0);
2361 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2362
2363 APInt Lo(32, Imm.getLoBits(numBits: 32).getZExtValue());
2364 APInt Hi(32, Imm.getHiBits(numBits: 32).getZExtValue());
2365 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstLo)
2366 .addImm(Val: Lo.getSExtValue());
2367 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi)
2368 .addImm(Val: Hi.getSExtValue());
2369 MI.eraseFromParent();
2370 break;
2371 }
2372 case AMDGPU::V_SET_INACTIVE_B32: {
2373 // Lower V_SET_INACTIVE_B32 to V_CNDMASK_B32.
2374 Register DstReg = MI.getOperand(i: 0).getReg();
2375 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2376 .add(MO: MI.getOperand(i: 3))
2377 .add(MO: MI.getOperand(i: 4))
2378 .add(MO: MI.getOperand(i: 1))
2379 .add(MO: MI.getOperand(i: 2))
2380 .add(MO: MI.getOperand(i: 5));
2381 MI.eraseFromParent();
2382 break;
2383 }
2384 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2385 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2386 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2387 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2388 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2389 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2390 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2391 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2392 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2393 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2394 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2395 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2396 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2397 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2398 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
2399 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
2400 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
2401 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
2402 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
2403 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V6:
2404 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V7:
2405 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
2406 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V9:
2407 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V10:
2408 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V11:
2409 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V12:
2410 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
2411 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
2412 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
2413 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
2414 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
2415 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
2416 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
2417 const TargetRegisterClass *EltRC = getOpRegClass(MI, OpNo: 2);
2418
2419 unsigned Opc;
2420 if (RI.hasVGPRs(RC: EltRC)) {
2421 Opc = AMDGPU::V_MOVRELD_B32_e32;
2422 } else {
2423 Opc = RI.getRegSizeInBits(RC: *EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
2424 : AMDGPU::S_MOVRELD_B32;
2425 }
2426
2427 const MCInstrDesc &OpDesc = get(Opcode: Opc);
2428 Register VecReg = MI.getOperand(i: 0).getReg();
2429 bool IsUndef = MI.getOperand(i: 1).isUndef();
2430 unsigned SubReg = MI.getOperand(i: 3).getImm();
2431 assert(VecReg == MI.getOperand(1).getReg());
2432
2433 MachineInstrBuilder MIB =
2434 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2435 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2436 .add(MO: MI.getOperand(i: 2))
2437 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2438 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2439
2440 const int ImpDefIdx =
2441 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2442 const int ImpUseIdx = ImpDefIdx + 1;
2443 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2444 MI.eraseFromParent();
2445 break;
2446 }
2447 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
2448 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
2449 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
2450 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
2451 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
2452 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V6:
2453 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V7:
2454 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
2455 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V9:
2456 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V10:
2457 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V11:
2458 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V12:
2459 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
2460 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
2461 assert(ST.useVGPRIndexMode());
2462 Register VecReg = MI.getOperand(i: 0).getReg();
2463 bool IsUndef = MI.getOperand(i: 1).isUndef();
2464 MachineOperand &Idx = MI.getOperand(i: 3);
2465 Register SubReg = MI.getOperand(i: 4).getImm();
2466
2467 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2468 .add(MO: Idx)
2469 .addImm(Val: AMDGPU::VGPRIndexMode::DST_ENABLE);
2470 SetOn->getOperand(i: 3).setIsUndef();
2471
2472 const MCInstrDesc &OpDesc = get(Opcode: AMDGPU::V_MOV_B32_indirect_write);
2473 MachineInstrBuilder MIB =
2474 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: OpDesc)
2475 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2476 .add(MO: MI.getOperand(i: 2))
2477 .addReg(RegNo: VecReg, Flags: RegState::ImplicitDefine)
2478 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2479
2480 const int ImpDefIdx =
2481 OpDesc.getNumOperands() + OpDesc.implicit_uses().size();
2482 const int ImpUseIdx = ImpDefIdx + 1;
2483 MIB->tieOperands(DefIdx: ImpDefIdx, UseIdx: ImpUseIdx);
2484
2485 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2486
2487 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2488
2489 MI.eraseFromParent();
2490 break;
2491 }
2492 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
2493 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
2494 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
2495 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
2496 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
2497 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V6:
2498 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V7:
2499 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
2500 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V9:
2501 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V10:
2502 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V11:
2503 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V12:
2504 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
2505 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
2506 assert(ST.useVGPRIndexMode());
2507 Register Dst = MI.getOperand(i: 0).getReg();
2508 Register VecReg = MI.getOperand(i: 1).getReg();
2509 bool IsUndef = MI.getOperand(i: 1).isUndef();
2510 Register SubReg = MI.getOperand(i: 3).getImm();
2511
2512 MachineInstr *SetOn = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_ON))
2513 .add(MO: MI.getOperand(i: 2))
2514 .addImm(Val: AMDGPU::VGPRIndexMode::SRC0_ENABLE);
2515 SetOn->getOperand(i: 3).setIsUndef();
2516
2517 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_indirect_read))
2518 .addDef(RegNo: Dst)
2519 .addReg(RegNo: RI.getSubReg(Reg: VecReg, Idx: SubReg), Flags: RegState::Undef)
2520 .addReg(RegNo: VecReg, Flags: RegState::Implicit | getUndefRegState(B: IsUndef));
2521
2522 MachineInstr *SetOff = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SET_GPR_IDX_OFF));
2523
2524 finalizeBundle(MBB, FirstMI: SetOn->getIterator(), LastMI: std::next(x: SetOff->getIterator()));
2525
2526 MI.eraseFromParent();
2527 break;
2528 }
2529 case AMDGPU::SI_PC_ADD_REL_OFFSET: {
2530 MachineFunction &MF = *MBB.getParent();
2531 Register Reg = MI.getOperand(i: 0).getReg();
2532 Register RegLo = RI.getSubReg(Reg, Idx: AMDGPU::sub0);
2533 Register RegHi = RI.getSubReg(Reg, Idx: AMDGPU::sub1);
2534 MachineOperand OpLo = MI.getOperand(i: 1);
2535 MachineOperand OpHi = MI.getOperand(i: 2);
2536
2537 // Create a bundle so these instructions won't be re-ordered by the
2538 // post-RA scheduler.
2539 MIBundleBuilder Bundler(MBB, MI);
2540 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2541
2542 // What we want here is an offset from the value returned by s_getpc (which
2543 // is the address of the s_add_u32 instruction) to the global variable, but
2544 // since the encoding of $symbol starts 4 bytes after the start of the
2545 // s_add_u32 instruction, we end up with an offset that is 4 bytes too
2546 // small. This requires us to add 4 to the global variable offset in order
2547 // to compute the correct address. Similarly for the s_addc_u32 instruction,
2548 // the encoding of $symbol starts 12 bytes after the start of the s_add_u32
2549 // instruction.
2550
2551 int64_t Adjust = 0;
2552 if (ST.hasGetPCZeroExtension()) {
2553 // Fix up hardware that does not sign-extend the 48-bit PC value by
2554 // inserting: s_sext_i32_i16 reghi, reghi
2555 Bundler.append(
2556 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16), DestReg: RegHi).addReg(RegNo: RegHi));
2557 Adjust += 4;
2558 }
2559
2560 if (OpLo.isGlobal())
2561 OpLo.setOffset(OpLo.getOffset() + Adjust + 4);
2562 Bundler.append(
2563 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32), DestReg: RegLo).addReg(RegNo: RegLo).add(MO: OpLo));
2564
2565 if (OpHi.isGlobal())
2566 OpHi.setOffset(OpHi.getOffset() + Adjust + 12);
2567 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32), DestReg: RegHi)
2568 .addReg(RegNo: RegHi)
2569 .add(MO: OpHi));
2570
2571 finalizeBundle(MBB, FirstMI: Bundler.begin());
2572
2573 MI.eraseFromParent();
2574 break;
2575 }
2576 case AMDGPU::SI_PC_ADD_REL_OFFSET64: {
2577 MachineFunction &MF = *MBB.getParent();
2578 Register Reg = MI.getOperand(i: 0).getReg();
2579 MachineOperand Op = MI.getOperand(i: 1);
2580
2581 // Create a bundle so these instructions won't be re-ordered by the
2582 // post-RA scheduler.
2583 MIBundleBuilder Bundler(MBB, MI);
2584 Bundler.append(MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: Reg));
2585 if (Op.isGlobal())
2586 Op.setOffset(Op.getOffset() + 4);
2587 Bundler.append(
2588 MI: BuildMI(MF, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U64), DestReg: Reg).addReg(RegNo: Reg).add(MO: Op));
2589
2590 finalizeBundle(MBB, FirstMI: Bundler.begin());
2591
2592 MI.eraseFromParent();
2593 break;
2594 }
2595 case AMDGPU::ENTER_STRICT_WWM: {
2596 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2597 // Whole Wave Mode is entered.
2598 MI.setDesc(get(Opcode: LMC.OrSaveExecOpc));
2599 break;
2600 }
2601 case AMDGPU::ENTER_STRICT_WQM: {
2602 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2603 // STRICT_WQM is entered.
2604 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: MI.getOperand(i: 0).getReg())
2605 .addReg(RegNo: LMC.ExecReg);
2606 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg).addReg(RegNo: LMC.ExecReg);
2607
2608 MI.eraseFromParent();
2609 break;
2610 }
2611 case AMDGPU::EXIT_STRICT_WWM:
2612 case AMDGPU::EXIT_STRICT_WQM: {
2613 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
2614 // WWM/STICT_WQM is exited.
2615 MI.setDesc(get(Opcode: LMC.MovOpc));
2616 break;
2617 }
2618 case AMDGPU::SI_RETURN: {
2619 const MachineFunction *MF = MBB.getParent();
2620 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2621 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2622 // Hiding the return address use with SI_RETURN may lead to extra kills in
2623 // the function and missing live-ins. We are fine in practice because callee
2624 // saved register handling ensures the register value is restored before
2625 // RET, but we need the undef flag here to appease the MachineVerifier
2626 // liveness checks.
2627 MachineInstrBuilder MIB =
2628 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64_return))
2629 .addReg(RegNo: TRI->getReturnAddressReg(MF: *MF), Flags: RegState::Undef);
2630
2631 MIB.copyImplicitOps(OtherMI: MI);
2632 MI.eraseFromParent();
2633 break;
2634 }
2635
2636 case AMDGPU::S_MUL_U64_U32_PSEUDO:
2637 case AMDGPU::S_MUL_I64_I32_PSEUDO:
2638 MI.setDesc(get(Opcode: AMDGPU::S_MUL_U64));
2639 break;
2640
2641 case AMDGPU::S_GETPC_B64_pseudo:
2642 MI.setDesc(get(Opcode: AMDGPU::S_GETPC_B64));
2643 if (ST.hasGetPCZeroExtension()) {
2644 Register Dst = MI.getOperand(i: 0).getReg();
2645 Register DstHi = RI.getSubReg(Reg: Dst, Idx: AMDGPU::sub1);
2646 // Fix up hardware that does not sign-extend the 48-bit PC value by
2647 // inserting: s_sext_i32_i16 dsthi, dsthi
2648 BuildMI(BB&: MBB, I: std::next(x: MI.getIterator()), MIMD: DL, MCID: get(Opcode: AMDGPU::S_SEXT_I32_I16),
2649 DestReg: DstHi)
2650 .addReg(RegNo: DstHi);
2651 }
2652 break;
2653
2654 case AMDGPU::V_MAX_BF16_PSEUDO_e64: {
2655 assert(ST.hasBF16PackedInsts());
2656 MI.setDesc(get(Opcode: AMDGPU::V_PK_MAX_NUM_BF16));
2657 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // op_sel
2658 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_lo
2659 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // neg_hi
2660 auto Op0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2661 Op0->setImm(Op0->getImm() | SISrcMods::OP_SEL_1);
2662 auto Op1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2663 Op1->setImm(Op1->getImm() | SISrcMods::OP_SEL_1);
2664 break;
2665 }
2666
2667 case AMDGPU::GET_STACK_BASE:
2668 // The stack starts at offset 0 unless we need to reserve some space at the
2669 // bottom.
2670 if (ST.getFrameLowering()->mayReserveScratchForCWSR(MF: *MBB.getParent())) {
2671 // When CWSR is used in dynamic VGPR mode, the trap handler needs to save
2672 // some of the VGPRs. The size of the required scratch space has already
2673 // been computed by prolog epilog insertion.
2674 const SIMachineFunctionInfo *MFI =
2675 MBB.getParent()->getInfo<SIMachineFunctionInfo>();
2676 unsigned VGPRSize = MFI->getScratchReservedForDynamicVGPRs();
2677 Register DestReg = MI.getOperand(i: 0).getReg();
2678 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETREG_B32), DestReg)
2679 .addImm(Val: AMDGPU::Hwreg::HwregEncoding::encode(
2680 Values: AMDGPU::Hwreg::ID_HW_ID2, Values: AMDGPU::Hwreg::OFFSET_ME_ID, Values: 2));
2681 // The MicroEngine ID is 0 for the graphics queue, and 1 or 2 for compute
2682 // (3 is unused, so we ignore it). Unfortunately, S_GETREG doesn't set
2683 // SCC, so we need to check for 0 manually.
2684 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CMP_LG_U32)).addImm(Val: 0).addReg(RegNo: DestReg);
2685 // Change the implicif-def of SCC to an explicit use (but first remove
2686 // the dead flag if present).
2687 MI.getOperand(i: MI.getNumExplicitOperands()).setIsDead(false);
2688 MI.getOperand(i: MI.getNumExplicitOperands()).setIsUse();
2689 MI.setDesc(get(Opcode: AMDGPU::S_CMOVK_I32));
2690 MI.addOperand(Op: MachineOperand::CreateImm(Val: VGPRSize));
2691 } else {
2692 MI.setDesc(get(Opcode: AMDGPU::S_MOV_B32));
2693 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
2694 MI.removeOperand(
2695 OpNo: MI.getNumExplicitOperands()); // Drop implicit def of SCC.
2696 }
2697 break;
2698 }
2699
2700 return true;
2701}
2702
2703void SIInstrInfo::reMaterialize(MachineBasicBlock &MBB,
2704 MachineBasicBlock::iterator I, Register DestReg,
2705 unsigned SubIdx, const MachineInstr &Orig,
2706 LaneBitmask UsedLanes) const {
2707
2708 // Try shrinking the instruction to remat only the part needed for current
2709 // context.
2710 // TODO: Handle more cases.
2711 unsigned Opcode = Orig.getOpcode();
2712 switch (Opcode) {
2713 case AMDGPU::S_MOV_B64:
2714 case AMDGPU::S_MOV_B64_IMM_PSEUDO: {
2715 if (SubIdx != 0)
2716 break;
2717
2718 if (!Orig.getOperand(i: 1).isImm())
2719 break;
2720
2721 // Shrink S_MOV_B64 to S_MOV_B32 when UsedLanes indicates only a single
2722 // 32-bit lane of the 64-bit value is live at the rematerialization point.
2723 if (UsedLanes.all())
2724 break;
2725
2726 // Determine which half of the 64-bit immediate corresponds to the use.
2727 unsigned OrigSubReg = Orig.getOperand(i: 0).getSubReg();
2728 unsigned LoSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub0);
2729 unsigned HiSubReg = RI.composeSubRegIndices(a: OrigSubReg, b: AMDGPU::sub1);
2730
2731 bool NeedLo = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: LoSubReg)).any();
2732 bool NeedHi = (UsedLanes & RI.getSubRegIndexLaneMask(SubIdx: HiSubReg)).any();
2733
2734 if (NeedLo && NeedHi)
2735 break;
2736
2737 int64_t Imm64 = Orig.getOperand(i: 1).getImm();
2738 int32_t Imm32 = NeedLo ? Lo_32(Value: Imm64) : Hi_32(Value: Imm64);
2739
2740 unsigned UseSubReg = NeedLo ? LoSubReg : HiSubReg;
2741
2742 // Emit S_MOV_B32 defining just the needed 32-bit subreg of DestReg.
2743 BuildMI(BB&: MBB, I, MIMD: Orig.getDebugLoc(), MCID: get(Opcode: AMDGPU::S_MOV_B32))
2744 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Undef, SubReg: UseSubReg)
2745 .addImm(Val: Imm32);
2746 return;
2747 }
2748
2749 case AMDGPU::S_LOAD_DWORDX16_IMM:
2750 case AMDGPU::S_LOAD_DWORDX8_IMM: {
2751 if (SubIdx != 0)
2752 break;
2753
2754 if (I == MBB.end())
2755 break;
2756
2757 if (I->isBundled())
2758 break;
2759
2760 // Look for a single use of the register that is also a subreg.
2761 Register RegToFind = Orig.getOperand(i: 0).getReg();
2762 MachineOperand *UseMO = nullptr;
2763 for (auto &CandMO : I->operands()) {
2764 if (!CandMO.isReg() || CandMO.getReg() != RegToFind || CandMO.isDef())
2765 continue;
2766 if (UseMO) {
2767 UseMO = nullptr;
2768 break;
2769 }
2770 UseMO = &CandMO;
2771 }
2772 if (!UseMO || UseMO->getSubReg() == AMDGPU::NoSubRegister)
2773 break;
2774
2775 unsigned Offset = RI.getSubRegIdxOffset(Idx: UseMO->getSubReg());
2776 unsigned SubregSize = RI.getSubRegIdxSize(Idx: UseMO->getSubReg());
2777
2778 MachineFunction *MF = MBB.getParent();
2779 MachineRegisterInfo &MRI = MF->getRegInfo();
2780 assert(MRI.use_nodbg_empty(DestReg) && "DestReg should have no users yet.");
2781
2782 unsigned NewOpcode = -1;
2783 if (SubregSize == 256)
2784 NewOpcode = AMDGPU::S_LOAD_DWORDX8_IMM;
2785 else if (SubregSize == 128)
2786 NewOpcode = AMDGPU::S_LOAD_DWORDX4_IMM;
2787 else
2788 break;
2789
2790 const MCInstrDesc &TID = get(Opcode: NewOpcode);
2791 const TargetRegisterClass *NewRC =
2792 RI.getAllocatableClass(RC: getRegClass(MCID: TID, OpNum: 0));
2793 MRI.setRegClass(Reg: DestReg, RC: NewRC);
2794
2795 UseMO->setReg(DestReg);
2796 UseMO->setSubReg(AMDGPU::NoSubRegister);
2797
2798 // Use a smaller load with the desired size, possibly with updated offset.
2799 MachineInstr *MI = MF->CloneMachineInstr(Orig: &Orig);
2800 MI->setDesc(TID);
2801 MI->getOperand(i: 0).setReg(DestReg);
2802 MI->getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
2803 if (Offset) {
2804 MachineOperand *OffsetMO = getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::offset);
2805 int64_t FinalOffset = OffsetMO->getImm() + Offset / 8;
2806 OffsetMO->setImm(FinalOffset);
2807 }
2808 SmallVector<MachineMemOperand *> NewMMOs;
2809 for (const MachineMemOperand *MemOp : Orig.memoperands())
2810 NewMMOs.push_back(Elt: MF->getMachineMemOperand(MMO: MemOp, PtrInfo: MemOp->getPointerInfo(),
2811 Size: SubregSize / 8));
2812 MI->setMemRefs(MF&: *MF, MemRefs: NewMMOs);
2813
2814 MBB.insert(I, MI);
2815 return;
2816 }
2817
2818 default:
2819 break;
2820 }
2821
2822 TargetInstrInfo::reMaterialize(MBB, MI: I, DestReg, SubIdx, Orig, UsedLanes);
2823}
2824
2825std::pair<MachineInstr*, MachineInstr*>
2826SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
2827 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
2828
2829 if (ST.hasVMovB64Inst() && ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
2830 AMDGPU::isLegalDPALU_DPPControl(
2831 ST, DC: getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl)->getImm())) {
2832 MI.setDesc(get(Opcode: AMDGPU::V_MOV_B64_dpp));
2833 return std::pair(&MI, nullptr);
2834 }
2835
2836 MachineBasicBlock &MBB = *MI.getParent();
2837 DebugLoc DL = MBB.findDebugLoc(MBBI: MI);
2838 MachineFunction *MF = MBB.getParent();
2839 MachineRegisterInfo &MRI = MF->getRegInfo();
2840 Register Dst = MI.getOperand(i: 0).getReg();
2841 unsigned Part = 0;
2842 MachineInstr *Split[2];
2843
2844 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
2845 auto MovDPP = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_dpp));
2846 if (Dst.isPhysical()) {
2847 MovDPP.addDef(RegNo: RI.getSubReg(Reg: Dst, Idx: Sub));
2848 } else {
2849 assert(MRI.isSSA());
2850 auto Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
2851 MovDPP.addDef(RegNo: Tmp);
2852 }
2853
2854 for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
2855 const MachineOperand &SrcOp = MI.getOperand(i: I);
2856 assert(!SrcOp.isFPImm());
2857 if (SrcOp.isImm()) {
2858 APInt Imm(64, SrcOp.getImm());
2859 Imm.ashrInPlace(ShiftAmt: Part * 32);
2860 MovDPP.addImm(Val: Imm.getLoBits(numBits: 32).getZExtValue());
2861 } else {
2862 assert(SrcOp.isReg());
2863 Register Src = SrcOp.getReg();
2864 if (Src.isPhysical())
2865 MovDPP.addReg(RegNo: RI.getSubReg(Reg: Src, Idx: Sub));
2866 else
2867 MovDPP.addReg(RegNo: Src, Flags: getUndefRegState(B: SrcOp.isUndef()), SubReg: Sub);
2868 }
2869 }
2870
2871 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI.explicit_operands(), N: 3))
2872 MovDPP.addImm(Val: MO.getImm());
2873
2874 Split[Part] = MovDPP;
2875 ++Part;
2876 }
2877
2878 if (Dst.isVirtual())
2879 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2880 .addReg(RegNo: Split[0]->getOperand(i: 0).getReg())
2881 .addImm(Val: AMDGPU::sub0)
2882 .addReg(RegNo: Split[1]->getOperand(i: 0).getReg())
2883 .addImm(Val: AMDGPU::sub1);
2884
2885 MI.eraseFromParent();
2886 return std::pair(Split[0], Split[1]);
2887}
2888
2889std::optional<DestSourcePair>
2890SIInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2891 if (MI.getOpcode() == AMDGPU::WWM_COPY)
2892 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2893
2894 return std::nullopt;
2895}
2896
2897bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0,
2898 AMDGPU::OpName Src0OpName,
2899 MachineOperand &Src1,
2900 AMDGPU::OpName Src1OpName) const {
2901 MachineOperand *Src0Mods = getNamedOperand(MI, OperandName: Src0OpName);
2902 if (!Src0Mods)
2903 return false;
2904
2905 MachineOperand *Src1Mods = getNamedOperand(MI, OperandName: Src1OpName);
2906 assert(Src1Mods &&
2907 "All commutable instructions have both src0 and src1 modifiers");
2908
2909 int Src0ModsVal = Src0Mods->getImm();
2910 int Src1ModsVal = Src1Mods->getImm();
2911
2912 Src1Mods->setImm(Src0ModsVal);
2913 Src0Mods->setImm(Src1ModsVal);
2914 return true;
2915}
2916
2917static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
2918 MachineOperand &RegOp,
2919 MachineOperand &NonRegOp) {
2920 Register Reg = RegOp.getReg();
2921 unsigned SubReg = RegOp.getSubReg();
2922 bool IsKill = RegOp.isKill();
2923 bool IsDead = RegOp.isDead();
2924 bool IsUndef = RegOp.isUndef();
2925 bool IsDebug = RegOp.isDebug();
2926
2927 if (NonRegOp.isImm())
2928 RegOp.ChangeToImmediate(ImmVal: NonRegOp.getImm());
2929 else if (NonRegOp.isFI())
2930 RegOp.ChangeToFrameIndex(Idx: NonRegOp.getIndex());
2931 else if (NonRegOp.isGlobal()) {
2932 RegOp.ChangeToGA(GV: NonRegOp.getGlobal(), Offset: NonRegOp.getOffset(),
2933 TargetFlags: NonRegOp.getTargetFlags());
2934 } else
2935 return nullptr;
2936
2937 // Make sure we don't reinterpret a subreg index in the target flags.
2938 RegOp.setTargetFlags(NonRegOp.getTargetFlags());
2939
2940 NonRegOp.ChangeToRegister(Reg, isDef: false, isImp: false, isKill: IsKill, isDead: IsDead, isUndef: IsUndef, isDebug: IsDebug);
2941 NonRegOp.setSubReg(SubReg);
2942
2943 return &MI;
2944}
2945
2946static MachineInstr *swapImmOperands(MachineInstr &MI,
2947 MachineOperand &NonRegOp1,
2948 MachineOperand &NonRegOp2) {
2949 unsigned TargetFlags = NonRegOp1.getTargetFlags();
2950 int64_t NonRegVal = NonRegOp1.getImm();
2951
2952 NonRegOp1.setImm(NonRegOp2.getImm());
2953 NonRegOp2.setImm(NonRegVal);
2954 NonRegOp1.setTargetFlags(NonRegOp2.getTargetFlags());
2955 NonRegOp2.setTargetFlags(TargetFlags);
2956 return &MI;
2957}
2958
2959bool SIInstrInfo::isLegalToSwap(const MachineInstr &MI, unsigned OpIdx0,
2960 unsigned OpIdx1) const {
2961 const MCInstrDesc &InstDesc = MI.getDesc();
2962 const MCOperandInfo &OpInfo0 = InstDesc.operands()[OpIdx0];
2963 const MCOperandInfo &OpInfo1 = InstDesc.operands()[OpIdx1];
2964
2965 unsigned Opc = MI.getOpcode();
2966 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
2967
2968 const MachineOperand &MO0 = MI.getOperand(i: OpIdx0);
2969 const MachineOperand &MO1 = MI.getOperand(i: OpIdx1);
2970
2971 // Swap doesn't breach constant bus or literal limits, but it may move a
2972 // constant out of src0. Allow that for immediates where the destination
2973 // operand can hold them, e.g. VOP3 sources on targets with VOP3 literal
2974 // support. Other non-register operands stay in src0.
2975 if (isVALU(MI, /*AllowLDSDMA=*/false)) {
2976 if ((int)OpIdx0 == Src0Idx && !MO0.isReg() &&
2977 (!MO0.isImm() || !isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0)))
2978 return false;
2979 if ((int)OpIdx1 == Src0Idx && !MO1.isReg() &&
2980 (!MO1.isImm() || !isImmOperandLegal(MI, OpNo: OpIdx0, MO: MO1)))
2981 return false;
2982 }
2983
2984 if ((int)OpIdx1 != Src0Idx && MO0.isReg()) {
2985 if (OpInfo1.RegClass == -1)
2986 return OpInfo1.OperandType == MCOI::OPERAND_UNKNOWN;
2987 return isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0) &&
2988 (!MO1.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1));
2989 }
2990 if ((int)OpIdx0 != Src0Idx && MO1.isReg()) {
2991 if (OpInfo0.RegClass == -1)
2992 return OpInfo0.OperandType == MCOI::OPERAND_UNKNOWN;
2993 return (!MO0.isReg() || isLegalRegOperand(MI, OpIdx: OpIdx1, MO: MO0)) &&
2994 isLegalRegOperand(MI, OpIdx: OpIdx0, MO: MO1);
2995 }
2996
2997 // No need to check 64-bit literals since swapping does not bring new
2998 // 64-bit literals into current instruction to fold to 32-bit
2999
3000 return isImmOperandLegal(MI, OpNo: OpIdx1, MO: MO0);
3001}
3002
3003bool SIInstrInfo::isNonCommutableDPP(const MachineInstr &MI) const {
3004 if (!isDPP(MI))
3005 return false;
3006 const MachineOperand *DppCtrl = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
3007 return !DppCtrl || DppCtrl->getImm() != AMDGPU::DPP::QUAD_PERM_ID;
3008}
3009
3010MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
3011 unsigned Src0Idx,
3012 unsigned Src1Idx) const {
3013 assert(!NewMI && "this should never be used");
3014
3015 if (isNonCommutableDPP(MI))
3016 return nullptr;
3017
3018 unsigned Opc = MI.getOpcode();
3019 int CommutedOpcode = commuteOpcode(Opcode: Opc);
3020 if (CommutedOpcode == -1)
3021 return nullptr;
3022
3023 if (Src0Idx > Src1Idx)
3024 std::swap(a&: Src0Idx, b&: Src1Idx);
3025
3026 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
3027 static_cast<int>(Src0Idx) &&
3028 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
3029 static_cast<int>(Src1Idx) &&
3030 "inconsistency with findCommutedOpIndices");
3031
3032 if (!isLegalToSwap(MI, OpIdx0: Src0Idx, OpIdx1: Src1Idx))
3033 return nullptr;
3034
3035 MachineInstr *CommutedMI = nullptr;
3036 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
3037 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
3038 if (Src0.isReg() && Src1.isReg()) {
3039 // Be sure to copy the source modifiers to the right place.
3040 CommutedMI =
3041 TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1: Src0Idx, OpIdx2: Src1Idx);
3042 } else if (Src0.isReg() && !Src1.isReg()) {
3043 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src0, NonRegOp&: Src1);
3044 } else if (!Src0.isReg() && Src1.isReg()) {
3045 CommutedMI = swapRegAndNonRegOperand(MI, RegOp&: Src1, NonRegOp&: Src0);
3046 } else if (Src0.isImm() && Src1.isImm()) {
3047 CommutedMI = swapImmOperands(MI, NonRegOp1&: Src0, NonRegOp2&: Src1);
3048 } else {
3049 // FIXME: Found two non registers to commute. This does happen.
3050 return nullptr;
3051 }
3052
3053 if (CommutedMI) {
3054 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_modifiers,
3055 Src1, Src1OpName: AMDGPU::OpName::src1_modifiers);
3056
3057 swapSourceModifiers(MI, Src0, Src0OpName: AMDGPU::OpName::src0_sel, Src1,
3058 Src1OpName: AMDGPU::OpName::src1_sel);
3059
3060 CommutedMI->setDesc(get(Opcode: CommutedOpcode));
3061 }
3062
3063 return CommutedMI;
3064}
3065
3066// This needs to be implemented because the source modifiers may be inserted
3067// between the true commutable operands, and the base
3068// TargetInstrInfo::commuteInstruction uses it.
3069bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
3070 unsigned &SrcOpIdx0,
3071 unsigned &SrcOpIdx1) const {
3072 if (isNonCommutableDPP(MI))
3073 return false;
3074
3075 return findCommutedOpIndices(Desc: MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
3076}
3077
3078bool SIInstrInfo::findCommutedOpIndices(const MCInstrDesc &Desc,
3079 unsigned &SrcOpIdx0,
3080 unsigned &SrcOpIdx1) const {
3081 if (!Desc.isCommutable())
3082 return false;
3083
3084 unsigned Opc = Desc.getOpcode();
3085 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
3086 if (Src0Idx == -1)
3087 return false;
3088
3089 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
3090 if (Src1Idx == -1)
3091 return false;
3092
3093 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx0, ResultIdx2&: SrcOpIdx1, CommutableOpIdx1: Src0Idx, CommutableOpIdx2: Src1Idx);
3094}
3095
3096bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
3097 int64_t BrOffset) const {
3098 // BranchRelaxation should never have to check s_setpc_b64 or s_add_pc_i64
3099 // because its dest block is unanalyzable.
3100 assert(isSOPP(BranchOp) || isSOPK(BranchOp));
3101
3102 // Convert to dwords.
3103 BrOffset /= 4;
3104
3105 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
3106 // from the next instruction.
3107 BrOffset -= 1;
3108
3109 return isIntN(N: BranchOffsetBits, x: BrOffset);
3110}
3111
3112MachineBasicBlock *
3113SIInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
3114 return MI.getOperand(i: 0).getMBB();
3115}
3116
3117bool SIInstrInfo::hasDivergentBranch(const MachineBasicBlock *MBB) const {
3118 for (const MachineInstr &MI : MBB->terminators()) {
3119 if (MI.getOpcode() == AMDGPU::SI_IF || MI.getOpcode() == AMDGPU::SI_ELSE ||
3120 MI.getOpcode() == AMDGPU::SI_LOOP ||
3121 MI.getOpcode() == AMDGPU::SI_WATERFALL_LOOP)
3122 return true;
3123 }
3124 return false;
3125}
3126
3127void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
3128 MachineBasicBlock &DestBB,
3129 MachineBasicBlock &RestoreBB,
3130 const DebugLoc &DL, int64_t BrOffset,
3131 RegScavenger *RS) const {
3132 assert(MBB.empty() &&
3133 "new block should be inserted for expanding unconditional branch");
3134 assert(MBB.pred_size() == 1);
3135 assert(RestoreBB.empty() &&
3136 "restore block should be inserted for restoring clobbered registers");
3137
3138 MachineFunction *MF = MBB.getParent();
3139 MachineRegisterInfo &MRI = MF->getRegInfo();
3140 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
3141 auto I = MBB.end();
3142 auto &MCCtx = MF->getContext();
3143
3144 if (ST.useAddPC64Inst()) {
3145 MCSymbol *Offset =
3146 MCCtx.createTempSymbol(Name: "offset", /*AlwaysAddSuffix=*/true);
3147 auto AddPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_PC_I64))
3148 .addSym(Sym: Offset, TargetFlags: MO_FAR_BRANCH_OFFSET);
3149 MCSymbol *PostAddPCLabel =
3150 MCCtx.createTempSymbol(Name: "post_addpc", /*AlwaysAddSuffix=*/true);
3151 AddPC->setPostInstrSymbol(MF&: *MF, Symbol: PostAddPCLabel);
3152 auto *OffsetExpr = MCBinaryExpr::createSub(
3153 LHS: MCSymbolRefExpr::create(Symbol: DestBB.getSymbol(), Ctx&: MCCtx),
3154 RHS: MCSymbolRefExpr::create(Symbol: PostAddPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3155 Offset->setVariableValue(OffsetExpr);
3156 return;
3157 }
3158
3159 assert(RS && "RegScavenger required for long branching");
3160
3161 // FIXME: Virtual register workaround for RegScavenger not working with empty
3162 // blocks.
3163 Register PCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
3164
3165 // Note: as this is used after hazard recognizer we need to apply some hazard
3166 // workarounds directly.
3167 const bool FlushSGPRWrites = (ST.isWave64() && ST.hasVALUMaskWriteHazard()) ||
3168 ST.hasVALUReadSGPRHazard();
3169 auto ApplyHazardWorkarounds = [this, &MBB, &I, &DL, FlushSGPRWrites]() {
3170 if (FlushSGPRWrites)
3171 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3172 .addImm(Val: AMDGPU::DepCtr::encodeFieldSaSdst(SaSdst: 0, STI: ST));
3173 };
3174
3175 // We need to compute the offset relative to the instruction immediately after
3176 // s_getpc_b64. Insert pc arithmetic code before last terminator.
3177 MachineInstr *GetPC = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_GETPC_B64), DestReg: PCReg);
3178 ApplyHazardWorkarounds();
3179
3180 MCSymbol *PostGetPCLabel =
3181 MCCtx.createTempSymbol(Name: "post_getpc", /*AlwaysAddSuffix=*/true);
3182 GetPC->setPostInstrSymbol(MF&: *MF, Symbol: PostGetPCLabel);
3183
3184 MCSymbol *OffsetLo =
3185 MCCtx.createTempSymbol(Name: "offset_lo", /*AlwaysAddSuffix=*/true);
3186 MCSymbol *OffsetHi =
3187 MCCtx.createTempSymbol(Name: "offset_hi", /*AlwaysAddSuffix=*/true);
3188 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADD_U32))
3189 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub0)
3190 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub0)
3191 .addSym(Sym: OffsetLo, TargetFlags: MO_FAR_BRANCH_OFFSET);
3192 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_ADDC_U32))
3193 .addReg(RegNo: PCReg, Flags: RegState::Define, SubReg: AMDGPU::sub1)
3194 .addReg(RegNo: PCReg, Flags: {}, SubReg: AMDGPU::sub1)
3195 .addSym(Sym: OffsetHi, TargetFlags: MO_FAR_BRANCH_OFFSET);
3196 ApplyHazardWorkarounds();
3197
3198 // Insert the indirect branch after the other terminator.
3199 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_SETPC_B64))
3200 .addReg(RegNo: PCReg);
3201
3202 // If a spill is needed for the pc register pair, we need to insert a spill
3203 // restore block right before the destination block, and insert a short branch
3204 // into the old destination block's fallthrough predecessor.
3205 // e.g.:
3206 //
3207 // s_cbranch_scc0 skip_long_branch:
3208 //
3209 // long_branch_bb:
3210 // spill s[8:9]
3211 // s_getpc_b64 s[8:9]
3212 // s_add_u32 s8, s8, restore_bb
3213 // s_addc_u32 s9, s9, 0
3214 // s_setpc_b64 s[8:9]
3215 //
3216 // skip_long_branch:
3217 // foo;
3218 //
3219 // .....
3220 //
3221 // dest_bb_fallthrough_predecessor:
3222 // bar;
3223 // s_branch dest_bb
3224 //
3225 // restore_bb:
3226 // restore s[8:9]
3227 // fallthrough dest_bb
3228 ///
3229 // dest_bb:
3230 // buzz;
3231
3232 Register LongBranchReservedReg = MFI->getLongBranchReservedReg();
3233 Register Scav;
3234
3235 // If we've previously reserved a register for long branches
3236 // avoid running the scavenger and just use those registers
3237 if (LongBranchReservedReg) {
3238 RS->enterBasicBlock(MBB);
3239 Scav = LongBranchReservedReg;
3240 } else {
3241 RS->enterBasicBlockEnd(MBB);
3242 Scav = RS->scavengeRegisterBackwards(
3243 RC: AMDGPU::SReg_64RegClass, To: MachineBasicBlock::iterator(GetPC),
3244 /* RestoreAfter */ false, SPAdj: 0, /* AllowSpill */ false);
3245 }
3246 if (Scav) {
3247 RS->setRegUsed(Reg: Scav);
3248 MRI.replaceRegWith(FromReg: PCReg, ToReg: Scav);
3249 MRI.clearVirtRegs();
3250 } else {
3251 // As SGPR needs VGPR to be spilled, we reuse the slot of temporary VGPR for
3252 // SGPR spill.
3253 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3254 const SIRegisterInfo *TRI = ST.getRegisterInfo();
3255 TRI->spillEmergencySGPR(MI: GetPC, RestoreMBB&: RestoreBB, SGPR: AMDGPU::SGPR0_SGPR1, RS);
3256 MRI.replaceRegWith(FromReg: PCReg, ToReg: AMDGPU::SGPR0_SGPR1);
3257 MRI.clearVirtRegs();
3258 }
3259
3260 MCSymbol *DestLabel = Scav ? DestBB.getSymbol() : RestoreBB.getSymbol();
3261 // Now, the distance could be defined.
3262 auto *Offset = MCBinaryExpr::createSub(
3263 LHS: MCSymbolRefExpr::create(Symbol: DestLabel, Ctx&: MCCtx),
3264 RHS: MCSymbolRefExpr::create(Symbol: PostGetPCLabel, Ctx&: MCCtx), Ctx&: MCCtx);
3265 // Add offset assignments.
3266 auto *Mask = MCConstantExpr::create(Value: 0xFFFFFFFFULL, Ctx&: MCCtx);
3267 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(LHS: Offset, RHS: Mask, Ctx&: MCCtx));
3268 auto *ShAmt = MCConstantExpr::create(Value: 32, Ctx&: MCCtx);
3269 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(LHS: Offset, RHS: ShAmt, Ctx&: MCCtx));
3270}
3271
3272unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
3273 switch (Cond) {
3274 case SIInstrInfo::SCC_TRUE:
3275 return AMDGPU::S_CBRANCH_SCC1;
3276 case SIInstrInfo::SCC_FALSE:
3277 return AMDGPU::S_CBRANCH_SCC0;
3278 case SIInstrInfo::VCCNZ:
3279 return AMDGPU::S_CBRANCH_VCCNZ;
3280 case SIInstrInfo::VCCZ:
3281 return AMDGPU::S_CBRANCH_VCCZ;
3282 case SIInstrInfo::EXECNZ:
3283 return AMDGPU::S_CBRANCH_EXECNZ;
3284 case SIInstrInfo::EXECZ:
3285 return AMDGPU::S_CBRANCH_EXECZ;
3286 default:
3287 llvm_unreachable("invalid branch predicate");
3288 }
3289}
3290
3291SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
3292 switch (Opcode) {
3293 case AMDGPU::S_CBRANCH_SCC0:
3294 return SCC_FALSE;
3295 case AMDGPU::S_CBRANCH_SCC1:
3296 return SCC_TRUE;
3297 case AMDGPU::S_CBRANCH_VCCNZ:
3298 return VCCNZ;
3299 case AMDGPU::S_CBRANCH_VCCZ:
3300 return VCCZ;
3301 case AMDGPU::S_CBRANCH_EXECNZ:
3302 return EXECNZ;
3303 case AMDGPU::S_CBRANCH_EXECZ:
3304 return EXECZ;
3305 default:
3306 return INVALID_BR;
3307 }
3308}
3309
3310bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
3311 MachineBasicBlock::iterator I,
3312 MachineBasicBlock *&TBB,
3313 MachineBasicBlock *&FBB,
3314 SmallVectorImpl<MachineOperand> &Cond,
3315 bool AllowModify) const {
3316 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3317 // Unconditional Branch
3318 TBB = I->getOperand(i: 0).getMBB();
3319 return false;
3320 }
3321
3322 BranchPredicate Pred = getBranchPredicate(Opcode: I->getOpcode());
3323 if (Pred == INVALID_BR)
3324 return true;
3325
3326 MachineBasicBlock *CondBB = I->getOperand(i: 0).getMBB();
3327 Cond.push_back(Elt: MachineOperand::CreateImm(Val: Pred));
3328 Cond.push_back(Elt: I->getOperand(i: 1)); // Save the branch register.
3329
3330 ++I;
3331
3332 if (I == MBB.end()) {
3333 // Conditional branch followed by fall-through.
3334 TBB = CondBB;
3335 return false;
3336 }
3337
3338 if (I->getOpcode() == AMDGPU::S_BRANCH) {
3339 TBB = CondBB;
3340 FBB = I->getOperand(i: 0).getMBB();
3341 return false;
3342 }
3343
3344 return true;
3345}
3346
3347bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
3348 MachineBasicBlock *&FBB,
3349 SmallVectorImpl<MachineOperand> &Cond,
3350 bool AllowModify) const {
3351 MachineBasicBlock::iterator I = MBB.getFirstTerminator();
3352 auto E = MBB.end();
3353 if (I == E)
3354 return false;
3355
3356 // Skip over the instructions that are artificially terminators for special
3357 // exec management.
3358 while (I != E && !I->isBranch() && !I->isReturn()) {
3359 switch (I->getOpcode()) {
3360 case AMDGPU::S_MOV_B64_term:
3361 case AMDGPU::S_XOR_B64_term:
3362 case AMDGPU::S_OR_B64_term:
3363 case AMDGPU::S_ANDN2_B64_term:
3364 case AMDGPU::S_AND_B64_term:
3365 case AMDGPU::S_AND_SAVEEXEC_B64_term:
3366 case AMDGPU::S_MOV_B32_term:
3367 case AMDGPU::S_XOR_B32_term:
3368 case AMDGPU::S_OR_B32_term:
3369 case AMDGPU::S_ANDN2_B32_term:
3370 case AMDGPU::S_AND_B32_term:
3371 case AMDGPU::S_AND_SAVEEXEC_B32_term:
3372 case AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term:
3373 case AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term:
3374 break;
3375 case AMDGPU::SI_IF:
3376 case AMDGPU::SI_ELSE:
3377 case AMDGPU::SI_KILL_I1_TERMINATOR:
3378 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
3379 // FIXME: It's messy that these need to be considered here at all.
3380 return true;
3381 default:
3382 llvm_unreachable("unexpected non-branch terminator inst");
3383 }
3384
3385 ++I;
3386 }
3387
3388 if (I == E)
3389 return false;
3390
3391 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
3392}
3393
3394unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
3395 int *BytesRemoved) const {
3396 unsigned Count = 0;
3397 unsigned RemovedSize = 0;
3398 for (MachineInstr &MI : llvm::make_early_inc_range(Range: MBB.terminators())) {
3399 // Skip over artificial terminators when removing instructions.
3400 if (MI.isBranch() || MI.isReturn()) {
3401 RemovedSize += getInstSizeInBytes(MI);
3402 MI.eraseFromParent();
3403 ++Count;
3404 }
3405 }
3406
3407 if (BytesRemoved)
3408 *BytesRemoved = RemovedSize;
3409
3410 return Count;
3411}
3412
3413// Copy the flags onto the implicit condition register operand.
3414static void preserveCondRegFlags(MachineOperand &CondReg,
3415 const MachineOperand &OrigCond) {
3416 CondReg.setIsUndef(OrigCond.isUndef());
3417 CondReg.setIsKill(OrigCond.isKill());
3418}
3419
3420unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
3421 MachineBasicBlock *TBB,
3422 MachineBasicBlock *FBB,
3423 ArrayRef<MachineOperand> Cond,
3424 const DebugLoc &DL,
3425 int *BytesAdded) const {
3426 if (!FBB && Cond.empty()) {
3427 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3428 .addMBB(MBB: TBB);
3429 if (BytesAdded)
3430 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3431 return 1;
3432 }
3433
3434 assert(TBB && Cond[0].isImm());
3435
3436 unsigned Opcode
3437 = getBranchOpcode(Cond: static_cast<BranchPredicate>(Cond[0].getImm()));
3438
3439 if (!FBB) {
3440 MachineInstr *CondBr =
3441 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3442 .addMBB(MBB: TBB);
3443
3444 // Copy the flags onto the implicit condition register operand.
3445 preserveCondRegFlags(CondReg&: CondBr->getOperand(i: 1), OrigCond: Cond[1]);
3446 fixImplicitOperands(MI&: *CondBr);
3447
3448 if (BytesAdded)
3449 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4;
3450 return 1;
3451 }
3452
3453 assert(TBB && FBB);
3454
3455 MachineInstr *CondBr =
3456 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode))
3457 .addMBB(MBB: TBB);
3458 fixImplicitOperands(MI&: *CondBr);
3459 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AMDGPU::S_BRANCH))
3460 .addMBB(MBB: FBB);
3461
3462 MachineOperand &CondReg = CondBr->getOperand(i: 1);
3463 CondReg.setIsUndef(Cond[1].isUndef());
3464 CondReg.setIsKill(Cond[1].isKill());
3465
3466 if (BytesAdded)
3467 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8;
3468
3469 return 2;
3470}
3471
3472bool SIInstrInfo::reverseBranchCondition(
3473 SmallVectorImpl<MachineOperand> &Cond) const {
3474 if (Cond.size() != 2) {
3475 return true;
3476 }
3477
3478 if (Cond[0].isImm()) {
3479 Cond[0].setImm(-Cond[0].getImm());
3480 return false;
3481 }
3482
3483 return true;
3484}
3485
3486namespace {
3487class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
3488private:
3489 /// The compare instruction for loop control
3490 const MachineInstr *CmpInst = nullptr;
3491 /// The normalized condition used by createTripCountGreaterCondition()
3492 SmallVector<MachineOperand, 2> Cond;
3493
3494public:
3495 AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
3496 const SmallVectorImpl<MachineOperand> &Cond)
3497 : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
3498
3499 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
3500 return CmpInst && MI == CmpInst;
3501 }
3502
3503 std::optional<bool> createTripCountGreaterCondition(
3504 int TC, MachineBasicBlock &MBB,
3505 SmallVectorImpl<MachineOperand> &CondParam) override {
3506 CondParam = this->Cond;
3507 return {};
3508 }
3509
3510 void adjustTripCount(int TripCountAdjust) override {}
3511
3512 void setPreheader(MachineBasicBlock *NewPreheader) override {}
3513};
3514} // namespace
3515
3516std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
3517SIInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
3518 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
3519 SmallVector<MachineOperand, 2> Cond;
3520 // Unanalyzable terminator.
3521 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
3522 return nullptr;
3523
3524 // Infinite loops are not supported.
3525 if (TBB == LoopBB && FBB == LoopBB)
3526 return nullptr;
3527
3528 // Must be conditional branch.
3529 if (FBB == nullptr)
3530 return nullptr;
3531
3532 assert((TBB == LoopBB || FBB == LoopBB) &&
3533 "The Loop must be a single-basic-block loop");
3534
3535 // Divergent (VCC/EXEC) back-edge is not supported.
3536 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3537 if (Pred != SCC_TRUE && Pred != SCC_FALSE)
3538 return nullptr;
3539
3540 // Calls and inline assembly are not supported.
3541 for (const MachineInstr &MI : *LoopBB)
3542 if (MI.isCall() || MI.isInlineAsm())
3543 return nullptr;
3544
3545 // Normalization for createTripCountGreaterCondition(): make Cond mean
3546 // "exit the loop" so the expander emits correct prolog guard branches.
3547 if (TBB == LoopBB)
3548 reverseBranchCondition(Cond);
3549
3550 auto Instructions = make_range(
3551 x: MachineBasicBlock::reverse_iterator(LoopBB->getFirstTerminator()),
3552 y: LoopBB->rend());
3553 auto CmpI = llvm::find_if(Range&: Instructions, P: [&](const MachineInstr &MI) {
3554 return MI.modifiesRegister(Reg: Cond[1].getReg(), TRI: &RI);
3555 });
3556
3557 if (CmpI == Instructions.end() || CmpI->isPHI())
3558 return nullptr;
3559 MachineInstr *CmpInst = &*CmpI;
3560
3561 return std::make_unique<AMDGPUPipelinerLoopInfo>(args&: CmpInst, args&: Cond);
3562}
3563
3564bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
3565 ArrayRef<MachineOperand> Cond,
3566 Register DstReg, Register TrueReg,
3567 Register FalseReg,
3568 SelectExpansion &Exp) const {
3569 switch (Cond[0].getImm()) {
3570 case VCCNZ:
3571 case VCCZ: {
3572 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3573 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3574 if (MRI.getRegClass(Reg: FalseReg) != RC)
3575 return false;
3576
3577 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3578 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
3579
3580 // Limit to equal cost for branch vs. N v_cndmask_b32s.
3581 return RI.hasVGPRs(RC) && NumInsts <= 6;
3582 }
3583 case SCC_TRUE:
3584 case SCC_FALSE: {
3585 // FIXME: We could insert for VGPRs if we could replace the original compare
3586 // with a vector one.
3587 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3588 const TargetRegisterClass *RC = MRI.getRegClass(Reg: TrueReg);
3589 if (MRI.getRegClass(Reg: FalseReg) != RC)
3590 return false;
3591
3592 int NumInsts = AMDGPU::getRegBitWidth(RC: *RC) / 32;
3593
3594 // Multiples of 8 can do s_cselect_b64
3595 if (NumInsts % 2 == 0)
3596 NumInsts /= 2;
3597
3598 Exp.CondCycles = Exp.TrueCycles = Exp.FalseCycles = NumInsts; // ???
3599 return RI.isSGPRClass(RC);
3600 }
3601 default:
3602 return false;
3603 }
3604}
3605
3606void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
3607 MachineBasicBlock::iterator I, const DebugLoc &DL,
3608 Register DstReg, ArrayRef<MachineOperand> Cond,
3609 Register TrueReg, Register FalseReg) const {
3610 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
3611 if (Pred == VCCZ || Pred == SCC_FALSE) {
3612 Pred = static_cast<BranchPredicate>(-Pred);
3613 std::swap(a&: TrueReg, b&: FalseReg);
3614 }
3615
3616 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
3617 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: DstReg);
3618 unsigned DstSize = RI.getRegSizeInBits(RC: *DstRC);
3619
3620 if (DstSize == 32) {
3621 MachineInstr *Select;
3622 if (Pred == SCC_TRUE) {
3623 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: DstReg)
3624 .addReg(RegNo: TrueReg)
3625 .addReg(RegNo: FalseReg);
3626 } else {
3627 // Instruction's operands are backwards from what is expected.
3628 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e32), DestReg: DstReg)
3629 .addReg(RegNo: FalseReg)
3630 .addReg(RegNo: TrueReg);
3631 }
3632
3633 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3634 return;
3635 }
3636
3637 if (DstSize == 64 && Pred == SCC_TRUE) {
3638 MachineInstr *Select =
3639 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
3640 .addReg(RegNo: TrueReg)
3641 .addReg(RegNo: FalseReg);
3642
3643 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3644 return;
3645 }
3646
3647 static const int16_t Sub0_15[] = {
3648 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
3649 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
3650 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
3651 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
3652 };
3653
3654 static const int16_t Sub0_15_64[] = {
3655 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
3656 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
3657 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
3658 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
3659 };
3660
3661 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
3662 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
3663 const int16_t *SubIndices = Sub0_15;
3664 int NElts = DstSize / 32;
3665
3666 // 64-bit select is only available for SALU.
3667 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
3668 if (Pred == SCC_TRUE) {
3669 if (NElts % 2) {
3670 SelOp = AMDGPU::S_CSELECT_B32;
3671 EltRC = &AMDGPU::SGPR_32RegClass;
3672 } else {
3673 SelOp = AMDGPU::S_CSELECT_B64;
3674 EltRC = &AMDGPU::SGPR_64RegClass;
3675 SubIndices = Sub0_15_64;
3676 NElts /= 2;
3677 }
3678 }
3679
3680 MachineInstrBuilder MIB = BuildMI(
3681 BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
3682
3683 I = MIB->getIterator();
3684
3685 SmallVector<Register, 8> Regs;
3686 for (int Idx = 0; Idx != NElts; ++Idx) {
3687 Register DstElt = MRI.createVirtualRegister(RegClass: EltRC);
3688 Regs.push_back(Elt: DstElt);
3689
3690 unsigned SubIdx = SubIndices[Idx];
3691
3692 MachineInstr *Select;
3693 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) {
3694 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3695 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx)
3696 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx);
3697 } else {
3698 Select = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SelOp), DestReg: DstElt)
3699 .addReg(RegNo: TrueReg, Flags: {}, SubReg: SubIdx)
3700 .addReg(RegNo: FalseReg, Flags: {}, SubReg: SubIdx);
3701 }
3702
3703 preserveCondRegFlags(CondReg&: Select->getOperand(i: 3), OrigCond: Cond[1]);
3704 fixImplicitOperands(MI&: *Select);
3705
3706 MIB.addReg(RegNo: DstElt)
3707 .addImm(Val: SubIdx);
3708 }
3709}
3710
3711bool SIInstrInfo::isXcntDrain(const MachineInstr &MI) {
3712
3713 if (MI.isBranch() || MI.isCall() || MI.isReturn() || MI.isIndirectBranch())
3714 return true;
3715
3716 switch (MI.getOpcode()) {
3717 case AMDGPU::S_ENDPGM:
3718 case AMDGPU::S_ENDPGM_SAVED:
3719 case AMDGPU::S_TRAP:
3720 case AMDGPU::S_GETREG_B32:
3721 case AMDGPU::S_SETREG_B32:
3722 case AMDGPU::S_SETREG_B32_mode:
3723 case AMDGPU::S_SETREG_IMM32_B32:
3724 case AMDGPU::S_SETREG_IMM32_B32_mode:
3725 case AMDGPU::S_SENDMSG:
3726 case AMDGPU::S_SENDMSGHALT:
3727 case AMDGPU::S_SENDMSG_RTN_B32:
3728 case AMDGPU::S_SENDMSG_RTN_B64:
3729 case AMDGPU::S_BARRIER_WAIT:
3730 case AMDGPU::S_BARRIER_SIGNAL_M0:
3731 case AMDGPU::S_BARRIER_SIGNAL_IMM:
3732 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
3733 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
3734 return true;
3735 default:
3736 return false;
3737 }
3738}
3739
3740bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
3741 switch (MI.getOpcode()) {
3742 case AMDGPU::V_MOV_B16_t16_e32:
3743 case AMDGPU::V_MOV_B16_t16_e64:
3744 case AMDGPU::V_MOV_B32_e32:
3745 case AMDGPU::V_MOV_B32_e64:
3746 case AMDGPU::V_MOV_B64_PSEUDO:
3747 case AMDGPU::V_MOV_B64_e32:
3748 case AMDGPU::V_MOV_B64_e64:
3749 case AMDGPU::S_MOV_B32:
3750 case AMDGPU::S_MOV_B64:
3751 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3752 case AMDGPU::COPY:
3753 case AMDGPU::WWM_COPY:
3754 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3755 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3756 case AMDGPU::V_ACCVGPR_MOV_B32:
3757 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3758 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3759 return true;
3760 default:
3761 return false;
3762 }
3763}
3764
3765unsigned SIInstrInfo::getFoldableCopySrcIdx(const MachineInstr &MI) {
3766 switch (MI.getOpcode()) {
3767 case AMDGPU::V_MOV_B16_t16_e32:
3768 case AMDGPU::V_MOV_B16_t16_e64:
3769 return 2;
3770 case AMDGPU::V_MOV_B32_e32:
3771 case AMDGPU::V_MOV_B32_e64:
3772 case AMDGPU::V_MOV_B64_PSEUDO:
3773 case AMDGPU::V_MOV_B64_e32:
3774 case AMDGPU::V_MOV_B64_e64:
3775 case AMDGPU::S_MOV_B32:
3776 case AMDGPU::S_MOV_B64:
3777 case AMDGPU::S_MOV_B64_IMM_PSEUDO:
3778 case AMDGPU::COPY:
3779 case AMDGPU::WWM_COPY:
3780 case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
3781 case AMDGPU::V_ACCVGPR_READ_B32_e64:
3782 case AMDGPU::V_ACCVGPR_MOV_B32:
3783 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
3784 case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
3785 return 1;
3786 default:
3787 llvm_unreachable("MI is not a foldable copy");
3788 }
3789}
3790
3791static constexpr AMDGPU::OpName ModifierOpNames[] = {
3792 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
3793 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::clamp,
3794 AMDGPU::OpName::omod, AMDGPU::OpName::op_sel};
3795
3796void SIInstrInfo::removeModOperands(MachineInstr &MI) const {
3797 unsigned Opc = MI.getOpcode();
3798 for (AMDGPU::OpName Name : reverse(C: ModifierOpNames)) {
3799 int Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name);
3800 if (Idx >= 0)
3801 MI.removeOperand(OpNo: Idx);
3802 }
3803}
3804
3805void SIInstrInfo::mutateAndCleanupImplicit(MachineInstr &MI,
3806 const MCInstrDesc &NewDesc) const {
3807 MI.setDesc(NewDesc);
3808
3809 // Remove any leftover implicit operands from mutating the instruction. e.g.
3810 // if we replace an s_and_b32 with a copy, we don't need the implicit scc def
3811 // anymore.
3812 const MCInstrDesc &Desc = MI.getDesc();
3813 unsigned NumOps = Desc.getNumOperands() + Desc.implicit_uses().size() +
3814 Desc.implicit_defs().size();
3815
3816 for (unsigned I = MI.getNumOperands() - 1; I >= NumOps; --I)
3817 MI.removeOperand(OpNo: I);
3818}
3819
3820std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
3821 unsigned SubRegIndex) {
3822 switch (SubRegIndex) {
3823 case AMDGPU::NoSubRegister:
3824 return Imm;
3825 case AMDGPU::sub0:
3826 return SignExtend64<32>(x: Imm);
3827 case AMDGPU::sub1:
3828 return SignExtend64<32>(x: Imm >> 32);
3829 case AMDGPU::lo16:
3830 return SignExtend64<16>(x: Imm);
3831 case AMDGPU::hi16:
3832 return SignExtend64<16>(x: Imm >> 16);
3833 case AMDGPU::sub1_lo16:
3834 return SignExtend64<16>(x: Imm >> 32);
3835 case AMDGPU::sub1_hi16:
3836 return SignExtend64<16>(x: Imm >> 48);
3837 default:
3838 return std::nullopt;
3839 }
3840
3841 llvm_unreachable("covered subregister switch");
3842}
3843
3844static unsigned getNewFMAAKInst(const GCNSubtarget &ST, unsigned Opc) {
3845 switch (Opc) {
3846 case AMDGPU::V_MAC_F16_e32:
3847 case AMDGPU::V_MAC_F16_e64:
3848 case AMDGPU::V_MAD_F16_e64:
3849 return AMDGPU::V_MADAK_F16;
3850 case AMDGPU::V_MAC_F32_e32:
3851 case AMDGPU::V_MAC_F32_e64:
3852 case AMDGPU::V_MAD_F32_e64:
3853 return AMDGPU::V_MADAK_F32;
3854 case AMDGPU::V_FMAC_F32_e32:
3855 case AMDGPU::V_FMAC_F32_e64:
3856 case AMDGPU::V_FMA_F32_e64:
3857 return AMDGPU::V_FMAAK_F32;
3858 case AMDGPU::V_FMAC_F16_e32:
3859 case AMDGPU::V_FMAC_F16_e64:
3860 case AMDGPU::V_FMAC_F16_t16_e64:
3861 case AMDGPU::V_FMAC_F16_fake16_e64:
3862 case AMDGPU::V_FMAC_F16_t16_e32:
3863 case AMDGPU::V_FMAC_F16_fake16_e32:
3864 case AMDGPU::V_FMA_F16_e64:
3865 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3866 ? AMDGPU::V_FMAAK_F16_t16
3867 : AMDGPU::V_FMAAK_F16_fake16
3868 : AMDGPU::V_FMAAK_F16;
3869 case AMDGPU::V_FMAC_F64_e32:
3870 case AMDGPU::V_FMAC_F64_e64:
3871 case AMDGPU::V_FMA_F64_e64:
3872 return AMDGPU::V_FMAAK_F64;
3873 default:
3874 llvm_unreachable("invalid instruction");
3875 }
3876}
3877
3878static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) {
3879 switch (Opc) {
3880 case AMDGPU::V_MAC_F16_e32:
3881 case AMDGPU::V_MAC_F16_e64:
3882 case AMDGPU::V_MAD_F16_e64:
3883 return AMDGPU::V_MADMK_F16;
3884 case AMDGPU::V_MAC_F32_e32:
3885 case AMDGPU::V_MAC_F32_e64:
3886 case AMDGPU::V_MAD_F32_e64:
3887 return AMDGPU::V_MADMK_F32;
3888 case AMDGPU::V_FMAC_F32_e32:
3889 case AMDGPU::V_FMAC_F32_e64:
3890 case AMDGPU::V_FMA_F32_e64:
3891 return AMDGPU::V_FMAMK_F32;
3892 case AMDGPU::V_FMAC_F16_e32:
3893 case AMDGPU::V_FMAC_F16_e64:
3894 case AMDGPU::V_FMAC_F16_t16_e64:
3895 case AMDGPU::V_FMAC_F16_fake16_e64:
3896 case AMDGPU::V_FMAC_F16_t16_e32:
3897 case AMDGPU::V_FMAC_F16_fake16_e32:
3898 case AMDGPU::V_FMA_F16_e64:
3899 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
3900 ? AMDGPU::V_FMAMK_F16_t16
3901 : AMDGPU::V_FMAMK_F16_fake16
3902 : AMDGPU::V_FMAMK_F16;
3903 case AMDGPU::V_FMAC_F64_e32:
3904 case AMDGPU::V_FMAC_F64_e64:
3905 case AMDGPU::V_FMA_F64_e64:
3906 return AMDGPU::V_FMAMK_F64;
3907 default:
3908 llvm_unreachable("invalid instruction");
3909 }
3910}
3911
3912bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
3913 Register Reg, MachineRegisterInfo *MRI) const {
3914 int64_t Imm;
3915 if (!getConstValDefinedInReg(MI: DefMI, Reg, ImmVal&: Imm))
3916 return false;
3917
3918 const bool HasMultipleUses = !MRI->hasOneNonDBGUse(RegNo: Reg);
3919
3920 assert(!DefMI.getOperand(0).getSubReg() && "Expected SSA form");
3921
3922 unsigned Opc = UseMI.getOpcode();
3923 if (Opc == AMDGPU::COPY) {
3924 assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
3925
3926 Register DstReg = UseMI.getOperand(i: 0).getReg();
3927 Register UseSubReg = UseMI.getOperand(i: 1).getSubReg();
3928
3929 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI: *MRI, Reg: DstReg);
3930
3931 if (HasMultipleUses) {
3932 // TODO: This should fold in more cases with multiple use, but we need to
3933 // more carefully consider what those uses are.
3934 unsigned ImmDefSize = RI.getRegSizeInBits(RC: *MRI->getRegClass(Reg));
3935
3936 // Avoid breaking up a 64-bit inline immediate into a subregister extract.
3937 if (UseSubReg != AMDGPU::NoSubRegister && ImmDefSize == 64)
3938 return false;
3939
3940 // Most of the time folding a 32-bit inline constant is free (though this
3941 // might not be true if we can't later fold it into a real user).
3942 //
3943 // FIXME: This isInlineConstant check is imprecise if
3944 // getConstValDefinedInReg handled the tricky non-mov cases.
3945 if (ImmDefSize == 32 &&
3946 !isInlineConstant(ImmVal: Imm, OperandType: AMDGPU::OPERAND_REG_IMM_INT32))
3947 return false;
3948 }
3949
3950 bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
3951 RI.getSubRegIdxSize(Idx: UseSubReg) == 16;
3952
3953 if (Is16Bit) {
3954 if (RI.hasVGPRs(RC: DstRC))
3955 return false; // Do not clobber vgpr_hi16
3956
3957 if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
3958 return false;
3959 }
3960
3961 MachineFunction *MF = UseMI.getMF();
3962
3963 unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
3964 MCRegister MovDstPhysReg =
3965 DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
3966
3967 std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, SubRegIndex: UseSubReg);
3968
3969 // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
3970 for (unsigned MovOp :
3971 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
3972 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
3973 const MCInstrDesc &MovDesc = get(Opcode: MovOp);
3974
3975 const TargetRegisterClass *MovDstRC = getRegClass(MCID: MovDesc, OpNum: 0);
3976 if (Is16Bit) {
3977 // We just need to find a correctly sized register class, so the
3978 // subregister index compatibility doesn't matter since we're statically
3979 // extracting the immediate value.
3980 MovDstRC = RI.getMatchingSuperRegClass(A: MovDstRC, B: DstRC, Idx: AMDGPU::lo16);
3981 if (!MovDstRC)
3982 continue;
3983
3984 if (MovDstPhysReg) {
3985 // FIXME: We probably should not do this. If there is a live value in
3986 // the high half of the register, it will be corrupted.
3987 MovDstPhysReg =
3988 RI.getMatchingSuperReg(Reg: MovDstPhysReg, SubIdx: AMDGPU::lo16, RC: MovDstRC);
3989 if (!MovDstPhysReg)
3990 continue;
3991 }
3992 }
3993
3994 // Result class isn't the right size, try the next instruction.
3995 if (MovDstPhysReg) {
3996 if (!MovDstRC->contains(Reg: MovDstPhysReg))
3997 return false;
3998 } else if (!MRI->constrainRegClass(Reg: DstReg, RC: MovDstRC)) {
3999 // TODO: This will be overly conservative in the case of 16-bit virtual
4000 // SGPRs. We could hack up the virtual register uses to use a compatible
4001 // 32-bit class.
4002 continue;
4003 }
4004
4005 const MCOperandInfo &OpInfo = MovDesc.operands()[1];
4006
4007 // Ensure the interpreted immediate value is a valid operand in the new
4008 // mov.
4009 //
4010 // FIXME: isImmOperandLegal should have form that doesn't require existing
4011 // MachineInstr or MachineOperand
4012 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType) &&
4013 !isInlineConstant(ImmVal: *SubRegImm, OperandType: OpInfo.OperandType))
4014 break;
4015
4016 NewOpc = MovOp;
4017 break;
4018 }
4019
4020 if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
4021 return false;
4022
4023 if (Is16Bit) {
4024 UseMI.getOperand(i: 0).setSubReg(AMDGPU::NoSubRegister);
4025 if (MovDstPhysReg)
4026 UseMI.getOperand(i: 0).setReg(MovDstPhysReg);
4027 assert(UseMI.getOperand(1).getReg().isVirtual());
4028 }
4029
4030 const MCInstrDesc &NewMCID = get(Opcode: NewOpc);
4031 UseMI.setDesc(NewMCID);
4032 UseMI.getOperand(i: 1).ChangeToImmediate(ImmVal: *SubRegImm);
4033 UseMI.addImplicitDefUseOperands(MF&: *MF);
4034 return true;
4035 }
4036
4037 if (HasMultipleUses)
4038 return false;
4039
4040 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 ||
4041 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4042 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 ||
4043 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
4044 Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4045 Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 ||
4046 Opc == AMDGPU::V_FMAC_F64_e64) {
4047 // Don't fold if we are using source or output modifiers. The new VOP2
4048 // instructions don't have them.
4049 if (hasAnyModifiersSet(MI: UseMI))
4050 return false;
4051
4052 // If this is a free constant, there's no reason to do this.
4053 // TODO: We could fold this here instead of letting SIFoldOperands do it
4054 // later.
4055 int Src0Idx = getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
4056
4057 // Any src operand can be used for the legality check.
4058 if (isInlineConstant(MI: UseMI, OpIdx: Src0Idx, ImmVal: Imm))
4059 return false;
4060
4061 MachineOperand *Src0 = &UseMI.getOperand(i: Src0Idx);
4062
4063 MachineOperand *Src1 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src1);
4064 MachineOperand *Src2 = getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2);
4065
4066 auto CopyRegOperandToNarrowerRC =
4067 [MRI, this](MachineInstr &MI, unsigned OpNo,
4068 const TargetRegisterClass *NewRC) -> void {
4069 if (!MI.getOperand(i: OpNo).isReg())
4070 return;
4071 Register Reg = MI.getOperand(i: OpNo).getReg();
4072 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI: *MRI, Reg);
4073 if (RI.getCommonSubClass(A: RC, B: NewRC) != NewRC)
4074 return;
4075 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4076 BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
4077 MCID: get(Opcode: AMDGPU::COPY), DestReg: Tmp)
4078 .addReg(RegNo: Reg);
4079 MI.getOperand(i: OpNo).setReg(Tmp);
4080 MI.getOperand(i: OpNo).setIsKill();
4081 };
4082
4083 // Multiplied part is the constant: Use v_madmk_{f16, f32}.
4084 if ((Src0->isReg() && Src0->getReg() == Reg) ||
4085 (Src1->isReg() && Src1->getReg() == Reg)) {
4086 MachineOperand *RegSrc =
4087 Src1->isReg() && Src1->getReg() == Reg ? Src0 : Src1;
4088 if (!RegSrc->isReg())
4089 return false;
4090 if (RI.isSGPRClass(RC: MRI->getRegClass(Reg: RegSrc->getReg())) &&
4091 ST.getConstantBusLimit(Opcode: Opc) < 2)
4092 return false;
4093
4094 if (!Src2->isReg() || RI.isSGPRClass(RC: MRI->getRegClass(Reg: Src2->getReg())))
4095 return false;
4096
4097 // If src2 is also a literal constant then we have to choose which one to
4098 // fold. In general it is better to choose madak so that the other literal
4099 // can be materialized in an sgpr instead of a vgpr:
4100 // s_mov_b32 s0, literal
4101 // v_madak_f32 v0, s0, v0, literal
4102 // Instead of:
4103 // v_mov_b32 v1, literal
4104 // v_madmk_f32 v0, v0, literal, v1
4105 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src2->getReg());
4106 if (Def && Def->isMoveImmediate() &&
4107 !isInlineConstant(MO: Def->getOperand(i: 1)))
4108 return false;
4109
4110 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4111 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4112 return false;
4113
4114 const std::optional<int64_t> SubRegImm = extractSubregFromImm(
4115 Imm, SubRegIndex: RegSrc == Src1 ? Src0->getSubReg() : Src1->getSubReg());
4116
4117 // FIXME: This would be a lot easier if we could return a new instruction
4118 // instead of having to modify in place.
4119
4120 Register SrcReg = RegSrc->getReg();
4121 unsigned SrcSubReg = RegSrc->getSubReg();
4122 Src0->setReg(SrcReg);
4123 Src0->setSubReg(SrcSubReg);
4124 Src0->setIsKill(RegSrc->isKill());
4125
4126 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4127 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4128 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4129 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4130 UseMI.untieRegOperand(
4131 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4132
4133 Src1->ChangeToImmediate(ImmVal: *SubRegImm);
4134
4135 removeModOperands(MI&: UseMI);
4136 UseMI.setDesc(get(Opcode: NewOpc));
4137
4138 if (NewOpc == AMDGPU::V_FMAMK_F16_t16 ||
4139 NewOpc == AMDGPU::V_FMAMK_F16_fake16) {
4140 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4141 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4142 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4143 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4144 DestReg: UseMI.getOperand(i: 0).getReg())
4145 .addReg(RegNo: Tmp, Flags: RegState::Kill);
4146 UseMI.getOperand(i: 0).setReg(Tmp);
4147 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4148 CopyRegOperandToNarrowerRC(UseMI, 3, NewRC);
4149 }
4150
4151 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4152 if (DeleteDef)
4153 DefMI.eraseFromParent();
4154
4155 return true;
4156 }
4157
4158 // Added part is the constant: Use v_madak_{f16, f32}.
4159 if (Src2->isReg() && Src2->getReg() == Reg) {
4160 if (ST.getConstantBusLimit(Opcode: Opc) < 2) {
4161 // Not allowed to use constant bus for another operand.
4162 // We can however allow an inline immediate as src0.
4163 bool Src0Inlined = false;
4164 if (Src0->isReg()) {
4165 // Try to inline constant if possible.
4166 // If the Def moves immediate and the use is single
4167 // We are saving VGPR here.
4168 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src0->getReg());
4169 if (Def && Def->isMoveImmediate() &&
4170 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4171 MRI->hasOneNonDBGUse(RegNo: Src0->getReg())) {
4172 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4173 Src0Inlined = true;
4174 } else if (ST.getConstantBusLimit(Opcode: Opc) <= 1 &&
4175 RI.isSGPRReg(MRI: *MRI, Reg: Src0->getReg())) {
4176 return false;
4177 }
4178 // VGPR is okay as Src0 - fallthrough
4179 }
4180
4181 if (Src1->isReg() && !Src0Inlined) {
4182 // We have one slot for inlinable constant so far - try to fill it
4183 MachineInstr *Def = MRI->getUniqueVRegDef(Reg: Src1->getReg());
4184 if (Def && Def->isMoveImmediate() &&
4185 isInlineConstant(MO: Def->getOperand(i: 1)) &&
4186 MRI->hasOneNonDBGUse(RegNo: Src1->getReg()) && commuteInstruction(MI&: UseMI))
4187 Src0->ChangeToImmediate(ImmVal: Def->getOperand(i: 1).getImm());
4188 else if (RI.isSGPRReg(MRI: *MRI, Reg: Src1->getReg()))
4189 return false;
4190 // VGPR is okay as Src1 - fallthrough
4191 }
4192 }
4193
4194 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4195 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4196 return false;
4197
4198 // FIXME: This would be a lot easier if we could return a new instruction
4199 // instead of having to modify in place.
4200
4201 if (Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
4202 Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_t16_e64 ||
4203 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
4204 Opc == AMDGPU::V_FMAC_F16_e64 || Opc == AMDGPU::V_FMAC_F64_e64)
4205 UseMI.untieRegOperand(
4206 OpIdx: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2));
4207
4208 const std::optional<int64_t> SubRegImm =
4209 extractSubregFromImm(Imm, SubRegIndex: Src2->getSubReg());
4210
4211 // ChangingToImmediate adds Src2 back to the instruction.
4212 Src2->ChangeToImmediate(ImmVal: *SubRegImm);
4213
4214 // These come before src2.
4215 removeModOperands(MI&: UseMI);
4216 UseMI.setDesc(get(Opcode: NewOpc));
4217
4218 if (NewOpc == AMDGPU::V_FMAAK_F16_t16 ||
4219 NewOpc == AMDGPU::V_FMAAK_F16_fake16) {
4220 const TargetRegisterClass *NewRC = getRegClass(MCID: get(Opcode: NewOpc), OpNum: 0);
4221 Register Tmp = MRI->createVirtualRegister(RegClass: NewRC);
4222 BuildMI(BB&: *UseMI.getParent(), I: std::next(x: UseMI.getIterator()),
4223 MIMD: UseMI.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
4224 DestReg: UseMI.getOperand(i: 0).getReg())
4225 .addReg(RegNo: Tmp, Flags: RegState::Kill);
4226 UseMI.getOperand(i: 0).setReg(Tmp);
4227 CopyRegOperandToNarrowerRC(UseMI, 1, NewRC);
4228 CopyRegOperandToNarrowerRC(UseMI, 2, NewRC);
4229 }
4230
4231 // It might happen that UseMI was commuted
4232 // and we now have SGPR as SRC1. If so 2 inlined
4233 // constant and SGPR are illegal.
4234 legalizeOperands(MI&: UseMI);
4235
4236 int NewSrc0Idx =
4237 AMDGPU::getNamedOperandIdx(Opcode: UseMI.getOpcode(), Name: AMDGPU::OpName::src0);
4238 if (!isOperandLegal(MI: UseMI, OpIdx: NewSrc0Idx))
4239 legalizeOpWithMove(MI&: UseMI, OpIdx: NewSrc0Idx);
4240
4241 bool DeleteDef = MRI->use_nodbg_empty(RegNo: Reg);
4242 if (DeleteDef)
4243 DefMI.eraseFromParent();
4244
4245 return true;
4246 }
4247 }
4248
4249 return false;
4250}
4251
4252static bool
4253memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1,
4254 ArrayRef<const MachineOperand *> BaseOps2) {
4255 if (BaseOps1.size() != BaseOps2.size())
4256 return false;
4257 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) {
4258 if (!BaseOps1[I]->isIdenticalTo(Other: *BaseOps2[I]))
4259 return false;
4260 }
4261 return true;
4262}
4263
4264static bool offsetsDoNotOverlap(LocationSize WidthA, int OffsetA,
4265 LocationSize WidthB, int OffsetB) {
4266 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
4267 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
4268 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
4269 return LowWidth.hasValue() &&
4270 LowOffset + (int)LowWidth.getValue() <= HighOffset;
4271}
4272
4273bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
4274 const MachineInstr &MIb) const {
4275 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1;
4276 int64_t Offset0, Offset1;
4277 LocationSize Dummy0 = LocationSize::precise(Value: 0);
4278 LocationSize Dummy1 = LocationSize::precise(Value: 0);
4279 bool Offset0IsScalable, Offset1IsScalable;
4280 if (!getMemOperandsWithOffsetWidth(LdSt: MIa, BaseOps&: BaseOps0, Offset&: Offset0, OffsetIsScalable&: Offset0IsScalable,
4281 Width&: Dummy0) ||
4282 !getMemOperandsWithOffsetWidth(LdSt: MIb, BaseOps&: BaseOps1, Offset&: Offset1, OffsetIsScalable&: Offset1IsScalable,
4283 Width&: Dummy1))
4284 return false;
4285
4286 if (!memOpsHaveSameBaseOperands(BaseOps1: BaseOps0, BaseOps2: BaseOps1))
4287 return false;
4288
4289 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
4290 // FIXME: Handle ds_read2 / ds_write2.
4291 return false;
4292 }
4293 LocationSize Width0 = MIa.memoperands().front()->getSize();
4294 LocationSize Width1 = MIb.memoperands().front()->getSize();
4295 return offsetsDoNotOverlap(WidthA: Width0, OffsetA: Offset0, WidthB: Width1, OffsetB: Offset1);
4296}
4297
4298bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
4299 const MachineInstr &MIb) const {
4300 assert(MIa.mayLoadOrStore() &&
4301 "MIa must load from or modify a memory location");
4302 assert(MIb.mayLoadOrStore() &&
4303 "MIb must load from or modify a memory location");
4304
4305 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
4306 return false;
4307
4308 // XXX - Can we relax this between address spaces?
4309 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
4310 return false;
4311
4312 if (isLDSDMA(MI: MIa) || isLDSDMA(MI: MIb))
4313 return false;
4314
4315 if (MIa.isBundle() || MIb.isBundle())
4316 return false;
4317
4318 // TODO: Should we check the address space from the MachineMemOperand? That
4319 // would allow us to distinguish objects we know don't alias based on the
4320 // underlying address space, even if it was lowered to a different one,
4321 // e.g. private accesses lowered to use MUBUF instructions on a scratch
4322 // buffer.
4323 if (isDS(MI: MIa)) {
4324 if (isDS(MI: MIb))
4325 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4326
4327 return !isFLAT(MI: MIb) || isSegmentSpecificFLAT(MI: MIb);
4328 }
4329
4330 if (isMUBUF(MI: MIa) || isMTBUF(MI: MIa)) {
4331 if (isMUBUF(MI: MIb) || isMTBUF(MI: MIb))
4332 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4333
4334 if (isFLAT(MI: MIb))
4335 return isFLATScratch(MI: MIb);
4336
4337 return !isSMRD(MI: MIb);
4338 }
4339
4340 if (isSMRD(MI: MIa)) {
4341 if (isSMRD(MI: MIb))
4342 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4343
4344 if (isFLAT(MI: MIb))
4345 return isFLATScratch(MI: MIb);
4346
4347 return !isMUBUF(MI: MIb) && !isMTBUF(MI: MIb);
4348 }
4349
4350 if (isFLAT(MI: MIa)) {
4351 if (isFLAT(MI: MIb)) {
4352 if ((isFLATScratch(MI: MIa) && isFLATGlobal(MI: MIb)) ||
4353 (isFLATGlobal(MI: MIa) && isFLATScratch(MI: MIb)))
4354 return true;
4355
4356 return checkInstOffsetsDoNotOverlap(MIa, MIb);
4357 }
4358
4359 return false;
4360 }
4361
4362 return false;
4363}
4364
4365static unsigned getNewFMAInst(const GCNSubtarget &ST, unsigned Opc) {
4366 switch (Opc) {
4367 case AMDGPU::V_MAC_F16_e32:
4368 case AMDGPU::V_MAC_F16_e64:
4369 return AMDGPU::V_MAD_F16_e64;
4370 case AMDGPU::V_MAC_F32_e32:
4371 case AMDGPU::V_MAC_F32_e64:
4372 return AMDGPU::V_MAD_F32_e64;
4373 case AMDGPU::V_MAC_LEGACY_F32_e32:
4374 case AMDGPU::V_MAC_LEGACY_F32_e64:
4375 return AMDGPU::V_MAD_LEGACY_F32_e64;
4376 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4377 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4378 return AMDGPU::V_FMA_LEGACY_F32_e64;
4379 case AMDGPU::V_FMAC_F16_e32:
4380 case AMDGPU::V_FMAC_F16_e64:
4381 case AMDGPU::V_FMAC_F16_t16_e64:
4382 case AMDGPU::V_FMAC_F16_fake16_e64:
4383 return ST.hasTrue16BitInsts() ? ST.useRealTrue16Insts()
4384 ? AMDGPU::V_FMA_F16_gfx9_t16_e64
4385 : AMDGPU::V_FMA_F16_gfx9_fake16_e64
4386 : AMDGPU::V_FMA_F16_gfx9_e64;
4387 case AMDGPU::V_FMAC_F32_e32:
4388 case AMDGPU::V_FMAC_F32_e64:
4389 return AMDGPU::V_FMA_F32_e64;
4390 case AMDGPU::V_FMAC_F64_e32:
4391 case AMDGPU::V_FMAC_F64_e64:
4392 return AMDGPU::V_FMA_F64_e64;
4393 default:
4394 llvm_unreachable("invalid instruction");
4395 }
4396}
4397
4398/// Helper struct for the implementation of 3-address conversion to communicate
4399/// updates made to instruction operands.
4400struct SIInstrInfo::ThreeAddressUpdates {
4401 /// Other instruction whose def is no longer used by the converted
4402 /// instruction.
4403 MachineInstr *RemoveMIUse = nullptr;
4404};
4405
4406MachineInstr *SIInstrInfo::convertToThreeAddress(MachineInstr &MI,
4407 LiveIntervals *LIS) const {
4408 MachineBasicBlock &MBB = *MI.getParent();
4409 MachineInstr *CandidateMI = &MI;
4410
4411 if (MI.isBundle()) {
4412 // This is a temporary placeholder for bundle handling that enables us to
4413 // exercise the relevant code paths in the two-address instruction pass.
4414 if (MI.getBundleSize() != 1)
4415 return nullptr;
4416 CandidateMI = MI.getNextNode();
4417 }
4418
4419 ThreeAddressUpdates U;
4420 MachineInstr *NewMI = convertToThreeAddressImpl(MI&: *CandidateMI, Updates&: U);
4421 if (!NewMI)
4422 return nullptr;
4423
4424 if (MI.isBundle()) {
4425 CandidateMI->eraseFromBundle();
4426
4427 for (MachineOperand &MO : MI.all_defs()) {
4428 if (MO.isTied())
4429 MI.untieRegOperand(OpIdx: MO.getOperandNo());
4430 }
4431 } else {
4432 if (LIS) {
4433 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewMI);
4434 // SlotIndex of defs needs to be updated when converting to early-clobber
4435 MachineOperand &Def = NewMI->getOperand(i: 0);
4436 if (Def.isEarlyClobber() && Def.isReg() &&
4437 LIS->hasInterval(Reg: Def.getReg())) {
4438 SlotIndex OldIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: false);
4439 SlotIndex NewIndex = LIS->getInstructionIndex(Instr: *NewMI).getRegSlot(EC: true);
4440 auto &LI = LIS->getInterval(Reg: Def.getReg());
4441 auto UpdateDefIndex = [&](LiveRange &LR) {
4442 auto *S = LR.find(Pos: OldIndex);
4443 if (S != LR.end() && S->start == OldIndex) {
4444 assert(S->valno && S->valno->def == OldIndex);
4445 S->start = NewIndex;
4446 S->valno->def = NewIndex;
4447 }
4448 };
4449 UpdateDefIndex(LI);
4450 for (auto &SR : LI.subranges())
4451 UpdateDefIndex(SR);
4452 }
4453 }
4454 }
4455
4456 if (U.RemoveMIUse) {
4457 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4458 // The only user is the instruction which will be killed.
4459 Register DefReg = U.RemoveMIUse->getOperand(i: 0).getReg();
4460
4461 if (MRI.hasOneNonDBGUse(RegNo: DefReg)) {
4462 // We cannot just remove the DefMI here, calling pass will crash.
4463 U.RemoveMIUse->setDesc(get(Opcode: AMDGPU::IMPLICIT_DEF));
4464 U.RemoveMIUse->getOperand(i: 0).setIsDead(true);
4465 for (unsigned I = U.RemoveMIUse->getNumOperands() - 1; I != 0; --I)
4466 U.RemoveMIUse->removeOperand(OpNo: I);
4467 }
4468
4469 if (MI.isBundle()) {
4470 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4471 if (!VRI.Reads && !VRI.Writes) {
4472 for (MachineOperand &MO : MI.all_uses()) {
4473 if (MO.isReg() && MO.getReg() == DefReg) {
4474 assert(MO.getSubReg() == 0 &&
4475 "tied sub-registers in bundles currently not supported");
4476 MI.removeOperand(OpNo: MO.getOperandNo());
4477 break;
4478 }
4479 }
4480
4481 if (LIS)
4482 LIS->shrinkToUses(li: &LIS->getInterval(Reg: DefReg));
4483 }
4484 } else if (LIS) {
4485 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
4486
4487 // We cannot delete the original instruction here, so hack out the use
4488 // in the original instruction with a dummy register so we can use
4489 // shrinkToUses to deal with any multi-use edge cases. Other targets do
4490 // not have the complexity of deleting a use to consider here.
4491 Register DummyReg = MRI.cloneVirtualRegister(VReg: DefReg);
4492 for (MachineOperand &MIOp : MI.uses()) {
4493 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4494 MIOp.setIsUndef(true);
4495 MIOp.setReg(DummyReg);
4496 }
4497 }
4498
4499 if (MI.isBundle()) {
4500 VirtRegInfo VRI = AnalyzeVirtRegInBundle(MI, Reg: DefReg);
4501 if (!VRI.Reads && !VRI.Writes) {
4502 for (MachineOperand &MIOp : MI.uses()) {
4503 if (MIOp.isReg() && MIOp.getReg() == DefReg) {
4504 MIOp.setIsUndef(true);
4505 MIOp.setReg(DummyReg);
4506 }
4507 }
4508 }
4509
4510 MI.addOperand(Op: MachineOperand::CreateReg(Reg: DummyReg, isDef: false, isImp: false, isKill: false,
4511 isDead: false, /*isUndef=*/true));
4512 }
4513
4514 LIS->shrinkToUses(li: &DefLI);
4515 }
4516 }
4517
4518 return MI.isBundle() ? &MI : NewMI;
4519}
4520
4521MachineInstr *
4522SIInstrInfo::convertToThreeAddressImpl(MachineInstr &MI,
4523 ThreeAddressUpdates &U) const {
4524 MachineBasicBlock &MBB = *MI.getParent();
4525 unsigned Opc = MI.getOpcode();
4526
4527 // Handle MFMA.
4528 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: Opc);
4529 if (NewMFMAOpc != -1) {
4530 MachineInstrBuilder MIB =
4531 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewMFMAOpc));
4532 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4533 MIB.add(MO: MI.getOperand(i: I));
4534 return MIB;
4535 }
4536
4537 if (SIInstrInfo::isWMMA(MI)) {
4538 unsigned NewOpc = AMDGPU::mapWMMA2AddrTo3AddrOpcode(Opc: MI.getOpcode());
4539 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4540 .setMIFlags(MI.getFlags());
4541 for (unsigned I = 0, E = MI.getNumExplicitOperands(); I != E; ++I)
4542 MIB->addOperand(Op: MI.getOperand(i: I));
4543 return MIB;
4544 }
4545
4546 assert(Opc != AMDGPU::V_FMAC_F16_t16_e32 &&
4547 Opc != AMDGPU::V_FMAC_F16_fake16_e32 &&
4548 "V_FMAC_F16_t16/fake16_e32 is not supported and not expected to be "
4549 "present pre-RA");
4550
4551 // Handle MAC/FMAC.
4552 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64;
4553 bool IsLegacy = Opc == AMDGPU::V_MAC_LEGACY_F32_e32 ||
4554 Opc == AMDGPU::V_MAC_LEGACY_F32_e64 ||
4555 Opc == AMDGPU::V_FMAC_LEGACY_F32_e32 ||
4556 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64;
4557 bool Src0Literal = false;
4558
4559 switch (Opc) {
4560 default:
4561 return nullptr;
4562 case AMDGPU::V_MAC_F16_e64:
4563 case AMDGPU::V_FMAC_F16_e64:
4564 case AMDGPU::V_FMAC_F16_t16_e64:
4565 case AMDGPU::V_FMAC_F16_fake16_e64:
4566 case AMDGPU::V_MAC_F32_e64:
4567 case AMDGPU::V_MAC_LEGACY_F32_e64:
4568 case AMDGPU::V_FMAC_F32_e64:
4569 case AMDGPU::V_FMAC_LEGACY_F32_e64:
4570 case AMDGPU::V_FMAC_F64_e64:
4571 break;
4572 case AMDGPU::V_MAC_F16_e32:
4573 case AMDGPU::V_FMAC_F16_e32:
4574 case AMDGPU::V_MAC_F32_e32:
4575 case AMDGPU::V_MAC_LEGACY_F32_e32:
4576 case AMDGPU::V_FMAC_F32_e32:
4577 case AMDGPU::V_FMAC_LEGACY_F32_e32:
4578 case AMDGPU::V_FMAC_F64_e32: {
4579 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
4580 Name: AMDGPU::OpName::src0);
4581 const MachineOperand *Src0 = &MI.getOperand(i: Src0Idx);
4582 if (!Src0->isReg() && !Src0->isImm())
4583 return nullptr;
4584
4585 if (Src0->isImm() && !isInlineConstant(MI, OpIdx: Src0Idx, MO: *Src0))
4586 Src0Literal = true;
4587
4588 break;
4589 }
4590 }
4591
4592 MachineInstrBuilder MIB;
4593 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
4594 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
4595 const MachineOperand *Src0Mods =
4596 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
4597 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
4598 const MachineOperand *Src1Mods =
4599 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
4600 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
4601 const MachineOperand *Src2Mods =
4602 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
4603 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
4604 const MachineOperand *Omod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
4605 const MachineOperand *OpSel = getNamedOperand(MI, OperandName: AMDGPU::OpName::op_sel);
4606
4607 if (!Src0Mods && !Src1Mods && !Src2Mods && !Clamp && !Omod && !IsLegacy &&
4608 (!IsF64 || ST.hasFmaakFmamkF64Insts()) &&
4609 // If we have an SGPR input, we will violate the constant bus restriction.
4610 (ST.getConstantBusLimit(Opcode: Opc) > 1 || !Src0->isReg() ||
4611 !RI.isSGPRReg(MRI: MBB.getParent()->getRegInfo(), Reg: Src0->getReg()))) {
4612 MachineInstr *DefMI = nullptr;
4613 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4614 std::optional<int64_t> ImmOpt;
4615 int64_t Imm;
4616
4617 if (!Src0Literal &&
4618 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src2, DefMI: &DefMI))) {
4619 unsigned NewOpc = getNewFMAAKInst(ST, Opc);
4620 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4621 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4622 .add(MO: *Dst)
4623 .add(MO: *Src0)
4624 .add(MO: *Src1)
4625 .addImm(Val: *ImmOpt)
4626 .setMIFlags(MI.getFlags());
4627 U.RemoveMIUse = DefMI;
4628 return MIB;
4629 }
4630 }
4631 unsigned NewOpc = getNewFMAMKInst(ST, Opc);
4632 if (!Src0Literal &&
4633 (ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src1, DefMI: &DefMI))) {
4634 if (pseudoToMCOpcode(Opcode: NewOpc) != -1) {
4635 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4636 .add(MO: *Dst)
4637 .add(MO: *Src0)
4638 .addImm(Val: *ImmOpt)
4639 .add(MO: *Src2)
4640 .setMIFlags(MI.getFlags());
4641 U.RemoveMIUse = DefMI;
4642 return MIB;
4643 }
4644 }
4645 if ((ImmOpt = getImmOrMaterializedImm(MRI, Op: *Src0, DefMI: &DefMI))) {
4646 Imm = *ImmOpt;
4647 if (pseudoToMCOpcode(Opcode: NewOpc) != -1 &&
4648 isOperandLegal(
4649 MI, OpIdx: AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::src0),
4650 MO: Src1)) {
4651 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4652 .add(MO: *Dst)
4653 .add(MO: *Src1)
4654 .addImm(Val: Imm)
4655 .add(MO: *Src2)
4656 .setMIFlags(MI.getFlags());
4657 U.RemoveMIUse = DefMI;
4658 return MIB;
4659 }
4660 }
4661 }
4662
4663 // VOP2 mac/fmac with a literal operand cannot be converted to VOP3 mad/fma
4664 // if VOP3 does not allow a literal operand.
4665 if (Src0Literal && !ST.hasVOP3Literal())
4666 return nullptr;
4667
4668 unsigned NewOpc = getNewFMAInst(ST, Opc);
4669
4670 if (pseudoToMCOpcode(Opcode: NewOpc) == -1)
4671 return nullptr;
4672
4673 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4674 .add(MO: *Dst)
4675 .addImm(Val: Src0Mods ? Src0Mods->getImm() : 0)
4676 .add(MO: *Src0)
4677 .addImm(Val: Src1Mods ? Src1Mods->getImm() : 0)
4678 .add(MO: *Src1)
4679 .addImm(Val: Src2Mods ? Src2Mods->getImm() : 0)
4680 .add(MO: *Src2)
4681 .addImm(Val: Clamp ? Clamp->getImm() : 0)
4682 .addImm(Val: Omod ? Omod->getImm() : 0)
4683 .setMIFlags(MI.getFlags());
4684 if (AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel))
4685 MIB.addImm(Val: OpSel ? OpSel->getImm() : 0);
4686 return MIB;
4687}
4688
4689// It's not generally safe to move VALU instructions across these since it will
4690// start using the register as a base index rather than directly.
4691// XXX - Why isn't hasSideEffects sufficient for these?
4692static bool changesVGPRIndexingMode(const MachineInstr &MI) {
4693 switch (MI.getOpcode()) {
4694 case AMDGPU::S_SET_GPR_IDX_ON:
4695 case AMDGPU::S_SET_GPR_IDX_MODE:
4696 case AMDGPU::S_SET_GPR_IDX_OFF:
4697 return true;
4698 default:
4699 return false;
4700 }
4701}
4702
4703bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
4704 const MachineBasicBlock *MBB,
4705 const MachineFunction &MF) const {
4706 // Skipping the check for SP writes in the base implementation. The reason it
4707 // was added was apparently due to compile time concerns.
4708 //
4709 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops
4710 // but is probably avoidable.
4711
4712 // Copied from base implementation.
4713 // Terminators and labels can't be scheduled around.
4714 if (MI.isTerminator() || MI.isPosition())
4715 return true;
4716
4717 // INLINEASM_BR can jump to another block
4718 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
4719 return true;
4720
4721 if (MI.getOpcode() == AMDGPU::SCHED_BARRIER && MI.getOperand(i: 0).getImm() == 0)
4722 return true;
4723
4724 // Target-independent instructions do not have an implicit-use of EXEC, even
4725 // when they operate on VGPRs. Treating EXEC modifications as scheduling
4726 // boundaries prevents incorrect movements of such instructions.
4727 return MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI) ||
4728 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
4729 MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
4730 MI.getOpcode() == AMDGPU::S_SETPRIO ||
4731 MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
4732 changesVGPRIndexingMode(MI);
4733}
4734
4735bool SIInstrInfo::isAlwaysGDS(uint32_t Opcode) const {
4736 return Opcode == AMDGPU::DS_ORDERED_COUNT ||
4737 Opcode == AMDGPU::DS_ADD_GS_REG_RTN ||
4738 Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode);
4739}
4740
4741bool SIInstrInfo::mayAccessScratch(const MachineInstr &MI) const {
4742 // Instructions that access scratch use FLAT encoding or BUF encodings.
4743 if ((!isFLAT(MI) || isFLATGlobal(MI)) && !isBUF(MI))
4744 return false;
4745
4746 // SCRATCH instructions always access scratch.
4747 if (isFLATScratch(MI))
4748 return true;
4749
4750 // If FLAT_SCRATCH registers are not initialized, we can never access scratch
4751 // via the aperture.
4752 if (MI.getMF()->getFunction().hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))
4753 return false;
4754
4755 // If there are no memory operands then conservatively assume the flat
4756 // operation may access scratch.
4757 if (MI.memoperands_empty())
4758 return true;
4759
4760 // See if any memory operand specifies an address space that involves scratch.
4761 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *Memop) {
4762 unsigned AS = Memop->getAddrSpace();
4763 if (AS == AMDGPUAS::FLAT_ADDRESS) {
4764 const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
4765 return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
4766 MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
4767 }
4768 return AS == AMDGPUAS::PRIVATE_ADDRESS;
4769 });
4770}
4771
4772bool SIInstrInfo::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {
4773 assert(isFLAT(MI));
4774
4775 // All flat instructions use the VMEM counter except prefetch.
4776 if (!usesVM_CNT(MI))
4777 return false;
4778
4779 // If there are no memory operands then conservatively assume the flat
4780 // operation may access VMEM.
4781 if (MI.memoperands_empty())
4782 return true;
4783
4784 // See if any memory operand specifies an address space that involves VMEM.
4785 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
4786 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
4787 // (GDS) address space is not supported by flat operations. Therefore, simply
4788 // return true unless only the LDS address space is found.
4789 for (const MachineMemOperand *Memop : MI.memoperands()) {
4790 unsigned AS = Memop->getAddrSpace();
4791 assert(AS != AMDGPUAS::REGION_ADDRESS);
4792 if (AS != AMDGPUAS::LOCAL_ADDRESS)
4793 return true;
4794 }
4795
4796 return false;
4797}
4798
4799bool SIInstrInfo::mayAccessLDSThroughFlat(const MachineInstr &MI,
4800 bool TgSplit) const {
4801 assert(isFLAT(MI));
4802
4803 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
4804 if (!usesLGKM_CNT(MI))
4805 return false;
4806
4807 // If in tgsplit mode then there can be no use of LDS.
4808 if (TgSplit)
4809 return false;
4810
4811 // If there are no memory operands then conservatively assume the flat
4812 // operation may access LDS.
4813 if (MI.memoperands_empty())
4814 return true;
4815
4816 // See if any memory operand specifies an address space that involves LDS.
4817 for (const MachineMemOperand *Memop : MI.memoperands()) {
4818 unsigned AS = Memop->getAddrSpace();
4819 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
4820 return true;
4821 }
4822
4823 return false;
4824}
4825
4826bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) {
4827 // Skip the full operand and register alias search modifiesRegister
4828 // does. There's only a handful of instructions that touch this, it's only an
4829 // implicit def, and doesn't alias any other registers.
4830 return is_contained(Range: MI.getDesc().implicit_defs(), Element: AMDGPU::MODE);
4831}
4832
4833bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
4834 unsigned Opcode = MI.getOpcode();
4835
4836 if (MI.mayStore() && isSMRD(MI))
4837 return true; // scalar store or atomic
4838
4839 // This will terminate the function when other lanes may need to continue.
4840 if (MI.isReturn())
4841 return true;
4842
4843 // These instructions cause shader I/O that may cause hardware lockups
4844 // when executed with an empty EXEC mask.
4845 //
4846 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
4847 // EXEC = 0, but checking for that case here seems not worth it
4848 // given the typical code patterns.
4849 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
4850 isEXP(Opcode) || Opcode == AMDGPU::DS_ORDERED_COUNT ||
4851 Opcode == AMDGPU::S_TRAP || Opcode == AMDGPU::S_WAIT_EVENT ||
4852 Opcode == AMDGPU::S_SETHALT)
4853 return true;
4854
4855 if (MI.isCall() || MI.isInlineAsm())
4856 return true; // conservative assumption
4857
4858 // V_PERM_PK16 must issue with EXEC != 0 so its follower (or an inserted
4859 // V_NOP) actually runs on the VALU pipe. Returning true here keeps the
4860 // s_cbranch_execz that skips this region when EXEC is empty.
4861 if (ST.hasVPermPk16Hazard() && isVPermPk16(Opcode))
4862 return true;
4863
4864 // Assume that barrier interactions are only intended with active lanes.
4865 if (isBarrier(Opcode))
4866 return true;
4867
4868 // A mode change is a scalar operation that influences vector instructions.
4869 if (modifiesModeRegister(MI))
4870 return true;
4871
4872 // These are like SALU instructions in terms of effects, so it's questionable
4873 // whether we should return true for those.
4874 //
4875 // However, executing them with EXEC = 0 causes them to operate on undefined
4876 // data, which we avoid by returning true here.
4877 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 ||
4878 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32 ||
4879 Opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR ||
4880 Opcode == AMDGPU::SI_SPILL_S32_TO_VGPR)
4881 return true;
4882
4883 return false;
4884}
4885
4886bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
4887 const MachineInstr &MI) const {
4888 if (MI.isMetaInstruction())
4889 return false;
4890
4891 // This won't read exec if this is an SGPR->SGPR copy.
4892 if (MI.isCopyLike()) {
4893 if (!RI.isSGPRReg(MRI, Reg: MI.getOperand(i: 0).getReg()))
4894 return true;
4895
4896 // Make sure this isn't copying exec as a normal operand
4897 return MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4898 }
4899
4900 // Make a conservative assumption about the callee.
4901 if (MI.isCall())
4902 return true;
4903
4904 // Be conservative with any unhandled generic opcodes.
4905 if (!isTargetSpecificOpcode(Opcode: MI.getOpcode()))
4906 return true;
4907
4908 return !isSALU(MI) || MI.readsRegister(Reg: AMDGPU::EXEC, TRI: &RI);
4909}
4910
4911bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
4912 switch (Imm.getBitWidth()) {
4913 case 1: // This likely will be a condition code mask.
4914 return true;
4915
4916 case 32:
4917 return AMDGPU::isInlinableLiteral32(Literal: Imm.getSExtValue(),
4918 HasInv2Pi: ST.hasInv2PiInlineImm());
4919 case 64:
4920 return AMDGPU::isInlinableLiteral64(Literal: Imm.getSExtValue(),
4921 HasInv2Pi: ST.hasInv2PiInlineImm());
4922 case 16:
4923 return ST.has16BitInsts() &&
4924 AMDGPU::isInlinableLiteralI16(Literal: Imm.getSExtValue(),
4925 HasInv2Pi: ST.hasInv2PiInlineImm());
4926 default:
4927 llvm_unreachable("invalid bitwidth");
4928 }
4929}
4930
4931bool SIInstrInfo::isInlineConstant(const APFloat &Imm) const {
4932 APInt IntImm = Imm.bitcastToAPInt();
4933 int64_t IntImmVal = IntImm.getSExtValue();
4934 bool HasInv2Pi = ST.hasInv2PiInlineImm();
4935 switch (APFloat::SemanticsToEnum(Sem: Imm.getSemantics())) {
4936 default:
4937 llvm_unreachable("invalid fltSemantics");
4938 case APFloatBase::S_IEEEsingle:
4939 case APFloatBase::S_IEEEdouble:
4940 return isInlineConstant(Imm: IntImm);
4941 case APFloatBase::S_BFloat:
4942 return ST.has16BitInsts() &&
4943 AMDGPU::isInlinableLiteralBF16(Literal: IntImmVal, HasInv2Pi);
4944 case APFloatBase::S_IEEEhalf:
4945 return ST.has16BitInsts() &&
4946 AMDGPU::isInlinableLiteralFP16(Literal: IntImmVal, HasInv2Pi);
4947 }
4948}
4949
4950bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
4951 // MachineOperand provides no way to tell the true operand size, since it only
4952 // records a 64-bit value. We need to know the size to determine if a 32-bit
4953 // floating point immediate bit pattern is legal for an integer immediate. It
4954 // would be for any 32-bit integer operand, but would not be for a 64-bit one.
4955 switch (OperandType) {
4956 case AMDGPU::OPERAND_REG_IMM_INT32:
4957 case AMDGPU::OPERAND_REG_IMM_FP32:
4958 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
4959 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
4960 case AMDGPU::OPERAND_REG_IMM_V2FP32:
4961 case AMDGPU::OPERAND_REG_IMM_V2INT32:
4962 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
4963 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
4964 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
4965 int32_t Trunc = static_cast<int32_t>(Imm);
4966 return AMDGPU::isInlinableLiteral32(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
4967 }
4968 case AMDGPU::OPERAND_REG_IMM_INT64:
4969 case AMDGPU::OPERAND_REG_IMM_FP64:
4970 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
4971 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
4972 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
4973 case AMDGPU::OPERAND_REG_IMM_V2FP64:
4974 case AMDGPU::OPERAND_REG_IMM_V2INT64:
4975 return AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm());
4976 case AMDGPU::OPERAND_REG_IMM_INT16:
4977 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
4978 // We would expect inline immediates to not be concerned with an integer/fp
4979 // distinction. However, in the case of 16-bit integer operations, the
4980 // "floating point" values appear to not work. It seems read the low 16-bits
4981 // of 32-bit immediates, which happens to always work for the integer
4982 // values.
4983 //
4984 // See llvm bugzilla 46302.
4985 //
4986 // TODO: Theoretically we could use op-sel to use the high bits of the
4987 // 32-bit FP values.
4988 return AMDGPU::isInlinableIntLiteral(Literal: Imm);
4989 case AMDGPU::OPERAND_REG_IMM_V2INT16:
4990 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
4991 return AMDGPU::isInlinableLiteralV2I16(Literal: Imm);
4992 case AMDGPU::OPERAND_REG_IMM_V2FP16:
4993 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
4994 return AMDGPU::isInlinableLiteralV2F16(Literal: Imm);
4995 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
4996 return AMDGPU::isPKFMACF16InlineConstant(Literal: Imm, IsGFX11Plus: ST.isGFX11Plus());
4997 case AMDGPU::OPERAND_REG_IMM_V2BF16:
4998 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
4999 return AMDGPU::isInlinableLiteralV2BF16(Literal: Imm);
5000 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5001 return false;
5002 case AMDGPU::OPERAND_REG_IMM_FP16:
5003 case AMDGPU::OPERAND_REG_INLINE_C_FP16: {
5004 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
5005 // A few special case instructions have 16-bit operands on subtargets
5006 // where 16-bit instructions are not legal.
5007 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
5008 // constants in these cases
5009 int16_t Trunc = static_cast<int16_t>(Imm);
5010 return ST.has16BitInsts() &&
5011 AMDGPU::isInlinableLiteralFP16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5012 }
5013
5014 return false;
5015 }
5016 case AMDGPU::OPERAND_REG_IMM_BF16:
5017 case AMDGPU::OPERAND_REG_INLINE_C_BF16: {
5018 if (isInt<16>(x: Imm) || isUInt<16>(x: Imm)) {
5019 int16_t Trunc = static_cast<int16_t>(Imm);
5020 return ST.has16BitInsts() &&
5021 AMDGPU::isInlinableLiteralBF16(Literal: Trunc, HasInv2Pi: ST.hasInv2PiInlineImm());
5022 }
5023 return false;
5024 }
5025 case AMDGPU::OPERAND_KIMM32:
5026 case AMDGPU::OPERAND_KIMM16:
5027 case AMDGPU::OPERAND_KIMM64:
5028 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
5029 return false;
5030 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5031 return isLegalAV64PseudoImm(Imm);
5032 case AMDGPU::OPERAND_INPUT_MODS:
5033 case MCOI::OPERAND_IMMEDIATE:
5034 // Always embedded in the instruction for free.
5035 return true;
5036 case MCOI::OPERAND_UNKNOWN:
5037 case MCOI::OPERAND_REGISTER:
5038 case MCOI::OPERAND_PCREL:
5039 case MCOI::OPERAND_GENERIC_0:
5040 case MCOI::OPERAND_GENERIC_1:
5041 case MCOI::OPERAND_GENERIC_2:
5042 case MCOI::OPERAND_GENERIC_3:
5043 case MCOI::OPERAND_GENERIC_4:
5044 case MCOI::OPERAND_GENERIC_5:
5045 // Just ignore anything else.
5046 return false;
5047 default:
5048 llvm_unreachable("invalid operand type");
5049 }
5050}
5051
5052static bool compareMachineOp(const MachineOperand &Op0,
5053 const MachineOperand &Op1) {
5054 if (Op0.getType() != Op1.getType())
5055 return false;
5056
5057 switch (Op0.getType()) {
5058 case MachineOperand::MO_Register:
5059 return Op0.getReg() == Op1.getReg();
5060 case MachineOperand::MO_Immediate:
5061 return Op0.getImm() == Op1.getImm();
5062 default:
5063 llvm_unreachable("Didn't expect to be comparing these operand types");
5064 }
5065}
5066
5067bool SIInstrInfo::isLiteralOperandLegal(const MCInstrDesc &InstDesc,
5068 const MCOperandInfo &OpInfo) const {
5069 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
5070 return true;
5071
5072 if (!RI.opCanUseLiteralConstant(OpType: OpInfo.OperandType))
5073 return false;
5074
5075 if (!isVOP3(Desc: InstDesc) || !AMDGPU::isSISrcOperand(OpInfo))
5076 return true;
5077
5078 return ST.hasVOP3Literal();
5079}
5080
5081bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5082 int64_t ImmVal) const {
5083 const unsigned Opc = InstDesc.getOpcode();
5084 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
5085 if (Src1Idx != -1 && isDPP(Opcode: Opc) && !ST.hasDPPSrc1SGPR() &&
5086 OpNo == static_cast<unsigned>(Src1Idx))
5087 return false;
5088
5089 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5090 if (isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType)) {
5091 if (isMAI(Desc: InstDesc) && ST.hasMFMAInlineLiteralBug() &&
5092 OpNo == (unsigned)AMDGPU::getNamedOperandIdx(Opcode: InstDesc.getOpcode(),
5093 Name: AMDGPU::OpName::src2))
5094 return false;
5095
5096 if (ST.hasBF16InlineConstFromUpperFP32() && isVOP1(Opcode: Opc)) {
5097 if ((OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
5098 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
5099 isInlineConstant(Imm: ImmVal, OperandType: OpInfo.OperandType))
5100 return false;
5101 }
5102
5103 return RI.opCanUseInlineConstant(OpType: OpInfo.OperandType);
5104 }
5105
5106 return isLiteralOperandLegal(InstDesc, OpInfo);
5107}
5108
5109bool SIInstrInfo::isImmOperandLegal(const MCInstrDesc &InstDesc, unsigned OpNo,
5110 const MachineOperand &MO) const {
5111 if (MO.isImm())
5112 return isImmOperandLegal(InstDesc, OpNo, ImmVal: MO.getImm());
5113
5114 assert((MO.isTargetIndex() || MO.isFI() || MO.isGlobal()) &&
5115 "unexpected imm-like operand kind");
5116 const MCOperandInfo &OpInfo = InstDesc.operands()[OpNo];
5117 return isLiteralOperandLegal(InstDesc, OpInfo);
5118}
5119
5120bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {
5121 // 2 32-bit inline constants packed into one.
5122 return AMDGPU::isInlinableLiteral32(Literal: Lo_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm()) &&
5123 AMDGPU::isInlinableLiteral32(Literal: Hi_32(Value: Imm), HasInv2Pi: ST.hasInv2PiInlineImm());
5124}
5125
5126bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
5127 // GFX90A does not have V_MUL_LEGACY_F32_e32.
5128 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
5129 return false;
5130
5131 int Op32 = AMDGPU::getVOPe32(Opcode);
5132 if (Op32 == -1)
5133 return false;
5134
5135 return pseudoToMCOpcode(Opcode: Op32) != -1;
5136}
5137
5138/// Return true if \p MI is a VALU comparison, i.e. an instruction that writes
5139/// a lane mask with one bit per lane, and zeroes the bits of lanes that were
5140/// inactive when it executed.
5141///
5142/// TODO: Also handle the sdst result of V_ADD_CO_U32 and V_SUB_CO_U32 and
5143/// V_DIV_SCALE_F32.
5144static bool isVCmp(const SIInstrInfo &TII, const MachineInstr &MI) {
5145 if (TII.isVOPC(MI))
5146 return true;
5147 int Op32 = AMDGPU::getVOPe32(Opcode: MI.getOpcode());
5148 return Op32 != -1 && TII.isVOPC(Opcode: Op32);
5149}
5150
5151bool SIInstrInfo::isMaskedByExec(Register Reg, const MachineInstr &Use,
5152 const MachineRegisterInfo &MRI,
5153 unsigned Depth) const {
5154 assert(MRI.isSSA() && "isMaskedByExec requires SSA form");
5155 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5156 const MachineBasicBlock *MBB = Use.getParent();
5157
5158 // EXEC itself is trivially masked by EXEC.
5159 if (Reg == LMC.ExecReg)
5160 return true;
5161
5162 // Maximum depth of the def-use walk.
5163 constexpr unsigned MaxDepth = 6;
5164 if (Depth >= MaxDepth || !Reg.isVirtual())
5165 return false;
5166
5167 // Only look at definitions that can execute under the same EXEC mask as the
5168 // use.
5169 const MachineInstr *Def = MRI.getVRegDef(Reg);
5170 if (!Def || Def->getParent() != MBB)
5171 return false;
5172
5173 if (isVCmp(TII: *this, MI: *Def))
5174 return true;
5175
5176 // Recurse into an operand, which must be a whole register to say anything
5177 // about the whole lane mask.
5178 auto Recurse = [&](unsigned OpIdx) {
5179 const MachineOperand &MO = Def->getOperand(i: OpIdx);
5180 return MO.isReg() && !MO.getSubReg() &&
5181 isMaskedByExec(Reg: MO.getReg(), Use, MRI, Depth: Depth + 1);
5182 };
5183
5184 unsigned Opc = Def->getOpcode();
5185 if (Opc == AMDGPU::COPY && Recurse(1))
5186 return true;
5187 if (Opc == LMC.AndOpc && (Recurse(1) || Recurse(2)))
5188 return true;
5189 if (Opc == LMC.AndN2Opc && Recurse(1))
5190 return true;
5191 if ((Opc == LMC.OrOpc || Opc == LMC.XorOpc) && Recurse(1) && Recurse(2))
5192 return true;
5193 // TODO: Sometimes we encounter "reg = S_CSELECT -1, 0". If Reg has no other
5194 // uses this could be optimized to "reg = S_CSELECT $exec, 0".
5195
5196 return false;
5197}
5198
5199bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
5200 // The src0_modifier operand is present on all instructions
5201 // that have modifiers.
5202
5203 return AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers);
5204}
5205
5206bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
5207 AMDGPU::OpName OpName) const {
5208 const MachineOperand *Mods = getNamedOperand(MI, OperandName: OpName);
5209 return Mods && Mods->getImm();
5210}
5211
5212bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
5213 return any_of(Range: ModifierOpNames,
5214 P: [&](AMDGPU::OpName Name) { return hasModifiersSet(MI, OpName: Name); });
5215}
5216
5217bool SIInstrInfo::canShrink(const MachineInstr &MI,
5218 const MachineRegisterInfo &MRI) const {
5219 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5220 // Can't shrink instruction with three operands.
5221 if (Src2) {
5222 switch (MI.getOpcode()) {
5223 default: return false;
5224
5225 case AMDGPU::V_ADDC_U32_e64:
5226 case AMDGPU::V_SUBB_U32_e64:
5227 case AMDGPU::V_SUBBREV_U32_e64: {
5228 const MachineOperand *Src1
5229 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5230 if (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()))
5231 return false;
5232 // Additional verification is needed for sdst/src2.
5233 return true;
5234 }
5235 case AMDGPU::V_MAC_F16_e64:
5236 case AMDGPU::V_MAC_F32_e64:
5237 case AMDGPU::V_MAC_LEGACY_F32_e64:
5238 case AMDGPU::V_FMAC_F16_e64:
5239 case AMDGPU::V_FMAC_F16_t16_e64:
5240 case AMDGPU::V_FMAC_F16_fake16_e64:
5241 case AMDGPU::V_FMAC_F32_e64:
5242 case AMDGPU::V_FMAC_F64_e64:
5243 case AMDGPU::V_FMAC_LEGACY_F32_e64:
5244 if (!Src2->isReg() || !RI.isVGPR(MRI, Reg: Src2->getReg()) ||
5245 hasModifiersSet(MI, OpName: AMDGPU::OpName::src2_modifiers))
5246 return false;
5247 break;
5248
5249 case AMDGPU::V_CNDMASK_B32_e64:
5250 break;
5251 }
5252 }
5253
5254 const MachineOperand *Src1 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
5255 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Reg: Src1->getReg()) ||
5256 hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers)))
5257 return false;
5258
5259 // Make sure src0 isn't using any modifiers.
5260 if (hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers))
5261 return false;
5262
5263 // Can it be shrunk to a valid 32 bit opcode?
5264 if (!hasVALU32BitEncoding(Opcode: MI.getOpcode()))
5265 return false;
5266
5267 const MachineOperand *Src0 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
5268 if (Src0 && Src0->isImm()) {
5269 unsigned Op32 = AMDGPU::getVOPe32(Opcode: MI.getOpcode());
5270 if (!isImmOperandLegal(
5271 InstDesc: get(Opcode: Op32), OpNo: AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src0),
5272 MO: *Src0))
5273 return false;
5274 }
5275
5276 // Check output modifiers
5277 return !hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) &&
5278 !hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
5279 !hasModifiersSet(MI, OpName: AMDGPU::OpName::byte_sel) &&
5280 // TODO: Can we avoid checking bound_ctrl/fi here?
5281 // They are only used by permlane*_swap special case.
5282 !hasModifiersSet(MI, OpName: AMDGPU::OpName::bound_ctrl) &&
5283 !hasModifiersSet(MI, OpName: AMDGPU::OpName::fi);
5284}
5285
5286// Set VCC operand with all flags from \p Orig, except for setting it as
5287// implicit.
5288static void copyFlagsToImplicitVCC(MachineInstr &MI,
5289 const MachineOperand &Orig) {
5290
5291 for (MachineOperand &Use : MI.implicit_operands()) {
5292 if (Use.isUse() &&
5293 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) {
5294 Use.setIsUndef(Orig.isUndef());
5295 Use.setIsKill(Orig.isKill());
5296 return;
5297 }
5298 }
5299}
5300
5301MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
5302 unsigned Op32) const {
5303 MachineBasicBlock *MBB = MI.getParent();
5304
5305 const MCInstrDesc &Op32Desc = get(Opcode: Op32);
5306 MachineInstrBuilder Inst32 =
5307 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: Op32Desc)
5308 .setMIFlags(MI.getFlags());
5309
5310 // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
5311 // For VOPC instructions, this is replaced by an implicit def of vcc.
5312
5313 // We assume the defs of the shrunk opcode are in the same order, and the
5314 // shrunk opcode loses the last def (SGPR def, in the VOP3->VOPC case).
5315 for (int I = 0, E = Op32Desc.getNumDefs(); I != E; ++I)
5316 Inst32.add(MO: MI.getOperand(i: I));
5317
5318 const MachineOperand *Src2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
5319
5320 int Idx = MI.getNumExplicitDefs();
5321 for (const MachineOperand &Use : MI.explicit_uses()) {
5322 int OpTy = MI.getDesc().operands()[Idx++].OperandType;
5323 if (OpTy == AMDGPU::OPERAND_INPUT_MODS || OpTy == MCOI::OPERAND_IMMEDIATE)
5324 continue;
5325
5326 if (&Use == Src2) {
5327 if (AMDGPU::getNamedOperandIdx(Opcode: Op32, Name: AMDGPU::OpName::src2) == -1) {
5328 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
5329 // replaced with an implicit read of vcc or vcc_lo. The implicit read
5330 // of vcc was already added during the initial BuildMI, but we
5331 // 1) may need to change vcc to vcc_lo to preserve the original register
5332 // 2) have to preserve the original flags.
5333 copyFlagsToImplicitVCC(MI&: *Inst32, Orig: *Src2);
5334 continue;
5335 }
5336 }
5337
5338 Inst32.add(MO: Use);
5339 }
5340
5341 // FIXME: Losing implicit operands
5342 fixImplicitOperands(MI&: *Inst32);
5343
5344 // The explicit carry/result def is dropped in favor of an implicit VCC def;
5345 // preserve the dead flag.
5346 const MachineOperand *OldSDst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
5347 if (OldSDst && OldSDst->isDead()) {
5348 if (MachineOperand *NewVCC =
5349 Inst32->findRegisterDefOperand(Reg: RI.getVCC(), TRI: &RI))
5350 NewVCC->setIsDead();
5351 }
5352
5353 return Inst32;
5354}
5355
5356bool SIInstrInfo::physRegUsesConstantBus(const MachineOperand &RegOp) const {
5357 // Null is free
5358 Register Reg = RegOp.getReg();
5359 if (Reg == AMDGPU::SGPR_NULL || Reg == AMDGPU::SGPR_NULL64)
5360 return false;
5361
5362 // SGPRs use the constant bus
5363
5364 // FIXME: implicit registers that are not part of the MCInstrDesc's implicit
5365 // physical register operands should also count, except for exec.
5366 if (RegOp.isImplicit())
5367 return Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::M0;
5368
5369 // SGPRs use the constant bus
5370 return AMDGPU::SReg_32RegClass.contains(Reg) ||
5371 AMDGPU::SReg_64RegClass.contains(Reg);
5372}
5373
5374bool SIInstrInfo::regUsesConstantBus(const MachineOperand &RegOp,
5375 const MachineRegisterInfo &MRI) const {
5376 Register Reg = RegOp.getReg();
5377 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5378 : physRegUsesConstantBus(RegOp);
5379}
5380
5381bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
5382 const MachineOperand &MO,
5383 const MCOperandInfo &OpInfo) const {
5384 // Literal constants use the constant bus.
5385 if (!MO.isReg())
5386 return !isInlineConstant(MO, OpInfo);
5387
5388 Register Reg = MO.getReg();
5389 return Reg.isVirtual() ? RI.isSGPRClass(RC: MRI.getRegClass(Reg))
5390 : physRegUsesConstantBus(RegOp: MO);
5391}
5392
5393static Register findImplicitSGPRRead(const MachineInstr &MI) {
5394 for (const MachineOperand &MO : MI.implicit_operands()) {
5395 // We only care about reads.
5396 if (MO.isDef())
5397 continue;
5398
5399 switch (MO.getReg()) {
5400 case AMDGPU::VCC:
5401 case AMDGPU::VCC_LO:
5402 case AMDGPU::VCC_HI:
5403 case AMDGPU::M0:
5404 case AMDGPU::FLAT_SCR:
5405 return MO.getReg();
5406
5407 default:
5408 break;
5409 }
5410 }
5411
5412 return Register();
5413}
5414
5415static bool shouldReadExec(const MachineInstr &MI) {
5416 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) {
5417 switch (MI.getOpcode()) {
5418 case AMDGPU::V_READLANE_B32:
5419 case AMDGPU::SI_RESTORE_S32_FROM_VGPR:
5420 case AMDGPU::V_WRITELANE_B32:
5421 case AMDGPU::SI_SPILL_S32_TO_VGPR:
5422 return false;
5423 }
5424
5425 return true;
5426 }
5427
5428 if (MI.isPreISelOpcode() ||
5429 SIInstrInfo::isGenericOpcode(Opc: MI.getOpcode()) ||
5430 SIInstrInfo::isSALU(MI) ||
5431 SIInstrInfo::isSMRD(MI))
5432 return false;
5433
5434 return true;
5435}
5436
5437static bool isRegOrFI(const MachineOperand &MO) {
5438 return MO.isReg() || MO.isFI();
5439}
5440
5441static bool isSubRegOf(const SIRegisterInfo &TRI,
5442 const MachineOperand &SuperVec,
5443 const MachineOperand &SubReg) {
5444 if (SubReg.getReg().isPhysical())
5445 return TRI.isSubRegister(RegA: SuperVec.getReg(), RegB: SubReg.getReg());
5446
5447 return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
5448 SubReg.getReg() == SuperVec.getReg();
5449}
5450
5451// Verify the illegal copy from vector register to SGPR for generic opcode COPY
5452bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
5453 const MachineRegisterInfo &MRI,
5454 StringRef &ErrInfo) const {
5455 Register DstReg = MI.getOperand(i: 0).getReg();
5456 Register SrcReg = MI.getOperand(i: 1).getReg();
5457 // This is a check for copy from vector register to SGPR
5458 if (RI.isVectorRegister(MRI, Reg: SrcReg) && RI.isSGPRReg(MRI, Reg: DstReg)) {
5459 ErrInfo = "illegal copy from vector register to SGPR";
5460 return false;
5461 }
5462 return true;
5463}
5464
5465bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
5466 StringRef &ErrInfo) const {
5467 uint32_t Opcode = MI.getOpcode();
5468 const MachineFunction *MF = MI.getMF();
5469 const MachineRegisterInfo &MRI = MF->getRegInfo();
5470
5471 // FIXME: At this point the COPY verify is done only for non-ssa forms.
5472 // Find a better property to recognize the point where instruction selection
5473 // is just done.
5474 // We can only enforce this check after SIFixSGPRCopies pass so that the
5475 // illegal copies are legalized and thereafter we don't expect a pass
5476 // inserting similar copies.
5477 if (!MRI.isSSA() && MI.isCopy())
5478 return verifyCopy(MI, MRI, ErrInfo);
5479
5480 if (SIInstrInfo::isGenericOpcode(Opc: Opcode))
5481 return true;
5482
5483 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
5484 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
5485 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2);
5486 int Src3Idx = -1;
5487 if (Src0Idx == -1) {
5488 // VOPD V_DUAL_* instructions use different operand names.
5489 Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0X);
5490 Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1X);
5491 Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0Y);
5492 Src3Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vsrc1Y);
5493 }
5494
5495 // Make sure the number of operands is correct.
5496 const MCInstrDesc &Desc = get(Opcode);
5497 if (!Desc.isVariadic() &&
5498 Desc.getNumOperands() != MI.getNumExplicitOperands()) {
5499 ErrInfo = "Instruction has wrong number of operands.";
5500 return false;
5501 }
5502
5503 if (MI.isInlineAsm()) {
5504 // Verify register classes for inlineasm constraints.
5505 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
5506 I != E; ++I) {
5507 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: I, TII: this, TRI: &RI);
5508 if (!RC)
5509 continue;
5510
5511 const MachineOperand &Op = MI.getOperand(i: I);
5512 if (!Op.isReg())
5513 continue;
5514
5515 Register Reg = Op.getReg();
5516 if (!Reg.isVirtual() && !RC->contains(Reg)) {
5517 ErrInfo = "inlineasm operand has incorrect register class.";
5518 return false;
5519 }
5520 }
5521
5522 return true;
5523 }
5524
5525 if (isImage(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) {
5526 ErrInfo = "missing memory operand from image instruction.";
5527 return false;
5528 }
5529
5530 // Make sure the register classes are correct.
5531 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
5532 const MachineOperand &MO = MI.getOperand(i);
5533 if (MO.isFPImm()) {
5534 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
5535 "all fp values to integers.";
5536 return false;
5537 }
5538
5539 const MCOperandInfo &OpInfo = Desc.operands()[i];
5540
5541 switch (OpInfo.OperandType) {
5542 case MCOI::OPERAND_REGISTER:
5543 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
5544 ErrInfo = "Illegal immediate value for operand.";
5545 return false;
5546 }
5547 break;
5548 case AMDGPU::OPERAND_REG_IMM_INT32:
5549 case AMDGPU::OPERAND_REG_IMM_INT16:
5550 case AMDGPU::OPERAND_REG_IMM_FP32:
5551 case AMDGPU::OPERAND_REG_IMM_BF16:
5552 case AMDGPU::OPERAND_REG_IMM_FP16:
5553 case AMDGPU::OPERAND_REG_IMM_V2FP16:
5554 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
5555 case AMDGPU::OPERAND_REG_IMM_V2INT16:
5556 case AMDGPU::OPERAND_REG_IMM_V2BF16:
5557 case AMDGPU::OPERAND_REG_IMM_V2FP64:
5558 case AMDGPU::OPERAND_REG_IMM_V2INT64:
5559 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
5560 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
5561 break;
5562 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
5563 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
5564 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
5565 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
5566 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
5567 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
5568 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
5569 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
5570 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
5571 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
5572 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
5573 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
5574 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
5575 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, OpIdx: i))) {
5576 ErrInfo = "Illegal immediate value for operand.";
5577 return false;
5578 }
5579 break;
5580 }
5581 case AMDGPU::OPERAND_REG_IMM_FP64:
5582 case AMDGPU::OPERAND_REG_IMM_INT64:
5583 case AMDGPU::OPERAND_REG_IMM_V2INT32:
5584 case AMDGPU::OPERAND_REG_IMM_V2FP32:
5585 if (ST.has64BitLiterals() && Desc.getSize() != 4 && MO.isImm() &&
5586 !isInlineConstant(MI, OpIdx: i) &&
5587 !AMDGPU::isValid32BitLiteral(Val: MO.getImm(),
5588 IsFP64: OpInfo.OperandType ==
5589 AMDGPU::OPERAND_REG_IMM_FP64)) {
5590 ErrInfo = "illegal 64-bit immediate value for operand.";
5591 return false;
5592 }
5593 break;
5594 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
5595 case AMDGPU::OPERAND_INPUT_MODS:
5596 if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, OpIdx: i)) {
5597 ErrInfo = "Expected inline constant for operand.";
5598 return false;
5599 }
5600 break;
5601 case AMDGPU::OPERAND_SDWA_VOPC_DST:
5602 case AMDGPU::OPERAND_KIMM16:
5603 break;
5604 case MCOI::OPERAND_IMMEDIATE:
5605 case AMDGPU::OPERAND_KIMM32:
5606 case AMDGPU::OPERAND_KIMM64:
5607 case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
5608 // Check if this operand is an immediate.
5609 // FrameIndex operands will be replaced by immediates, so they are
5610 // allowed.
5611 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
5612 ErrInfo = "Expected immediate, but got non-immediate";
5613 return false;
5614 }
5615 break;
5616 case MCOI::OPERAND_UNKNOWN:
5617 case MCOI::OPERAND_MEMORY:
5618 case MCOI::OPERAND_PCREL:
5619 break;
5620 default:
5621 if (OpInfo.isGenericType())
5622 continue;
5623 break;
5624 }
5625 }
5626
5627 // Verify SDWA
5628 if (isSDWA(MI)) {
5629 if (!ST.hasSDWA()) {
5630 ErrInfo = "SDWA is not supported on this target";
5631 return false;
5632 }
5633
5634 for (auto Op : {AMDGPU::OpName::src0_sel, AMDGPU::OpName::src1_sel,
5635 AMDGPU::OpName::dst_sel}) {
5636 const MachineOperand *MO = getNamedOperand(MI, OperandName: Op);
5637 if (!MO)
5638 continue;
5639 int64_t Imm = MO->getImm();
5640 if (Imm < 0 || Imm > AMDGPU::SDWA::SdwaSel::DWORD) {
5641 ErrInfo = "Invalid SDWA selection";
5642 return false;
5643 }
5644 }
5645
5646 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdst);
5647
5648 for (int OpIdx : {DstIdx, Src0Idx, Src1Idx, Src2Idx}) {
5649 if (OpIdx == -1)
5650 continue;
5651 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5652
5653 if (!ST.hasSDWAScalar()) {
5654 // Only VGPRS on VI
5655 if (!MO.isReg() || !RI.hasVGPRs(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg()))) {
5656 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
5657 return false;
5658 }
5659 } else {
5660 // No immediates on GFX9
5661 if (!MO.isReg()) {
5662 ErrInfo =
5663 "Only reg allowed as operands in SDWA instructions on GFX9+";
5664 return false;
5665 }
5666 }
5667 }
5668
5669 if (!ST.hasSDWAOmod()) {
5670 // No omod allowed on VI
5671 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5672 if (OMod != nullptr &&
5673 (!OMod->isImm() || OMod->getImm() != 0)) {
5674 ErrInfo = "OMod not allowed in SDWA instructions on VI";
5675 return false;
5676 }
5677 }
5678
5679 if (Opcode == AMDGPU::V_CVT_F32_FP8_sdwa ||
5680 Opcode == AMDGPU::V_CVT_F32_BF8_sdwa ||
5681 Opcode == AMDGPU::V_CVT_PK_F32_FP8_sdwa ||
5682 Opcode == AMDGPU::V_CVT_PK_F32_BF8_sdwa) {
5683 const MachineOperand *Src0ModsMO =
5684 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
5685 unsigned Mods = Src0ModsMO->getImm();
5686 if (Mods & SISrcMods::ABS || Mods & SISrcMods::NEG ||
5687 Mods & SISrcMods::SEXT) {
5688 ErrInfo = "sext, abs and neg are not allowed on this instruction";
5689 return false;
5690 }
5691 }
5692
5693 uint32_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
5694 if (isVOPC(Opcode: BasicOpcode)) {
5695 if (!ST.hasSDWASdst() && DstIdx != -1) {
5696 // Only vcc allowed as dst on VI for VOPC
5697 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5698 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
5699 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
5700 return false;
5701 }
5702 } else if (!ST.hasSDWAOutModsVOPC()) {
5703 // No clamp allowed on GFX9 for VOPC
5704 const MachineOperand *Clamp = getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
5705 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
5706 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
5707 return false;
5708 }
5709
5710 // No omod allowed on GFX9 for VOPC
5711 const MachineOperand *OMod = getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
5712 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
5713 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
5714 return false;
5715 }
5716 }
5717 }
5718
5719 const MachineOperand *DstUnused = getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
5720 if (DstUnused && DstUnused->isImm() &&
5721 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
5722 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5723 if (!Dst.isReg() || !Dst.isTied()) {
5724 ErrInfo = "Dst register should have tied register";
5725 return false;
5726 }
5727
5728 const MachineOperand &TiedMO =
5729 MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: DstIdx));
5730 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
5731 ErrInfo =
5732 "Dst register should be tied to implicit use of preserved register";
5733 return false;
5734 }
5735 if (TiedMO.getReg().isPhysical() && Dst.getReg() != TiedMO.getReg()) {
5736 ErrInfo = "Dst register should use same physical register as preserved";
5737 return false;
5738 }
5739 }
5740 }
5741
5742 if (isDPP(MI) && !ST.hasDPPSrc1SGPR() && Src1Idx != -1) {
5743 const MachineOperand &Src1MO = MI.getOperand(i: Src1Idx);
5744 if (Src1MO.isReg() && RI.isSGPRReg(MRI, Reg: Src1MO.getReg())) {
5745 ErrInfo = "DPP src1 cannot be SGPR on this subtarget";
5746 return false;
5747 }
5748 if (Src1MO.isImm()) {
5749 ErrInfo = "DPP src1 cannot be an immediate on this subtarget";
5750 return false;
5751 }
5752 }
5753
5754 // Verify MIMG / VIMAGE / VSAMPLE
5755 if (isImage(Opcode) && !MI.mayStore()) {
5756 // Ensure that the return type used is large enough for all the options
5757 // being used TFE/LWE require an extra result register.
5758 const MachineOperand *DMask = getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
5759 if (DMask) {
5760 uint64_t DMaskImm = DMask->getImm();
5761 uint32_t RegCount = isGather4(Opcode) ? 4 : llvm::popcount(Value: DMaskImm);
5762 const MachineOperand *TFE = getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
5763 const MachineOperand *LWE = getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
5764 const MachineOperand *D16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
5765
5766 // Adjust for packed 16 bit values
5767 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
5768 RegCount = divideCeil(Numerator: RegCount, Denominator: 2);
5769
5770 // Adjust if using LWE or TFE
5771 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
5772 RegCount += 1;
5773
5774 const uint32_t DstIdx =
5775 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::vdata);
5776 const MachineOperand &Dst = MI.getOperand(i: DstIdx);
5777 if (Dst.isReg()) {
5778 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: DstIdx);
5779 uint32_t DstSize = RI.getRegSizeInBits(RC: *DstRC) / 32;
5780 if (RegCount > DstSize) {
5781 ErrInfo = "Image instruction returns too many registers for dst "
5782 "register class";
5783 return false;
5784 }
5785 }
5786 }
5787 }
5788
5789 // Verify VOP*. Ignore multiple sgpr operands on writelane.
5790 if (isVALU(MI, /*AllowLDSDMA=*/false) &&
5791 Desc.getOpcode() != AMDGPU::V_WRITELANE_B32) {
5792 unsigned ConstantBusCount = 0;
5793 bool UsesLiteral = false;
5794 const MachineOperand *LiteralVal = nullptr;
5795
5796 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::imm);
5797 if (ImmIdx != -1) {
5798 ++ConstantBusCount;
5799 UsesLiteral = true;
5800 LiteralVal = &MI.getOperand(i: ImmIdx);
5801 }
5802
5803 SmallVector<Register, 2> SGPRsUsed;
5804 Register SGPRUsed;
5805
5806 // Only look at the true operands. Only a real operand can use the constant
5807 // bus, and we don't want to check pseudo-operands like the source modifier
5808 // flags.
5809 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx, Src3Idx}) {
5810 if (OpIdx == -1)
5811 continue;
5812 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5813 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5814 if (MO.isReg()) {
5815 SGPRUsed = MO.getReg();
5816 if (!llvm::is_contained(Range&: SGPRsUsed, Element: SGPRUsed)) {
5817 ++ConstantBusCount;
5818 SGPRsUsed.push_back(Elt: SGPRUsed);
5819 }
5820 } else if (!MO.isFI()) { // Treat FI like a register.
5821 if (!UsesLiteral) {
5822 ++ConstantBusCount;
5823 UsesLiteral = true;
5824 LiteralVal = &MO;
5825 } else if (!MO.isIdenticalTo(Other: *LiteralVal)) {
5826 assert(isVOP2(MI) || isVOP3(MI));
5827 ErrInfo = "VOP2/VOP3 instruction uses more than one literal";
5828 return false;
5829 }
5830 }
5831 }
5832 }
5833
5834 SGPRUsed = findImplicitSGPRRead(MI);
5835 if (SGPRUsed) {
5836 // Implicit uses may safely overlap true operands
5837 if (llvm::all_of(Range&: SGPRsUsed, P: [this, SGPRUsed](unsigned SGPR) {
5838 return !RI.regsOverlap(RegA: SGPRUsed, RegB: SGPR);
5839 })) {
5840 ++ConstantBusCount;
5841 SGPRsUsed.push_back(Elt: SGPRUsed);
5842 }
5843 }
5844
5845 // v_writelane_b32 is an exception from constant bus restriction:
5846 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
5847 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
5848 Opcode != AMDGPU::V_WRITELANE_B32) {
5849 ErrInfo = "VOP* instruction violates constant bus restriction";
5850 return false;
5851 }
5852
5853 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) {
5854 ErrInfo = "VOP3 instruction uses literal";
5855 return false;
5856 }
5857 }
5858
5859 // Special case for writelane - this can break the multiple constant bus rule,
5860 // but still can't use more than one SGPR register
5861 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
5862 unsigned SGPRCount = 0;
5863 Register SGPRUsed;
5864
5865 for (int OpIdx : {Src0Idx, Src1Idx}) {
5866 if (OpIdx == -1)
5867 break;
5868
5869 const MachineOperand &MO = MI.getOperand(i: OpIdx);
5870
5871 if (usesConstantBus(MRI, MO, OpInfo: MI.getDesc().operands()[OpIdx])) {
5872 if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
5873 if (MO.getReg() != SGPRUsed)
5874 ++SGPRCount;
5875 SGPRUsed = MO.getReg();
5876 }
5877 }
5878 if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
5879 ErrInfo = "WRITELANE instruction violates constant bus restriction";
5880 return false;
5881 }
5882 }
5883 }
5884
5885 // Verify misc. restrictions on specific instructions.
5886 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 ||
5887 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) {
5888 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5889 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5890 const MachineOperand &Src2 = MI.getOperand(i: Src2Idx);
5891 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
5892 if (!compareMachineOp(Op0: Src0, Op1: Src1) &&
5893 !compareMachineOp(Op0: Src0, Op1: Src2)) {
5894 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
5895 return false;
5896 }
5897 }
5898 if ((getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm() &
5899 SISrcMods::ABS) ||
5900 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm() &
5901 SISrcMods::ABS) ||
5902 (getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm() &
5903 SISrcMods::ABS)) {
5904 ErrInfo = "ABS not allowed in VOP3B instructions";
5905 return false;
5906 }
5907 }
5908
5909 if (isSOP2(MI) || isSOPC(MI)) {
5910 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5911 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
5912
5913 if (!isRegOrFI(MO: Src0) && !isRegOrFI(MO: Src1) &&
5914 !isInlineConstant(MO: Src0, OpInfo: Desc.operands()[Src0Idx]) &&
5915 !isInlineConstant(MO: Src1, OpInfo: Desc.operands()[Src1Idx]) &&
5916 !Src0.isIdenticalTo(Other: Src1)) {
5917 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
5918 return false;
5919 }
5920 }
5921
5922 if (isSOPK(MI)) {
5923 const auto *Op = getNamedOperand(MI, OperandName: AMDGPU::OpName::simm16);
5924 if (Desc.isBranch()) {
5925 if (!Op->isMBB()) {
5926 ErrInfo = "invalid branch target for SOPK instruction";
5927 return false;
5928 }
5929 } else {
5930 uint64_t Imm = Op->getImm();
5931 if (sopkIsZext(Opcode)) {
5932 if (!isUInt<16>(x: Imm)) {
5933 ErrInfo = "invalid immediate for SOPK instruction";
5934 return false;
5935 }
5936 } else {
5937 if (!isInt<16>(x: Imm)) {
5938 ErrInfo = "invalid immediate for SOPK instruction";
5939 return false;
5940 }
5941 }
5942 }
5943 }
5944
5945 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
5946 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
5947 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5948 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
5949 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
5950 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
5951
5952 const unsigned StaticNumOps =
5953 Desc.getNumOperands() + Desc.implicit_uses().size();
5954 const unsigned NumImplicitOps = IsDst ? 2 : 1;
5955
5956 // Require additional implicit operands. This allows a fixup done by the
5957 // post RA scheduler where the main implicit operand is killed and
5958 // implicit-defs are added for sub-registers that remain live after this
5959 // instruction.
5960 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
5961 ErrInfo = "missing implicit register operands";
5962 return false;
5963 }
5964
5965 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
5966 if (IsDst) {
5967 if (!Dst->isUse()) {
5968 ErrInfo = "v_movreld_b32 vdst should be a use operand";
5969 return false;
5970 }
5971
5972 unsigned UseOpIdx;
5973 if (!MI.isRegTiedToUseOperand(DefOpIdx: StaticNumOps, UseOpIdx: &UseOpIdx) ||
5974 UseOpIdx != StaticNumOps + 1) {
5975 ErrInfo = "movrel implicit operands should be tied";
5976 return false;
5977 }
5978 }
5979
5980 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
5981 const MachineOperand &ImpUse
5982 = MI.getOperand(i: StaticNumOps + NumImplicitOps - 1);
5983 if (!ImpUse.isReg() || !ImpUse.isUse() ||
5984 !isSubRegOf(TRI: RI, SuperVec: ImpUse, SubReg: IsDst ? *Dst : Src0)) {
5985 ErrInfo = "src0 should be subreg of implicit vector use";
5986 return false;
5987 }
5988 }
5989
5990 // Make sure we aren't losing exec uses in the td files. This mostly requires
5991 // being careful when using let Uses to try to add other use registers.
5992 if (shouldReadExec(MI)) {
5993 if (!MI.hasRegisterImplicitUseOperand(Reg: AMDGPU::EXEC)) {
5994 ErrInfo = "VALU instruction does not implicitly read exec mask";
5995 return false;
5996 }
5997 }
5998
5999 if (isSMRD(MI)) {
6000 if (MI.mayStore() &&
6001 ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) {
6002 // The register offset form of scalar stores may only use m0 as the
6003 // soffset register.
6004 const MachineOperand *Soff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
6005 if (Soff && Soff->getReg() != AMDGPU::M0) {
6006 ErrInfo = "scalar stores must use m0 as offset register";
6007 return false;
6008 }
6009 }
6010 }
6011
6012 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) {
6013 const MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
6014 if (Offset->getImm() != 0) {
6015 ErrInfo = "subtarget does not support offsets in flat instructions";
6016 return false;
6017 }
6018 }
6019
6020 if (isDS(MI) && !ST.hasGDS()) {
6021 const MachineOperand *GDSOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::gds);
6022 if (GDSOp && GDSOp->getImm() != 0) {
6023 ErrInfo = "GDS is not supported on this subtarget";
6024 return false;
6025 }
6026 }
6027
6028 if (isImage(MI)) {
6029 const MachineOperand *DimOp = getNamedOperand(MI, OperandName: AMDGPU::OpName::dim);
6030 if (DimOp) {
6031 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
6032 Name: AMDGPU::OpName::vaddr0);
6033 AMDGPU::OpName RSrcOpName =
6034 isMIMG(MI) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
6035 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: RSrcOpName);
6036 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: Opcode);
6037 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
6038 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
6039 const AMDGPU::MIMGDimInfo *Dim =
6040 AMDGPU::getMIMGDimInfoByEncoding(DimEnc: DimOp->getImm());
6041
6042 if (!Dim) {
6043 ErrInfo = "dim is out of range";
6044 return false;
6045 }
6046
6047 bool IsA16 = false;
6048 if (ST.hasR128A16()) {
6049 const MachineOperand *R128A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::r128);
6050 IsA16 = R128A16->getImm() != 0;
6051 } else if (ST.hasA16()) {
6052 const MachineOperand *A16 = getNamedOperand(MI, OperandName: AMDGPU::OpName::a16);
6053 IsA16 = A16->getImm() != 0;
6054 }
6055
6056 bool IsNSA = RsrcIdx - VAddr0Idx > 1;
6057
6058 unsigned AddrWords =
6059 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, IsG16Supported: ST.hasG16());
6060
6061 unsigned VAddrWords;
6062 if (IsNSA) {
6063 VAddrWords = RsrcIdx - VAddr0Idx;
6064 if (ST.hasPartialNSAEncoding() &&
6065 AddrWords > ST.getNSAMaxSize(HasSampler: isVSAMPLE(MI))) {
6066 unsigned LastVAddrIdx = RsrcIdx - 1;
6067 VAddrWords += getOpSize(MI, OpNo: LastVAddrIdx) / 4 - 1;
6068 }
6069 } else {
6070 VAddrWords = getOpSize(MI, OpNo: VAddr0Idx) / 4;
6071 if (AddrWords > 12)
6072 AddrWords = 16;
6073 }
6074
6075 if (VAddrWords != AddrWords) {
6076 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords
6077 << " but got " << VAddrWords << "\n");
6078 ErrInfo = "bad vaddr size";
6079 return false;
6080 }
6081 }
6082 }
6083
6084 const MachineOperand *DppCt = getNamedOperand(MI, OperandName: AMDGPU::OpName::dpp_ctrl);
6085 if (DppCt) {
6086 using namespace AMDGPU::DPP;
6087
6088 unsigned DC = DppCt->getImm();
6089 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
6090 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
6091 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) ||
6092 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) ||
6093 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) ||
6094 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) ||
6095 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) {
6096 ErrInfo = "Invalid dpp_ctrl value";
6097 return false;
6098 }
6099 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
6100 !ST.hasDPPWavefrontShifts()) {
6101 ErrInfo = "Invalid dpp_ctrl value: "
6102 "wavefront shifts are not supported on GFX10+";
6103 return false;
6104 }
6105 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
6106 !ST.hasDPPBroadcasts()) {
6107 ErrInfo = "Invalid dpp_ctrl value: "
6108 "broadcasts are not supported on GFX10+";
6109 return false;
6110 }
6111 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST &&
6112 ST.getGeneration() < AMDGPUSubtarget::GFX10) {
6113 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST &&
6114 DC <= DppCtrl::ROW_NEWBCAST_LAST &&
6115 !ST.hasGFX90AInsts()) {
6116 ErrInfo = "Invalid dpp_ctrl value: "
6117 "row_newbroadcast/row_share is not supported before "
6118 "GFX90A/GFX10";
6119 return false;
6120 }
6121 if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) {
6122 ErrInfo = "Invalid dpp_ctrl value: "
6123 "row_share and row_xmask are not supported before GFX10";
6124 return false;
6125 }
6126 }
6127
6128 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
6129 !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&
6130 ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
6131 AMDGPU::isDPALU_DPP(OpDesc: Desc, MII: *this, ST)) {
6132 ErrInfo = "Invalid dpp_ctrl value: "
6133 "DP ALU dpp only support row_newbcast";
6134 return false;
6135 }
6136 }
6137
6138 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) {
6139 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
6140 AMDGPU::OpName DataName =
6141 isDS(Opcode) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata;
6142 const MachineOperand *Data = getNamedOperand(MI, OperandName: DataName);
6143 const MachineOperand *Data2 = getNamedOperand(MI, OperandName: AMDGPU::OpName::data1);
6144 if (Data && !Data->isReg())
6145 Data = nullptr;
6146
6147 if (!ST.hasGFX90AInsts()) {
6148 if ((Dst && RI.isAGPR(MRI, Reg: Dst->getReg())) ||
6149 (Data && RI.isAGPR(MRI, Reg: Data->getReg())) ||
6150 (Data2 && RI.isAGPR(MRI, Reg: Data2->getReg()))) {
6151 ErrInfo = "Invalid register class: "
6152 "agpr loads and stores not supported on this GPU";
6153 return false;
6154 }
6155 }
6156 }
6157
6158 if (ST.needsAlignedVGPRs()) {
6159 const auto isAlignedReg = [&MI, &MRI, this](AMDGPU::OpName OpName) -> bool {
6160 const MachineOperand *Op = getNamedOperand(MI, OperandName: OpName);
6161 if (!Op)
6162 return true;
6163 Register Reg = Op->getReg();
6164 if (Reg.isPhysical())
6165 return !(RI.getHWRegIndex(Reg) & 1);
6166 const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
6167 return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
6168 !(RI.getChannelFromSubReg(SubReg: Op->getSubReg()) & 1);
6169 };
6170
6171 if (isMIMG(MI)) {
6172 if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
6173 ErrInfo = "Subtarget requires even aligned vector registers "
6174 "for vaddr operand of image instructions";
6175 return false;
6176 }
6177 }
6178 }
6179
6180 if (Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts()) {
6181 const MachineOperand *Src = getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
6182 if (Src->isReg() && RI.isSGPRReg(MRI, Reg: Src->getReg())) {
6183 ErrInfo = "Invalid register class: "
6184 "v_accvgpr_write with an SGPR is not supported on this GPU";
6185 return false;
6186 }
6187 }
6188
6189 if (Desc.getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS) {
6190 const MachineOperand &SrcOp = MI.getOperand(i: 1);
6191 if (!SrcOp.isReg() || SrcOp.getReg().isVirtual()) {
6192 ErrInfo = "pseudo expects only physical SGPRs";
6193 return false;
6194 }
6195 }
6196
6197 if (const MachineOperand *CPol = getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
6198 if (CPol->getImm() & AMDGPU::CPol::SCAL) {
6199 if (!ST.hasScaleOffset()) {
6200 ErrInfo = "Subtarget does not support offset scaling";
6201 return false;
6202 }
6203 if (!AMDGPU::supportsScaleOffset(MII: *this, Opcode: MI.getOpcode())) {
6204 ErrInfo = "Instruction does not support offset scaling";
6205 return false;
6206 }
6207 }
6208 }
6209
6210 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6211 if (AMDGPU::isSingleSGPRReadInst(Opc: Opcode)) {
6212 for (unsigned I = 0; I < 3; ++I) {
6213 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: I))
6214 return false;
6215 }
6216 }
6217
6218 if (ST.hasFlatScratchHiInB64InstHazard() && isSALU(MI) &&
6219 MI.readsRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI: nullptr)) {
6220 const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
6221 if ((Dst && RI.getRegClassForReg(MRI, Reg: Dst->getReg()) ==
6222 &AMDGPU::SReg_64RegClass) ||
6223 Opcode == AMDGPU::S_BITCMP0_B64 || Opcode == AMDGPU::S_BITCMP1_B64) {
6224 ErrInfo = "Instruction cannot read flat_scratch_base_hi";
6225 return false;
6226 }
6227 }
6228
6229 return true;
6230}
6231
6232unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
6233 if (MI.getOpcode() == AMDGPU::S_MOV_B32) {
6234 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6235 return MI.getOperand(i: 1).isReg() || RI.isAGPR(MRI, Reg: MI.getOperand(i: 0).getReg())
6236 ? AMDGPU::COPY
6237 : AMDGPU::V_MOV_B32_e32;
6238 }
6239 return getVALUOp(Opc: MI.getOpcode());
6240}
6241
6242// It is more readable to list mapped opcodes on the same line.
6243// clang-format off
6244
6245unsigned SIInstrInfo::getVALUOp(unsigned Opc) const {
6246 switch (Opc) {
6247 default: return AMDGPU::INSTRUCTION_LIST_END;
6248 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
6249 case AMDGPU::COPY: return AMDGPU::COPY;
6250 case AMDGPU::PHI: return AMDGPU::PHI;
6251 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
6252 case AMDGPU::WQM: return AMDGPU::WQM;
6253 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
6254 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM;
6255 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM;
6256 case AMDGPU::S_ADD_I32:
6257 return ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
6258 case AMDGPU::S_ADDC_U32:
6259 return AMDGPU::V_ADDC_U32_e32;
6260 case AMDGPU::S_SUB_I32:
6261 return ST.hasAddNoCarryInsts() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32;
6262 // FIXME: These are not consistently handled, and selected when the carry is
6263 // used.
6264 case AMDGPU::S_ADD_U32:
6265 return AMDGPU::V_ADD_CO_U32_e32;
6266 case AMDGPU::S_SUB_U32:
6267 return AMDGPU::V_SUB_CO_U32_e32;
6268 case AMDGPU::S_ADD_U64_PSEUDO:
6269 return AMDGPU::V_ADD_U64_PSEUDO;
6270 case AMDGPU::S_SUB_U64_PSEUDO:
6271 return AMDGPU::V_SUB_U64_PSEUDO;
6272 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
6273 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64;
6274 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64;
6275 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64;
6276 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
6277 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
6278 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
6279 case AMDGPU::S_XNOR_B32:
6280 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
6281 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
6282 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
6283 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
6284 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
6285 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
6286 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64;
6287 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
6288 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64;
6289 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
6290 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64;
6291 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64;
6292 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64;
6293 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64;
6294 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64;
6295 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
6296 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
6297 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
6298 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
6299 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e64;
6300 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e64;
6301 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e64;
6302 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e64;
6303 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e64;
6304 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e64;
6305 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e64;
6306 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e64;
6307 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e64;
6308 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e64;
6309 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e64;
6310 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e64;
6311 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e64;
6312 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e64;
6313 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
6314 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
6315 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
6316 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
6317 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
6318 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
6319 case AMDGPU::S_CVT_F32_I32: return AMDGPU::V_CVT_F32_I32_e64;
6320 case AMDGPU::S_CVT_F32_U32: return AMDGPU::V_CVT_F32_U32_e64;
6321 case AMDGPU::S_CVT_I32_F32: return AMDGPU::V_CVT_I32_F32_e64;
6322 case AMDGPU::S_CVT_U32_F32: return AMDGPU::V_CVT_U32_F32_e64;
6323 case AMDGPU::S_CVT_F32_F16:
6324 case AMDGPU::S_CVT_HI_F32_F16:
6325 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F32_F16_t16_e64
6326 : AMDGPU::V_CVT_F32_F16_fake16_e64;
6327 case AMDGPU::S_CVT_F16_F32:
6328 return ST.useRealTrue16Insts() ? AMDGPU::V_CVT_F16_F32_t16_e64
6329 : AMDGPU::V_CVT_F16_F32_fake16_e64;
6330 case AMDGPU::S_CEIL_F32: return AMDGPU::V_CEIL_F32_e64;
6331 case AMDGPU::S_FLOOR_F32: return AMDGPU::V_FLOOR_F32_e64;
6332 case AMDGPU::S_TRUNC_F32: return AMDGPU::V_TRUNC_F32_e64;
6333 case AMDGPU::S_RNDNE_F32: return AMDGPU::V_RNDNE_F32_e64;
6334 case AMDGPU::S_CEIL_F16:
6335 return ST.useRealTrue16Insts() ? AMDGPU::V_CEIL_F16_t16_e64
6336 : AMDGPU::V_CEIL_F16_fake16_e64;
6337 case AMDGPU::S_FLOOR_F16:
6338 return ST.useRealTrue16Insts() ? AMDGPU::V_FLOOR_F16_t16_e64
6339 : AMDGPU::V_FLOOR_F16_fake16_e64;
6340 case AMDGPU::S_TRUNC_F16:
6341 return ST.useRealTrue16Insts() ? AMDGPU::V_TRUNC_F16_t16_e64
6342 : AMDGPU::V_TRUNC_F16_fake16_e64;
6343 case AMDGPU::S_RNDNE_F16:
6344 return ST.useRealTrue16Insts() ? AMDGPU::V_RNDNE_F16_t16_e64
6345 : AMDGPU::V_RNDNE_F16_fake16_e64;
6346 case AMDGPU::S_ADD_F32: return AMDGPU::V_ADD_F32_e64;
6347 case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
6348 case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
6349 case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
6350 case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
6351 case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
6352 case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
6353 case AMDGPU::S_ADD_F16:
6354 return ST.useRealTrue16Insts() ? AMDGPU::V_ADD_F16_t16_e64
6355 : AMDGPU::V_ADD_F16_fake16_e64;
6356 case AMDGPU::S_SUB_F16:
6357 return ST.useRealTrue16Insts() ? AMDGPU::V_SUB_F16_t16_e64
6358 : AMDGPU::V_SUB_F16_fake16_e64;
6359 case AMDGPU::S_MIN_F16:
6360 return ST.useRealTrue16Insts() ? AMDGPU::V_MIN_F16_t16_e64
6361 : AMDGPU::V_MIN_F16_fake16_e64;
6362 case AMDGPU::S_MAX_F16:
6363 return ST.useRealTrue16Insts() ? AMDGPU::V_MAX_F16_t16_e64
6364 : AMDGPU::V_MAX_F16_fake16_e64;
6365 case AMDGPU::S_MINIMUM_F16:
6366 return ST.useRealTrue16Insts() ? AMDGPU::V_MINIMUM_F16_t16_e64
6367 : AMDGPU::V_MINIMUM_F16_fake16_e64;
6368 case AMDGPU::S_MAXIMUM_F16:
6369 return ST.useRealTrue16Insts() ? AMDGPU::V_MAXIMUM_F16_t16_e64
6370 : AMDGPU::V_MAXIMUM_F16_fake16_e64;
6371 case AMDGPU::S_MUL_F16:
6372 return ST.useRealTrue16Insts() ? AMDGPU::V_MUL_F16_t16_e64
6373 : AMDGPU::V_MUL_F16_fake16_e64;
6374 case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
6375 case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
6376 case AMDGPU::S_FMAC_F16:
6377 return ST.useRealTrue16Insts() ? AMDGPU::V_FMAC_F16_t16_e64
6378 : AMDGPU::V_FMAC_F16_fake16_e64;
6379 case AMDGPU::S_FMAMK_F32: return AMDGPU::V_FMAMK_F32;
6380 case AMDGPU::S_FMAAK_F32: return AMDGPU::V_FMAAK_F32;
6381 case AMDGPU::S_CMP_LT_F32: return AMDGPU::V_CMP_LT_F32_e64;
6382 case AMDGPU::S_CMP_EQ_F32: return AMDGPU::V_CMP_EQ_F32_e64;
6383 case AMDGPU::S_CMP_LE_F32: return AMDGPU::V_CMP_LE_F32_e64;
6384 case AMDGPU::S_CMP_GT_F32: return AMDGPU::V_CMP_GT_F32_e64;
6385 case AMDGPU::S_CMP_LG_F32: return AMDGPU::V_CMP_LG_F32_e64;
6386 case AMDGPU::S_CMP_GE_F32: return AMDGPU::V_CMP_GE_F32_e64;
6387 case AMDGPU::S_CMP_O_F32: return AMDGPU::V_CMP_O_F32_e64;
6388 case AMDGPU::S_CMP_U_F32: return AMDGPU::V_CMP_U_F32_e64;
6389 case AMDGPU::S_CMP_NGE_F32: return AMDGPU::V_CMP_NGE_F32_e64;
6390 case AMDGPU::S_CMP_NLG_F32: return AMDGPU::V_CMP_NLG_F32_e64;
6391 case AMDGPU::S_CMP_NGT_F32: return AMDGPU::V_CMP_NGT_F32_e64;
6392 case AMDGPU::S_CMP_NLE_F32: return AMDGPU::V_CMP_NLE_F32_e64;
6393 case AMDGPU::S_CMP_NEQ_F32: return AMDGPU::V_CMP_NEQ_F32_e64;
6394 case AMDGPU::S_CMP_NLT_F32: return AMDGPU::V_CMP_NLT_F32_e64;
6395 case AMDGPU::S_CMP_LT_F16:
6396 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LT_F16_t16_e64
6397 : AMDGPU::V_CMP_LT_F16_fake16_e64;
6398 case AMDGPU::S_CMP_EQ_F16:
6399 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_EQ_F16_t16_e64
6400 : AMDGPU::V_CMP_EQ_F16_fake16_e64;
6401 case AMDGPU::S_CMP_LE_F16:
6402 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LE_F16_t16_e64
6403 : AMDGPU::V_CMP_LE_F16_fake16_e64;
6404 case AMDGPU::S_CMP_GT_F16:
6405 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GT_F16_t16_e64
6406 : AMDGPU::V_CMP_GT_F16_fake16_e64;
6407 case AMDGPU::S_CMP_LG_F16:
6408 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_LG_F16_t16_e64
6409 : AMDGPU::V_CMP_LG_F16_fake16_e64;
6410 case AMDGPU::S_CMP_GE_F16:
6411 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_GE_F16_t16_e64
6412 : AMDGPU::V_CMP_GE_F16_fake16_e64;
6413 case AMDGPU::S_CMP_O_F16:
6414 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_O_F16_t16_e64
6415 : AMDGPU::V_CMP_O_F16_fake16_e64;
6416 case AMDGPU::S_CMP_U_F16:
6417 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_U_F16_t16_e64
6418 : AMDGPU::V_CMP_U_F16_fake16_e64;
6419 case AMDGPU::S_CMP_NGE_F16:
6420 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGE_F16_t16_e64
6421 : AMDGPU::V_CMP_NGE_F16_fake16_e64;
6422 case AMDGPU::S_CMP_NLG_F16:
6423 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLG_F16_t16_e64
6424 : AMDGPU::V_CMP_NLG_F16_fake16_e64;
6425 case AMDGPU::S_CMP_NGT_F16:
6426 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NGT_F16_t16_e64
6427 : AMDGPU::V_CMP_NGT_F16_fake16_e64;
6428 case AMDGPU::S_CMP_NLE_F16:
6429 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLE_F16_t16_e64
6430 : AMDGPU::V_CMP_NLE_F16_fake16_e64;
6431 case AMDGPU::S_CMP_NEQ_F16:
6432 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NEQ_F16_t16_e64
6433 : AMDGPU::V_CMP_NEQ_F16_fake16_e64;
6434 case AMDGPU::S_CMP_NLT_F16:
6435 return ST.useRealTrue16Insts() ? AMDGPU::V_CMP_NLT_F16_t16_e64
6436 : AMDGPU::V_CMP_NLT_F16_fake16_e64;
6437 case AMDGPU::V_S_EXP_F32_e64: return AMDGPU::V_EXP_F32_e64;
6438 case AMDGPU::V_S_EXP_F16_e64:
6439 return ST.useRealTrue16Insts() ? AMDGPU::V_EXP_F16_t16_e64
6440 : AMDGPU::V_EXP_F16_fake16_e64;
6441 case AMDGPU::V_S_LOG_F32_e64: return AMDGPU::V_LOG_F32_e64;
6442 case AMDGPU::V_S_LOG_F16_e64:
6443 return ST.useRealTrue16Insts() ? AMDGPU::V_LOG_F16_t16_e64
6444 : AMDGPU::V_LOG_F16_fake16_e64;
6445 case AMDGPU::V_S_RCP_F32_e64: return AMDGPU::V_RCP_F32_e64;
6446 case AMDGPU::V_S_RCP_F16_e64:
6447 return ST.useRealTrue16Insts() ? AMDGPU::V_RCP_F16_t16_e64
6448 : AMDGPU::V_RCP_F16_fake16_e64;
6449 case AMDGPU::V_S_RSQ_F32_e64: return AMDGPU::V_RSQ_F32_e64;
6450 case AMDGPU::V_S_RSQ_F16_e64:
6451 return ST.useRealTrue16Insts() ? AMDGPU::V_RSQ_F16_t16_e64
6452 : AMDGPU::V_RSQ_F16_fake16_e64;
6453 case AMDGPU::V_S_SQRT_F32_e64: return AMDGPU::V_SQRT_F32_e64;
6454 case AMDGPU::V_S_SQRT_F16_e64:
6455 return ST.useRealTrue16Insts() ? AMDGPU::V_SQRT_F16_t16_e64
6456 : AMDGPU::V_SQRT_F16_fake16_e64;
6457 }
6458 llvm_unreachable(
6459 "Unexpected scalar opcode without corresponding vector one!");
6460}
6461
6462// clang-format on
6463
6464void SIInstrInfo::insertScratchExecCopy(MachineFunction &MF,
6465 MachineBasicBlock &MBB,
6466 MachineBasicBlock::iterator MBBI,
6467 const DebugLoc &DL, Register Reg,
6468 bool IsSCCLive,
6469 SlotIndexes *Indexes) const {
6470 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
6471 const SIInstrInfo *TII = ST.getInstrInfo();
6472 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6473 if (IsSCCLive) {
6474 // Insert two move instructions, one to save the original value of EXEC and
6475 // the other to turn on all bits in EXEC. This is required as we can't use
6476 // the single instruction S_OR_SAVEEXEC that clobbers SCC.
6477 auto StoreExecMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: Reg)
6478 .addReg(RegNo: LMC.ExecReg, Flags: RegState::Kill);
6479 auto FlipExecMI =
6480 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
6481 if (Indexes) {
6482 Indexes->insertMachineInstrInMaps(MI&: *StoreExecMI);
6483 Indexes->insertMachineInstrInMaps(MI&: *FlipExecMI);
6484 }
6485 } else {
6486 auto SaveExec =
6487 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.OrSaveExecOpc), DestReg: Reg).addImm(Val: -1);
6488 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
6489 if (Indexes)
6490 Indexes->insertMachineInstrInMaps(MI&: *SaveExec);
6491 }
6492}
6493
6494void SIInstrInfo::restoreExec(MachineFunction &MF, MachineBasicBlock &MBB,
6495 MachineBasicBlock::iterator MBBI,
6496 const DebugLoc &DL, Register Reg,
6497 SlotIndexes *Indexes) const {
6498 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
6499 auto ExecRestoreMI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
6500 .addReg(RegNo: Reg, Flags: RegState::Kill);
6501 if (Indexes)
6502 Indexes->insertMachineInstrInMaps(MI&: *ExecRestoreMI);
6503}
6504
6505MachineInstr *
6506SIInstrInfo::getWholeWaveFunctionSetup(MachineFunction &MF) const {
6507 assert(MF.getInfo<SIMachineFunctionInfo>()->isWholeWaveFunction() &&
6508 "Not a whole wave func");
6509 MachineBasicBlock &MBB = *MF.begin();
6510 for (MachineInstr &MI : MBB)
6511 if (MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_SETUP ||
6512 MI.getOpcode() == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
6513 return &MI;
6514
6515 llvm_unreachable("Couldn't find SI_SETUP_WHOLE_WAVE_FUNC instruction");
6516}
6517
6518const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
6519 unsigned OpNo) const {
6520 const MCInstrDesc &Desc = get(Opcode: MI.getOpcode());
6521 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
6522 Desc.operands()[OpNo].RegClass == -1) {
6523 Register Reg = MI.getOperand(i: OpNo).getReg();
6524
6525 if (Reg.isVirtual()) {
6526 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6527 return MRI.getRegClass(Reg);
6528 }
6529 return RI.getPhysRegBaseClass(Reg);
6530 }
6531
6532 int16_t RegClass = getOpRegClassID(OpInfo: Desc.operands()[OpNo]);
6533 return RegClass < 0 ? nullptr : RI.getRegClass(i: RegClass);
6534}
6535
6536// Convert VOP3 operand index to source number.
6537static unsigned VOP3OpIdxToSrcN(const MachineInstr &MI, unsigned OpIdx) {
6538 constexpr AMDGPU::OpName OpNames[] = {
6539 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
6540
6541 for (auto [I, OpName] : enumerate(First: OpNames)) {
6542 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[I]);
6543 if (static_cast<unsigned>(SrcIdx) == OpIdx)
6544 return I;
6545 }
6546
6547 return UINT_MAX;
6548}
6549
6550void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
6551 MachineBasicBlock::iterator I = MI;
6552 MachineBasicBlock *MBB = MI.getParent();
6553 MachineOperand &MO = MI.getOperand(i: OpIdx);
6554 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
6555 unsigned RCID = getOpRegClassID(OpInfo: get(Opcode: MI.getOpcode()).operands()[OpIdx]);
6556 const TargetRegisterClass *RC = RI.getRegClass(i: RCID);
6557 unsigned Size = RI.getRegSizeInBits(RC: *RC);
6558 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO
6559 : Size == 16 ? AMDGPU::V_MOV_B16_t16_e64
6560 : AMDGPU::V_MOV_B32_e32;
6561 if (MO.isReg())
6562 Opcode = AMDGPU::COPY;
6563 else if (RI.isSGPRClass(RC))
6564 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
6565
6566 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: RC);
6567 Register Reg = MRI.createVirtualRegister(RegClass: VRC);
6568 DebugLoc DL = MBB->findDebugLoc(MBBI: I);
6569
6570 if (Size == 128 && AMDGPU::isPackedSingleSGPR64BitInst(Opc: MI.getOpcode()) &&
6571 isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx))) {
6572 // Special case for V_PK_*64 instructions: these do not have OPSEL but SGPR
6573 // sources behave like OPSEL is set replicating low 64-bits into high. VGPR
6574 // sources in turn read actual 4 registers. To move operand from an SGPR to
6575 // a VGPR we need to replicate low half.
6576 // We also do not select immediates for these instructions so it always has
6577 // to be an SGPR register here.
6578 // Operands which are not legal as per isLegalSingleSGPRReadInstOperand()
6579 // sent here specifically to fix a non-splat SGPR and shall perform a full
6580 // copy.
6581
6582 const TargetRegisterClass *VRC64 = RI.getVGPRClassForBitWidth(BitWidth: 64);
6583 Register Low64 = MRI.createVirtualRegister(RegClass: VRC64);
6584 assert(MO.isReg() && RI.isSGPRReg(MRI, MO.getReg()));
6585 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Low64)
6586 .addReg(RegNo: MO.getReg(), Flags: {}, SubReg: AMDGPU::sub0_sub1);
6587 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Reg)
6588 .addReg(RegNo: Low64)
6589 .addImm(Val: AMDGPU::sub0_sub1)
6590 .addReg(RegNo: Low64)
6591 .addImm(Val: AMDGPU::sub2_sub3);
6592 } else if (Opcode == AMDGPU::V_MOV_B16_t16_e64) {
6593 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg)
6594 .addImm(Val: 0) // src0_modifiers
6595 .add(MO)
6596 .addImm(Val: 0); // op_sel
6597 } else {
6598 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: Reg).add(MO);
6599 }
6600
6601 MO.ChangeToRegister(Reg, isDef: false);
6602}
6603
6604unsigned SIInstrInfo::buildExtractSubReg(
6605 MachineBasicBlock::iterator MI, MachineRegisterInfo &MRI,
6606 const MachineOperand &SuperReg, const TargetRegisterClass *SuperRC,
6607 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6608 if (!SuperReg.getReg().isVirtual())
6609 return RI.getSubReg(Reg: SuperReg.getReg(), Idx: SubIdx);
6610
6611 MachineBasicBlock *MBB = MI->getParent();
6612 const DebugLoc &DL = MI->getDebugLoc();
6613 Register SubReg = MRI.createVirtualRegister(RegClass: SubRC);
6614
6615 unsigned NewSubIdx = RI.composeSubRegIndices(a: SuperReg.getSubReg(), b: SubIdx);
6616 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: SubReg)
6617 .addReg(RegNo: SuperReg.getReg(), Flags: {}, SubReg: NewSubIdx);
6618 return SubReg;
6619}
6620
6621MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
6622 MachineBasicBlock::iterator MII, MachineRegisterInfo &MRI,
6623 const MachineOperand &Op, const TargetRegisterClass *SuperRC,
6624 unsigned SubIdx, const TargetRegisterClass *SubRC) const {
6625 if (Op.isImm()) {
6626 if (SubIdx == AMDGPU::sub0)
6627 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm()));
6628 if (SubIdx == AMDGPU::sub1)
6629 return MachineOperand::CreateImm(Val: static_cast<int32_t>(Op.getImm() >> 32));
6630
6631 llvm_unreachable("Unhandled register index for immediate");
6632 }
6633
6634 unsigned SubReg = buildExtractSubReg(MI: MII, MRI, SuperReg: Op, SuperRC,
6635 SubIdx, SubRC);
6636 return MachineOperand::CreateReg(Reg: SubReg, isDef: false);
6637}
6638
6639// Change the order of operands from (0, 1, 2) to (0, 2, 1)
6640void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
6641 assert(Inst.getNumExplicitOperands() == 3);
6642 MachineOperand Op1 = Inst.getOperand(i: 1);
6643 Inst.removeOperand(OpNo: 1);
6644 Inst.addOperand(Op: Op1);
6645}
6646
6647bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
6648 const MCOperandInfo &OpInfo,
6649 const MachineOperand &MO) const {
6650 if (!MO.isReg())
6651 return false;
6652
6653 Register Reg = MO.getReg();
6654
6655 const TargetRegisterClass *DRC = RI.getRegClass(i: getOpRegClassID(OpInfo));
6656 if (Reg.isPhysical())
6657 return DRC->contains(Reg);
6658
6659 const TargetRegisterClass *RC = MRI.getRegClass(Reg);
6660
6661 if (MO.getSubReg()) {
6662 const TargetRegisterClass *SuperRC =
6663 RI.getLargestLegalSuperClass(RC, MF: MRI.getMF());
6664 if (!SuperRC)
6665 return false;
6666 return RI.getMatchingSuperRegClass(A: SuperRC, B: DRC, Idx: MO.getSubReg()) != nullptr;
6667 }
6668
6669 return RI.getCommonSubClass(A: DRC, B: RC) != nullptr;
6670}
6671
6672bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
6673 const MachineOperand &MO) const {
6674 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
6675 const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
6676 unsigned Opc = MI.getOpcode();
6677
6678 // See SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
6679 if (MO.isReg() && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6680 AMDGPU::isSingleSGPRReadInst(Opc: MI.getOpcode()) &&
6681 !isLegalSingleSGPRReadInstOperand(MRI, MI, SrcN: VOP3OpIdxToSrcN(MI, OpIdx),
6682 MO: &MO))
6683 return false;
6684
6685 if (!isLegalRegOperand(MRI, OpInfo, MO))
6686 return false;
6687
6688 // check Accumulate GPR operand
6689 bool IsAGPR = RI.isAGPR(MRI, Reg: MO.getReg());
6690 if (IsAGPR && !ST.hasMAIInsts())
6691 return false;
6692 if (IsAGPR && (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) &&
6693 (MI.mayLoad() || MI.mayStore() || isDS(Opcode: Opc) || isMIMG(Opcode: Opc)))
6694 return false;
6695 // Atomics should have both vdst and vdata either vgpr or agpr.
6696 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
6697 const int DataIdx = AMDGPU::getNamedOperandIdx(
6698 Opcode: Opc, Name: isDS(Opcode: Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata);
6699 if ((int)OpIdx == VDstIdx && DataIdx != -1 &&
6700 MI.getOperand(i: DataIdx).isReg() &&
6701 RI.isAGPR(MRI, Reg: MI.getOperand(i: DataIdx).getReg()) != IsAGPR)
6702 return false;
6703 if ((int)OpIdx == DataIdx) {
6704 if (VDstIdx != -1 &&
6705 RI.isAGPR(MRI, Reg: MI.getOperand(i: VDstIdx).getReg()) != IsAGPR)
6706 return false;
6707 // DS instructions with 2 src operands also must have tied RC.
6708 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data1);
6709 if (Data1Idx != -1 && MI.getOperand(i: Data1Idx).isReg() &&
6710 RI.isAGPR(MRI, Reg: MI.getOperand(i: Data1Idx).getReg()) != IsAGPR)
6711 return false;
6712 }
6713
6714 // Check V_ACCVGPR_WRITE_B32_e64
6715 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && !ST.hasGFX90AInsts() &&
6716 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0) &&
6717 RI.isSGPRReg(MRI, Reg: MO.getReg()))
6718 return false;
6719
6720 if (ST.hasFlatScratchHiInB64InstHazard() &&
6721 MO.getReg() == AMDGPU::SRC_FLAT_SCRATCH_BASE_HI && isSALU(MI)) {
6722 if (const MachineOperand *Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
6723 if (AMDGPU::getRegBitWidth(RC: *RI.getRegClassForReg(MRI, Reg: Dst->getReg())) ==
6724 64)
6725 return false;
6726 }
6727 if (Opc == AMDGPU::S_BITCMP0_B64 || Opc == AMDGPU::S_BITCMP1_B64)
6728 return false;
6729 }
6730 if (!ST.hasDPPSrc1SGPR() && isDPP(MI) && RI.isSGPRReg(MRI, Reg: MO.getReg()) &&
6731 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1))
6732 return false;
6733
6734 return true;
6735}
6736
6737bool SIInstrInfo::isLegalSingleSGPRReadInstOperand(
6738 const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
6739 const MachineOperand *MO) const {
6740 constexpr unsigned NumOps = 3;
6741 constexpr AMDGPU::OpName OpNames[NumOps * 2] = {
6742 AMDGPU::OpName::src0, AMDGPU::OpName::src1,
6743 AMDGPU::OpName::src2, AMDGPU::OpName::src0_modifiers,
6744 AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
6745
6746 assert(SrcN < NumOps);
6747
6748 if (!MO) {
6749 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[SrcN]);
6750 if (SrcIdx == -1)
6751 return true;
6752 MO = &MI.getOperand(i: SrcIdx);
6753 }
6754
6755 if (!MO->isReg() || !RI.isSGPRReg(MRI, Reg: MO->getReg()))
6756 return true;
6757
6758 int ModsIdx =
6759 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpNames[NumOps + SrcN]);
6760 if (ModsIdx == -1)
6761 return false;
6762
6763 unsigned Mods = MI.getOperand(i: ModsIdx).getImm();
6764 bool OpSel = Mods & SISrcMods::OP_SEL_0;
6765 bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
6766
6767 return !OpSel && !OpSelHi;
6768}
6769
6770// Check if V_PK_FMA_F32 does not have the invalid opsel conditions.
6771bool SIInstrInfo::isLegalPkFMAF32OpSel(const MachineInstr &MI) const {
6772 if (!ST.hasPkF32InvalidOpSel() || MI.getOpcode() != AMDGPU::V_PK_FMA_F32)
6773 return true;
6774
6775 const MachineOperand *Src1Mods =
6776 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
6777 const MachineOperand *Src2Mods =
6778 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers);
6779
6780 auto HasOpSel0 = [](const MachineOperand *Mods) {
6781 return Mods->getImm() & SISrcMods::OP_SEL_0;
6782 };
6783
6784 // Illegal iff opsel[2:1] == 2'b01, 2'b10, or 2'b11
6785 return !HasOpSel0(Src1Mods) && !HasOpSel0(Src2Mods);
6786}
6787
6788void SIInstrInfo::decomposePkFMAF32(MachineInstr &MI,
6789 MachineRegisterInfo &MRI) const {
6790 assert(MI.getOpcode() == AMDGPU::V_PK_FMA_F32);
6791
6792 MachineBasicBlock &MBB = *MI.getParent();
6793 const DebugLoc &DL = MI.getDebugLoc();
6794
6795 MachineOperand *Srcs[3] = {getNamedOperand(MI, OperandName: AMDGPU::OpName::src0),
6796 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1),
6797 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)};
6798 const unsigned Mods[3] = {
6799 static_cast<unsigned>(
6800 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm()),
6801 static_cast<unsigned>(
6802 getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm()),
6803 static_cast<unsigned>(
6804 getNamedOperand(MI, OperandName: AMDGPU::OpName::src2_modifiers)->getImm())};
6805
6806 // extract op_sel / op_sel_hi
6807 bool OpSel[3][2];
6808 for (unsigned I = 0; I != 3; ++I) {
6809 OpSel[I][0] = Mods[I] & SISrcMods::OP_SEL_0;
6810 OpSel[I][1] = Mods[I] & SISrcMods::OP_SEL_1;
6811 }
6812
6813 // The packed op can read only one SGPR on the constant bus (limit is 1 on
6814 // gfx9), so all its SGPR sources are the same pair. Each scalar lane can also
6815 // read only one SGPR, but op_sel may make a lane read different halves of
6816 // that pair, e.g.
6817 // v_pk_fma_f32 v[0:1], s[0:1], 0, s[0:1] op_sel:[0,0,1] op_sel_hi:[1,1,0]
6818 // => v_fma_f32 v0, s0, 0, s1 / v_fma_f32 v1, s1, 0, s0
6819 // If so, copy the pair to a VGPR once before extracting the lanes, rather
6820 // than moving a scalar into a VGPR per lane.
6821 SmallVector<unsigned, 3> SGPRSrcs;
6822 bool DivergentSGPRs = false;
6823 for (unsigned I = 0; I != 3; ++I) {
6824 if (!Srcs[I]->isReg() || !RI.isSGPRReg(MRI, Reg: Srcs[I]->getReg()))
6825 continue;
6826 if (!SGPRSrcs.empty()) {
6827 unsigned First = SGPRSrcs.front();
6828 assert(Srcs[I]->getReg() == Srcs[First]->getReg() &&
6829 Srcs[I]->getSubReg() == Srcs[First]->getSubReg() &&
6830 "constant bus limit allows only one SGPR source");
6831 DivergentSGPRs |=
6832 OpSel[I][0] != OpSel[First][0] || OpSel[I][1] != OpSel[First][1];
6833 }
6834 SGPRSrcs.push_back(Elt: I);
6835 }
6836
6837 if (DivergentSGPRs) {
6838 unsigned First = SGPRSrcs.front();
6839 legalizeOpWithMove(MI, OpIdx: MI.getOperandNo(I: Srcs[First]));
6840 for (unsigned I : ArrayRef(SGPRSrcs).drop_front())
6841 Srcs[I]->ChangeToRegister(Reg: Srcs[First]->getReg(), /*isDef=*/false);
6842 }
6843
6844 // Extract the low or high lane of a packed VOP3P source operand.
6845 auto ExtractLane = [&](MachineOperand &Op, bool UseSub1) -> MachineOperand {
6846 unsigned SubIdx = (Op.isReg() && UseSub1) ? AMDGPU::sub1 : AMDGPU::sub0;
6847 const TargetRegisterClass *RC =
6848 Op.isReg() ? MRI.getRegClass(Reg: Op.getReg()) : nullptr;
6849 const TargetRegisterClass *SubRC = !Op.isReg() ? nullptr
6850 : RI.isVGPR(MRI, Reg: Op.getReg())
6851 ? RI.getVGPRClassForBitWidth(BitWidth: 32)
6852 : RI.getSGPRClassForBitWidth(BitWidth: 32);
6853 return buildExtractSubRegOrImm(MII: MI, MRI, Op, SuperRC: RC, SubIdx, SubRC);
6854 };
6855
6856 // neg_lo/neg_hi on the packed operand become a plain NEG modifier.
6857 auto LaneMods = [](unsigned Mods, bool Hi) -> unsigned {
6858 if (Hi)
6859 return (Mods & SISrcMods::NEG_HI) ? static_cast<unsigned>(SISrcMods::NEG)
6860 : 0u;
6861 return Mods & SISrcMods::NEG;
6862 };
6863
6864 auto BuildLane = [&](Register Dst, bool Hi) {
6865 MachineOperand L0 = ExtractLane(*Srcs[0], OpSel[0][Hi]);
6866 MachineOperand L1 = ExtractLane(*Srcs[1], OpSel[1][Hi]);
6867 MachineOperand L2 = ExtractLane(*Srcs[2], OpSel[2][Hi]);
6868 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_FMA_F32_e64), DestReg: Dst)
6869 .addImm(Val: LaneMods(Mods[0], Hi))
6870 .add(MO: L0)
6871 .addImm(Val: LaneMods(Mods[1], Hi))
6872 .add(MO: L1)
6873 .addImm(Val: LaneMods(Mods[2], Hi))
6874 .add(MO: L2)
6875 .addImm(Val: 0) // clamp
6876 .addImm(Val: 0); // omod
6877 };
6878
6879 const TargetRegisterClass *VGPR32 = RI.getVGPRClassForBitWidth(BitWidth: 32);
6880 Register Lo = MRI.createVirtualRegister(RegClass: VGPR32);
6881 Register Hi = MRI.createVirtualRegister(RegClass: VGPR32);
6882 BuildLane(Lo, false);
6883 BuildLane(Hi, true);
6884
6885 Register Dst = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)->getReg();
6886 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
6887 .addReg(RegNo: Lo)
6888 .addImm(Val: AMDGPU::sub0)
6889 .addReg(RegNo: Hi)
6890 .addImm(Val: AMDGPU::sub1);
6891
6892 MI.eraseFromParent();
6893}
6894
6895bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
6896 const MachineOperand *MO) const {
6897 const MachineFunction &MF = *MI.getMF();
6898 const MachineRegisterInfo &MRI = MF.getRegInfo();
6899 const MCInstrDesc &InstDesc = MI.getDesc();
6900 const MCOperandInfo &OpInfo = InstDesc.operands()[OpIdx];
6901 int64_t RegClass = getOpRegClassID(OpInfo);
6902 const TargetRegisterClass *DefinedRC =
6903 RegClass != -1 ? RI.getRegClass(i: RegClass) : nullptr;
6904 if (!MO)
6905 MO = &MI.getOperand(i: OpIdx);
6906
6907 const bool IsInlineConst = !MO->isReg() && isInlineConstant(MO: *MO, OpInfo);
6908
6909 if (isVALU(MI, /*AllowLDSDMA=*/false) && !IsInlineConst &&
6910 usesConstantBus(MRI, MO: *MO, OpInfo)) {
6911 const MachineOperand *UsedLiteral = nullptr;
6912
6913 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: MI.getOpcode());
6914 int LiteralLimit = !isVOP3(MI) || ST.hasVOP3Literal() ? 1 : 0;
6915
6916 // TODO: Be more permissive with frame indexes.
6917 if (!MO->isReg() && !isInlineConstant(MO: *MO, OpInfo)) {
6918 if (!LiteralLimit--)
6919 return false;
6920
6921 UsedLiteral = MO;
6922 }
6923
6924 SmallDenseSet<RegSubRegPair> SGPRsUsed;
6925 if (MO->isReg())
6926 SGPRsUsed.insert(V: RegSubRegPair(MO->getReg(), MO->getSubReg()));
6927
6928 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6929 if (i == OpIdx)
6930 continue;
6931 const MachineOperand &Op = MI.getOperand(i);
6932 if (Op.isReg()) {
6933 if (Op.isUse()) {
6934 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
6935 if (regUsesConstantBus(RegOp: Op, MRI) && SGPRsUsed.insert(V: SGPR).second) {
6936 if (--ConstantBusLimit <= 0)
6937 return false;
6938 }
6939 }
6940 } else if (AMDGPU::isSISrcOperand(OpInfo: InstDesc.operands()[i]) &&
6941 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i])) {
6942 // The same literal may be used multiple times.
6943 if (!UsedLiteral)
6944 UsedLiteral = &Op;
6945 else if (UsedLiteral->isIdenticalTo(Other: Op))
6946 continue;
6947
6948 if (!LiteralLimit--)
6949 return false;
6950 if (--ConstantBusLimit <= 0)
6951 return false;
6952 }
6953 }
6954 } else if (!IsInlineConst && !MO->isReg() && isSALU(MI)) {
6955 // There can be at most one literal operand, but it can be repeated.
6956 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
6957 if (i == OpIdx)
6958 continue;
6959 const MachineOperand &Op = MI.getOperand(i);
6960 if (!Op.isReg() && !Op.isFI() && !Op.isRegMask() &&
6961 !isInlineConstant(MO: Op, OpInfo: InstDesc.operands()[i]) &&
6962 !Op.isIdenticalTo(Other: *MO))
6963 return false;
6964
6965 // Do not fold a non-inlineable and non-register operand into an
6966 // instruction that already has a frame index. The frame index handling
6967 // code could not handle well when a frame index co-exists with another
6968 // non-register operand, unless that operand is an inlineable immediate.
6969 if (Op.isFI())
6970 return false;
6971 }
6972 }
6973
6974 if (MO->isReg()) {
6975 if (!DefinedRC)
6976 return OpInfo.OperandType == MCOI::OPERAND_UNKNOWN;
6977 return isLegalRegOperand(MI, OpIdx, MO: *MO);
6978 }
6979
6980 if (MO->isImm()) {
6981 uint64_t Imm = MO->getImm();
6982 bool Is64BitFPOp = OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_FP64 ||
6983 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP64;
6984 bool Is64BitOp = Is64BitFPOp ||
6985 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_INT64 ||
6986 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT32 ||
6987 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32 ||
6988 OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_V2INT64;
6989 if (Is64BitOp &&
6990 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: ST.hasInv2PiInlineImm())) {
6991 if (!AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: Is64BitFPOp) &&
6992 (!ST.has64BitLiterals() || InstDesc.getSize() != 4))
6993 return false;
6994
6995 // FIXME: We can use sign extended 64-bit literals, but only for signed
6996 // operands. At the moment we do not know if an operand is signed.
6997 // Such operand will be encoded as its low 32 bits and then either
6998 // correctly sign extended or incorrectly zero extended by HW.
6999 // If 64-bit literals are supported and the literal will be encoded
7000 // as full 64 bit we still can use it.
7001 if (!Is64BitFPOp && (int32_t)Imm < 0 &&
7002 (!ST.has64BitLiterals() || AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: false)))
7003 return false;
7004 }
7005 }
7006
7007 // Handle non-register types that are treated like immediates.
7008 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
7009
7010 if (!DefinedRC) {
7011 // This operand expects an immediate.
7012 return true;
7013 }
7014
7015 return isImmOperandLegal(MI, OpNo: OpIdx, MO: *MO);
7016}
7017
7018bool SIInstrInfo::isNeverCoissue(MachineInstr &MI) const {
7019 bool IsGFX950Only = ST.hasGFX950Insts();
7020 bool IsGFX940Only = ST.hasGFX940Insts();
7021
7022 if (!IsGFX950Only && !IsGFX940Only)
7023 return false;
7024
7025 if (!isVALU(MI, /*AllowLDSDMA=*/false))
7026 return false;
7027
7028 // V_COS, V_EXP, V_RCP, etc.
7029 if (isTRANS(MI))
7030 return true;
7031
7032 // DOT2, DOT2C, DOT4, etc.
7033 if (isDOT(MI))
7034 return true;
7035
7036 // MFMA, SMFMA
7037 if (isMFMA(MI))
7038 return true;
7039
7040 unsigned Opcode = MI.getOpcode();
7041 switch (Opcode) {
7042 case AMDGPU::V_CVT_PK_BF8_F32_e64:
7043 case AMDGPU::V_CVT_PK_FP8_F32_e64:
7044 case AMDGPU::V_MQSAD_PK_U16_U8_e64:
7045 case AMDGPU::V_MQSAD_U32_U8_e64:
7046 case AMDGPU::V_PK_ADD_F16:
7047 case AMDGPU::V_PK_ADD_F32:
7048 case AMDGPU::V_PK_ADD_I16:
7049 case AMDGPU::V_PK_ADD_U16:
7050 case AMDGPU::V_PK_ASHRREV_I16:
7051 case AMDGPU::V_PK_FMA_F16:
7052 case AMDGPU::V_PK_FMA_F32:
7053 case AMDGPU::V_PK_FMAC_F16_e32:
7054 case AMDGPU::V_PK_FMAC_F16_e64:
7055 case AMDGPU::V_PK_LSHLREV_B16:
7056 case AMDGPU::V_PK_LSHRREV_B16:
7057 case AMDGPU::V_PK_MAD_I16:
7058 case AMDGPU::V_PK_MAD_U16:
7059 case AMDGPU::V_PK_MAX_F16:
7060 case AMDGPU::V_PK_MAX_I16:
7061 case AMDGPU::V_PK_MAX_U16:
7062 case AMDGPU::V_PK_MIN_F16:
7063 case AMDGPU::V_PK_MIN_I16:
7064 case AMDGPU::V_PK_MIN_U16:
7065 case AMDGPU::V_PK_MOV_B32:
7066 case AMDGPU::V_PK_MUL_F16:
7067 case AMDGPU::V_PK_MUL_F32:
7068 case AMDGPU::V_PK_MUL_LO_U16:
7069 case AMDGPU::V_PK_SUB_I16:
7070 case AMDGPU::V_PK_SUB_U16:
7071 case AMDGPU::V_QSAD_PK_U16_U8_e64:
7072 return true;
7073 default:
7074 return false;
7075 }
7076}
7077
7078void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
7079 MachineInstr &MI) const {
7080 unsigned Opc = MI.getOpcode();
7081 const MCInstrDesc &InstrDesc = get(Opcode: Opc);
7082
7083 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
7084 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
7085
7086 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
7087 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
7088
7089 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
7090 // we need to only have one constant bus use before GFX10.
7091 bool HasImplicitSGPR = findImplicitSGPRRead(MI);
7092 if (HasImplicitSGPR && ST.getConstantBusLimit(Opcode: Opc) <= 1 && Src0.isReg() &&
7093 RI.isSGPRReg(MRI, Reg: Src0.getReg()))
7094 legalizeOpWithMove(MI, OpIdx: Src0Idx);
7095
7096 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
7097 // both the value to write (src0) and lane select (src1). Fix up non-SGPR
7098 // src0/src1 with V_READFIRSTLANE.
7099 if (Opc == AMDGPU::V_WRITELANE_B32) {
7100 const DebugLoc &DL = MI.getDebugLoc();
7101 if (Src0.isReg() && RI.isVGPR(MRI, Reg: Src0.getReg())) {
7102 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7103 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7104 .add(MO: Src0);
7105 Src0.ChangeToRegister(Reg, isDef: false);
7106 }
7107 if (Src1.isReg() && RI.isVGPR(MRI, Reg: Src1.getReg())) {
7108 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7109 const DebugLoc &DL = MI.getDebugLoc();
7110 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7111 .add(MO: Src1);
7112 Src1.ChangeToRegister(Reg, isDef: false);
7113 }
7114 return;
7115 }
7116
7117 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2.
7118 if (Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F16_e32) {
7119 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
7120 if (!RI.isVGPR(MRI, Reg: MI.getOperand(i: Src2Idx).getReg()))
7121 legalizeOpWithMove(MI, OpIdx: Src2Idx);
7122 }
7123
7124 // VOP2 src0 instructions support all operand types, so we don't need to check
7125 // their legality. If src1 is already legal, we don't need to do anything.
7126 if (isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src1))
7127 return;
7128
7129 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
7130 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
7131 // select is uniform.
7132 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
7133 RI.isVGPR(MRI, Reg: Src1.getReg())) {
7134 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7135 const DebugLoc &DL = MI.getDebugLoc();
7136 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7137 .add(MO: Src1);
7138 Src1.ChangeToRegister(Reg, isDef: false);
7139 return;
7140 }
7141
7142 // We do not use commuteInstruction here because it is too aggressive and will
7143 // commute if it is possible. We only want to commute here if it improves
7144 // legality. This can be called a fairly large number of times so don't waste
7145 // compile time pointlessly swapping and checking legality again.
7146 if (HasImplicitSGPR || !MI.isCommutable()) {
7147 legalizeOpWithMove(MI, OpIdx: Src1Idx);
7148 return;
7149 }
7150
7151 // If src0 can be used as src1, commuting will make the operands legal.
7152 // Otherwise we have to give up and insert a move.
7153 //
7154 // TODO: Other immediate-like operand kinds could be commuted if there was a
7155 // MachineOperand::ChangeTo* for them.
7156 if ((!Src1.isImm() && !Src1.isReg()) ||
7157 !isLegalRegOperand(MRI, OpInfo: InstrDesc.operands()[Src1Idx], MO: Src0)) {
7158 legalizeOpWithMove(MI, OpIdx: Src1Idx);
7159 return;
7160 }
7161
7162 int CommutedOpc = commuteOpcode(MI);
7163 if (CommutedOpc == -1) {
7164 legalizeOpWithMove(MI, OpIdx: Src1Idx);
7165 return;
7166 }
7167
7168 MI.setDesc(get(Opcode: CommutedOpc));
7169
7170 Register Src0Reg = Src0.getReg();
7171 unsigned Src0SubReg = Src0.getSubReg();
7172 bool Src0Kill = Src0.isKill();
7173
7174 if (Src1.isImm())
7175 Src0.ChangeToImmediate(ImmVal: Src1.getImm());
7176 else if (Src1.isReg()) {
7177 Src0.ChangeToRegister(Reg: Src1.getReg(), isDef: false, isImp: false, isKill: Src1.isKill());
7178 Src0.setSubReg(Src1.getSubReg());
7179 } else
7180 llvm_unreachable("Should only have register or immediate operands");
7181
7182 Src1.ChangeToRegister(Reg: Src0Reg, isDef: false, isImp: false, isKill: Src0Kill);
7183 Src1.setSubReg(Src0SubReg);
7184 fixImplicitOperands(MI);
7185}
7186
7187// Legalize VOP3 operands. All operand types are supported for any operand
7188// but only one literal constant and only starting from GFX10.
7189void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
7190 MachineInstr &MI) const {
7191 unsigned Opc = MI.getOpcode();
7192
7193 int VOP3Idx[3] = {
7194 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0),
7195 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1),
7196 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2)
7197 };
7198
7199 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 ||
7200 Opc == AMDGPU::V_PERMLANEX16_B32_e64 ||
7201 Opc == AMDGPU::V_PERMLANE_BCAST_B32_e64 ||
7202 Opc == AMDGPU::V_PERMLANE_UP_B32_e64 ||
7203 Opc == AMDGPU::V_PERMLANE_DOWN_B32_e64 ||
7204 Opc == AMDGPU::V_PERMLANE_XOR_B32_e64 ||
7205 Opc == AMDGPU::V_PERMLANE_IDX_GEN_B32_e64) {
7206 // src1 and src2 must be scalar
7207 MachineOperand &Src1 = MI.getOperand(i: VOP3Idx[1]);
7208 const DebugLoc &DL = MI.getDebugLoc();
7209 if (Src1.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()))) {
7210 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7211 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7212 .add(MO: Src1);
7213 Src1.ChangeToRegister(Reg, isDef: false);
7214 }
7215 if (VOP3Idx[2] != -1) {
7216 MachineOperand &Src2 = MI.getOperand(i: VOP3Idx[2]);
7217 if (Src2.isReg() && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src2.getReg()))) {
7218 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7219 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
7220 .add(MO: Src2);
7221 Src2.ChangeToRegister(Reg, isDef: false);
7222 }
7223 }
7224 }
7225
7226 // Find the one SGPR operand we are allowed to use.
7227 int ConstantBusLimit = ST.getConstantBusLimit(Opcode: Opc);
7228 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
7229 SmallDenseSet<unsigned> SGPRsUsed;
7230 Register SGPRReg = findUsedSGPR(MI, OpIndices: VOP3Idx);
7231 if (SGPRReg) {
7232 SGPRsUsed.insert(V: SGPRReg);
7233 --ConstantBusLimit;
7234 }
7235
7236 for (int Idx : VOP3Idx) {
7237 if (Idx == -1)
7238 break;
7239 MachineOperand &MO = MI.getOperand(i: Idx);
7240
7241 if (!MO.isReg()) {
7242 if (isInlineConstant(MO, OpInfo: get(Opcode: Opc).operands()[Idx]))
7243 continue;
7244
7245 if (LiteralLimit > 0 && ConstantBusLimit > 0) {
7246 --LiteralLimit;
7247 --ConstantBusLimit;
7248 continue;
7249 }
7250
7251 --LiteralLimit;
7252 --ConstantBusLimit;
7253 legalizeOpWithMove(MI, OpIdx: Idx);
7254 continue;
7255 }
7256
7257 if (!RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg: MO.getReg())))
7258 continue; // VGPRs are legal
7259
7260 // We can use one SGPR in each VOP3 instruction prior to GFX10
7261 // and two starting from GFX10.
7262 if (SGPRsUsed.count(V: MO.getReg()))
7263 continue;
7264 if (ConstantBusLimit > 0) {
7265 SGPRsUsed.insert(V: MO.getReg());
7266 --ConstantBusLimit;
7267 continue;
7268 }
7269
7270 // If we make it this far, then the operand is not legal and we must
7271 // legalize it.
7272 legalizeOpWithMove(MI, OpIdx: Idx);
7273 }
7274
7275 // Special case: V_FMAC_F32 and V_FMAC_F16 have src2 tied to vdst.
7276 if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64 ||
7277 Opc == AMDGPU::V_FMAC_F16_fake16_e64 ||
7278 Opc == AMDGPU::V_FMAC_F16_t16_e64) &&
7279 !RI.isVGPR(MRI, Reg: MI.getOperand(i: VOP3Idx[2]).getReg()))
7280 legalizeOpWithMove(MI, OpIdx: VOP3Idx[2]);
7281
7282 // Fix the register class of single-sgpr-read instructions on gfx12+. See
7283 // SIInstrInfo::isLegalSingleSGPRReadInstOperand for more information.
7284 if (AMDGPU::isSingleSGPRReadInst(Opc)) {
7285 for (unsigned I = 0; I < 3; ++I) {
7286 if (!isLegalSingleSGPRReadInstOperand(MRI, MI, /*SrcN=*/I))
7287 legalizeOpWithMove(MI, OpIdx: VOP3Idx[I]);
7288 }
7289 }
7290}
7291
7292Register SIInstrInfo::readlaneVGPRToSGPR(
7293 Register SrcReg, MachineInstr &UseMI, MachineRegisterInfo &MRI,
7294 const TargetRegisterClass *DstRC /*=nullptr*/) const {
7295 const TargetRegisterClass *VRC = MRI.getRegClass(Reg: SrcReg);
7296 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
7297 if (DstRC)
7298 SRC = RI.getCommonSubClass(A: SRC, B: DstRC);
7299
7300 Register DstReg = MRI.createVirtualRegister(RegClass: SRC);
7301 unsigned SubRegs = RI.getRegSizeInBits(RC: *VRC) / 32;
7302
7303 if (RI.hasAGPRs(RC: VRC)) {
7304 VRC = RI.getEquivalentVGPRClass(SRC: VRC);
7305 Register NewSrcReg = MRI.createVirtualRegister(RegClass: VRC);
7306 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7307 MCID: get(Opcode: TargetOpcode::COPY), DestReg: NewSrcReg)
7308 .addReg(RegNo: SrcReg);
7309 SrcReg = NewSrcReg;
7310 }
7311
7312 if (SubRegs == 1) {
7313 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7314 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
7315 .addReg(RegNo: SrcReg);
7316 return DstReg;
7317 }
7318
7319 SmallVector<Register, 8> SRegs;
7320 for (unsigned i = 0; i < SubRegs; ++i) {
7321 Register SGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7322 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7323 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SGPR)
7324 .addReg(RegNo: SrcReg, Flags: {}, SubReg: RI.getSubRegFromChannel(Channel: i));
7325 SRegs.push_back(Elt: SGPR);
7326 }
7327
7328 MachineInstrBuilder MIB =
7329 BuildMI(BB&: *UseMI.getParent(), I&: UseMI, MIMD: UseMI.getDebugLoc(),
7330 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
7331 for (unsigned i = 0; i < SubRegs; ++i) {
7332 MIB.addReg(RegNo: SRegs[i]);
7333 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
7334 }
7335 return DstReg;
7336}
7337
7338void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
7339 MachineInstr &MI) const {
7340
7341 // If the pointer is store in VGPRs, then we need to move them to
7342 // SGPRs using v_readfirstlane. This is safe because we only select
7343 // loads with uniform pointers to SMRD instruction so we know the
7344 // pointer value is uniform.
7345 MachineOperand *SBase = getNamedOperand(MI, OperandName: AMDGPU::OpName::sbase);
7346 if (SBase && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SBase->getReg()))) {
7347 Register SGPR = readlaneVGPRToSGPR(SrcReg: SBase->getReg(), UseMI&: MI, MRI);
7348 SBase->setReg(SGPR);
7349 }
7350 MachineOperand *SOff = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
7351 if (SOff && !RI.isSGPRReg(MRI, Reg: SOff->getReg())) {
7352 Register SGPR = readlaneVGPRToSGPR(SrcReg: SOff->getReg(), UseMI&: MI, MRI);
7353 SOff->setReg(SGPR);
7354 }
7355}
7356
7357bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const {
7358 unsigned Opc = Inst.getOpcode();
7359 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
7360 if (OldSAddrIdx < 0)
7361 return false;
7362
7363 assert(isSegmentSpecificFLAT(Inst) || (isFLAT(Inst) && ST.hasFlatGVSMode()));
7364
7365 int NewOpc = AMDGPU::getGlobalVaddrOp(Opcode: Opc);
7366 if (NewOpc < 0)
7367 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opcode: Opc);
7368 if (NewOpc < 0)
7369 return false;
7370
7371 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
7372 MachineOperand &SAddr = Inst.getOperand(i: OldSAddrIdx);
7373 if (RI.isSGPRReg(MRI, Reg: SAddr.getReg()))
7374 return false;
7375
7376 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vaddr);
7377 if (NewVAddrIdx < 0)
7378 return false;
7379
7380 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
7381
7382 // Check vaddr, it shall be zero or absent.
7383 MachineInstr *VAddrDef = nullptr;
7384 if (OldVAddrIdx >= 0) {
7385 MachineOperand &VAddr = Inst.getOperand(i: OldVAddrIdx);
7386 VAddrDef = MRI.getUniqueVRegDef(Reg: VAddr.getReg());
7387 if (!VAddrDef || !VAddrDef->isMoveImmediate() ||
7388 !VAddrDef->getOperand(i: 1).isImm() ||
7389 VAddrDef->getOperand(i: 1).getImm() != 0)
7390 return false;
7391 }
7392
7393 const MCInstrDesc &NewDesc = get(Opcode: NewOpc);
7394 Inst.setDesc(NewDesc);
7395
7396 // Callers expect iterator to be valid after this call, so modify the
7397 // instruction in place.
7398 if (OldVAddrIdx == NewVAddrIdx) {
7399 MachineOperand &NewVAddr = Inst.getOperand(i: NewVAddrIdx);
7400 // Clear use list from the old vaddr holding a zero register.
7401 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7402 MRI.moveOperands(Dst: &NewVAddr, Src: &SAddr, NumOps: 1);
7403 Inst.removeOperand(OpNo: OldSAddrIdx);
7404 // Update the use list with the pointer we have just moved from vaddr to
7405 // saddr position. Otherwise new vaddr will be missing from the use list.
7406 MRI.removeRegOperandFromUseList(MO: &NewVAddr);
7407 MRI.addRegOperandToUseList(MO: &NewVAddr);
7408 } else {
7409 assert(OldSAddrIdx == NewVAddrIdx);
7410
7411 if (OldVAddrIdx >= 0) {
7412 int NewVDstIn = AMDGPU::getNamedOperandIdx(Opcode: NewOpc,
7413 Name: AMDGPU::OpName::vdst_in);
7414
7415 // removeOperand doesn't try to fixup tied operand indexes at it goes, so
7416 // it asserts. Untie the operands for now and retie them afterwards.
7417 if (NewVDstIn != -1) {
7418 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
7419 Inst.untieRegOperand(OpIdx: OldVDstIn);
7420 }
7421
7422 Inst.removeOperand(OpNo: OldVAddrIdx);
7423
7424 if (NewVDstIn != -1) {
7425 int NewVDst = AMDGPU::getNamedOperandIdx(Opcode: NewOpc, Name: AMDGPU::OpName::vdst);
7426 Inst.tieOperands(DefIdx: NewVDst, UseIdx: NewVDstIn);
7427 }
7428 }
7429 }
7430
7431 if (VAddrDef && MRI.use_nodbg_empty(RegNo: VAddrDef->getOperand(i: 0).getReg()))
7432 VAddrDef->eraseFromParent();
7433
7434 return true;
7435}
7436
7437// FIXME: Remove this when SelectionDAG is obsoleted.
7438void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI,
7439 MachineInstr &MI) const {
7440 if (!isSegmentSpecificFLAT(MI) && !ST.hasFlatGVSMode())
7441 return;
7442
7443 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence
7444 // thinks they are uniform, so a readfirstlane should be valid.
7445 MachineOperand *SAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::saddr);
7446 if (!SAddr || RI.isSGPRClass(RC: MRI.getRegClass(Reg: SAddr->getReg())))
7447 return;
7448
7449 if (moveFlatAddrToVGPR(Inst&: MI))
7450 return;
7451
7452 const TargetRegisterClass *DeclaredRC =
7453 getRegClass(MCID: MI.getDesc(), OpNum: SAddr->getOperandNo());
7454
7455 Register ToSGPR = readlaneVGPRToSGPR(SrcReg: SAddr->getReg(), UseMI&: MI, MRI, DstRC: DeclaredRC);
7456 SAddr->setReg(ToSGPR);
7457}
7458
7459void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
7460 MachineBasicBlock::iterator I,
7461 const TargetRegisterClass *DstRC,
7462 MachineOperand &Op,
7463 MachineRegisterInfo &MRI,
7464 const DebugLoc &DL) const {
7465 Register OpReg = Op.getReg();
7466 unsigned OpSubReg = Op.getSubReg();
7467
7468 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
7469 RI.getRegClassForReg(MRI, Reg: OpReg), OpSubReg);
7470
7471 // Check if operand is already the correct register class.
7472 if (DstRC == OpRC)
7473 return;
7474
7475 Register DstReg = MRI.createVirtualRegister(RegClass: DstRC);
7476 auto Copy = BuildMI(BB&: InsertMBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: DstReg)
7477 .addReg(RegNo: OpReg, Flags: {}, SubReg: OpSubReg);
7478 Op.setReg(DstReg);
7479 Op.setSubReg(AMDGPU::NoSubRegister);
7480
7481 MachineInstr *Def = MRI.getVRegDef(Reg: OpReg);
7482 if (!Def)
7483 return;
7484
7485 // Try to eliminate the copy if it is copying an immediate value.
7486 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
7487 foldImmediate(UseMI&: *Copy, DefMI&: *Def, Reg: OpReg, MRI: &MRI);
7488
7489 bool ImpDef = Def->isImplicitDef();
7490 while (!ImpDef && Def && Def->isCopy()) {
7491 if (Def->getOperand(i: 1).getReg().isPhysical())
7492 break;
7493 Def = MRI.getUniqueVRegDef(Reg: Def->getOperand(i: 1).getReg());
7494 ImpDef = Def && Def->isImplicitDef();
7495 }
7496 if (!RI.isSGPRClass(RC: DstRC) && !Copy->readsRegister(Reg: AMDGPU::EXEC, TRI: &RI) &&
7497 !ImpDef)
7498 Copy.addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
7499}
7500
7501// Emit the actual waterfall loop, executing the wrapped instruction for each
7502// unique value of \p ScalarOps across all lanes. In the best case we execute 1
7503// iteration, in the worst case we execute 64 (once per lane).
7504static void emitLoadScalarOpsFromVGPRLoop(
7505 const SIInstrInfo &TII, MachineRegisterInfo &MRI, MachineBasicBlock &PredBB,
7506 MachineBasicBlock &LoopBB, MachineBasicBlock &BodyBB, const DebugLoc &DL,
7507 ArrayRef<MachineOperand *> ScalarOps, ArrayRef<Register> PhySGPRs = {}) {
7508 MachineFunction &MF = *LoopBB.getParent();
7509 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7510 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7511 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7512 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7513
7514 // Emit v_cmpx_eq and s_andn2_wrexec when both instructions are
7515 // available. Otherwise, use the previous pattern of v_cmp_eq,
7516 // s_and_saveexec, and s_xor.
7517 bool UseNewExecInstructions =
7518 ST.hasNoSdstCMPX() && TII.pseudoToMCOpcode(Opcode: LMC.AndN2WrExecOpc) != -1;
7519
7520 MachineBasicBlock::iterator I = LoopBB.begin();
7521 Register CondReg;
7522
7523 Register PhiExec;
7524 Register NewExec;
7525
7526 if (UseNewExecInstructions) {
7527 PhiExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7528 NewExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7529 Register InitExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7530 BuildMI(BB&: PredBB, I: PredBB.end(), MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: InitExec)
7531 .addReg(RegNo: LMC.ExecReg);
7532
7533 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
7534 .addReg(RegNo: InitExec)
7535 .addMBB(MBB: &PredBB)
7536 .addReg(RegNo: NewExec)
7537 .addMBB(MBB: &BodyBB);
7538 }
7539
7540 // Placement of v_cmpx instructions (when index is longer than 64 bit)
7541 // involves a trade-off between register pressure and latency:
7542 // (a) Defering all v_cmpx after all v_readfirstlane may increase
7543 // register pressure because arguments and results of all
7544 // v_readfirstlane instructions must stay live until deferred v_cmpx use them.
7545 // (b) Interleaving v_cmpx with v_readfirstlanes may reduce live ranges and
7546 // increase latency by placing v_readfirstlane instructions
7547 // immediately before v_cmpx instruction that directly depend on it.
7548 ///
7549 // Emitting interleaved v_cmpx and v_readfirstlane requires
7550 // block splitting because v_cmpx changes EXEC mask and therefore for safety
7551 // v_cmpx needs to be treated as terminator until after register allocation
7552 // (spill placement) and instruction reordering.
7553 //
7554 // Current implementation defers v_cmpx and leaves other instruction
7555 // scheduling decisions to later passes, where register pressure is known or
7556 // easier to approximate.
7557 // Non-terminators (V_READFIRSTLANE and REG_SEQUENCE) are inserted before I;
7558 // v_cmpx instructions are inserted at the end of LoopBB.
7559 // After the first v_cmpx is emitted, I is updated to point to it
7560 // so subsequent non-terminators are inserted before all v_cmpx instructions.
7561 for (auto [Idx, ScalarOp] : enumerate(First&: ScalarOps)) {
7562 unsigned RegSize = TRI->getRegSizeInBits(Reg: ScalarOp->getReg(), MRI);
7563 unsigned NumSubRegs = RegSize / 32;
7564 Register VScalarOp = ScalarOp->getReg();
7565
7566 const TargetRegisterClass *RFLSrcRC =
7567 TII.getRegClass(MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), OpNum: 1);
7568
7569 if (NumSubRegs == 1) {
7570 const TargetRegisterClass *VScalarOpRC = MRI.getRegClass(Reg: VScalarOp);
7571 if (const TargetRegisterClass *Common =
7572 TRI->getCommonSubClass(A: VScalarOpRC, B: RFLSrcRC);
7573 Common != VScalarOpRC) {
7574 Register VRReg = MRI.createVirtualRegister(RegClass: Common);
7575 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VRReg).addReg(RegNo: VScalarOp);
7576 VScalarOp = VRReg;
7577 }
7578 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7579
7580 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurReg)
7581 .addReg(RegNo: VScalarOp);
7582
7583 if (UseNewExecInstructions) {
7584 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7585 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U32_nosdst_e32_term))
7586 .addReg(RegNo: CurReg)
7587 .addReg(RegNo: VScalarOp);
7588 if (I == LoopBB.end())
7589 I = CmpxMI.getInstr()->getIterator();
7590 } else {
7591 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7592
7593 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: NewCondReg)
7594 .addReg(RegNo: CurReg)
7595 .addReg(RegNo: VScalarOp);
7596
7597 // Combine the comparison results with AND.
7598 if (!CondReg) { // First.
7599 CondReg = NewCondReg;
7600 } else { // If not the first, we create an AND.
7601 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7602 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7603 .addReg(RegNo: CondReg)
7604 .addReg(RegNo: NewCondReg)
7605 .setOperandDead(3);
7606 CondReg = AndReg;
7607 }
7608 }
7609
7610 // Update ScalarOp operand to use the SGPR ScalarOp.
7611 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7612 ScalarOp->setReg(CurReg);
7613 else {
7614 // Insert into the same block of use
7615 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7616 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7617 .addReg(RegNo: CurReg);
7618 ScalarOp->setReg(PhySGPRs[Idx]);
7619 }
7620 } else {
7621 SmallVector<Register, 8> ReadlanePieces;
7622 RegState VScalarOpUndef = getUndefRegState(B: ScalarOp->isUndef());
7623 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 &&
7624 "Unhandled register size");
7625
7626 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) {
7627 Register CurRegLo =
7628 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7629 Register CurRegHi =
7630 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7631
7632 // Read the next variant <- also loop target.
7633 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegLo)
7634 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg: TRI->getSubRegFromChannel(Channel: Idx));
7635
7636 // Read the next variant <- also loop target.
7637 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurRegHi)
7638 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef,
7639 SubReg: TRI->getSubRegFromChannel(Channel: Idx + 1));
7640
7641 ReadlanePieces.push_back(Elt: CurRegLo);
7642 ReadlanePieces.push_back(Elt: CurRegHi);
7643
7644 // Comparison is to be done as 64-bit.
7645 Register CurReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_64RegClass);
7646 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: CurReg)
7647 .addReg(RegNo: CurRegLo)
7648 .addImm(Val: AMDGPU::sub0)
7649 .addReg(RegNo: CurRegHi)
7650 .addImm(Val: AMDGPU::sub1);
7651
7652 unsigned SubReg =
7653 NumSubRegs <= 2 ? 0 : TRI->getSubRegFromChannel(Channel: Idx, NumRegs: 2);
7654
7655 if (UseNewExecInstructions) {
7656 auto CmpxMI = BuildMI(BB&: LoopBB, I: LoopBB.end(), MIMD: DL,
7657 MCID: TII.get(Opcode: AMDGPU::V_CMPX_EQ_U64_nosdst_e32_term))
7658 .addReg(RegNo: CurReg)
7659 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7660 if (I == LoopBB.end())
7661 I = CmpxMI.getInstr()->getIterator();
7662 } else {
7663 Register NewCondReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7664 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U64_e64), DestReg: NewCondReg)
7665 .addReg(RegNo: CurReg)
7666 .addReg(RegNo: VScalarOp, Flags: VScalarOpUndef, SubReg);
7667
7668 // Combine the comparison results with AND.
7669 if (!CondReg) { // First.
7670 CondReg = NewCondReg;
7671 } else { // If not the first, we create an AND.
7672 Register AndReg = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7673 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndOpc), DestReg: AndReg)
7674 .addReg(RegNo: CondReg)
7675 .addReg(RegNo: NewCondReg)
7676 .setOperandDead(3);
7677 CondReg = AndReg;
7678 }
7679 }
7680 } // End for loop.
7681
7682 const auto *SScalarOpRC =
7683 TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: VScalarOp));
7684 Register SScalarOp = MRI.createVirtualRegister(RegClass: SScalarOpRC);
7685
7686 // Build scalar ScalarOp.
7687 auto Merge =
7688 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: SScalarOp);
7689 unsigned Channel = 0;
7690 for (Register Piece : ReadlanePieces) {
7691 Merge.addReg(RegNo: Piece).addImm(Val: TRI->getSubRegFromChannel(Channel: Channel++));
7692 }
7693
7694 // Update ScalarOp operand to use the SGPR ScalarOp.
7695 if (PhySGPRs.empty() || !PhySGPRs[Idx].isValid())
7696 ScalarOp->setReg(SScalarOp);
7697 else {
7698 BuildMI(BB&: *ScalarOp->getParent()->getParent(), I: ScalarOp->getParent(), MIMD: DL,
7699 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhySGPRs[Idx])
7700 .addReg(RegNo: SScalarOp);
7701 ScalarOp->setReg(PhySGPRs[Idx]);
7702 }
7703 }
7704 }
7705
7706 // AndSaveExecOpc modifies EXEC but can't be isTerminator=1: terminators
7707 // that define virtual registers aren't supported.
7708 Register SaveExec;
7709 if (!UseNewExecInstructions) {
7710 SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7711 MRI.setSimpleHint(VReg: SaveExec, PrefReg: CondReg);
7712
7713 // Update EXEC to matching lanes, saving original to SaveExec.
7714 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndSaveExecOpc), DestReg: SaveExec)
7715 .addReg(RegNo: CondReg)
7716 .setOperandDead(3);
7717 }
7718
7719 // The original instruction is here; we insert the terminators after it.
7720 I = BodyBB.end();
7721
7722 if (UseNewExecInstructions) {
7723 // Compute the remaining lanes into a plain virtual register and write EXEC
7724 // from a terminator, so spill code for NewExec is placed before EXEC
7725 // changes. SIOptimizeExecMasking opportunistically folds the pair back
7726 // into S_ANDN2_WREXEC after register allocation.
7727 MRI.setSimpleHint(VReg: NewExec, PrefReg: PhiExec);
7728 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.AndN2Opc), DestReg: NewExec)
7729 .addReg(RegNo: PhiExec)
7730 .addReg(RegNo: LMC.ExecReg)
7731 .setOperandDead(3);
7732 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.MovTermOpc), DestReg: LMC.ExecReg)
7733 .addReg(RegNo: NewExec);
7734 } else {
7735 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
7736 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
7737 .addReg(RegNo: LMC.ExecReg)
7738 .addReg(RegNo: SaveExec)
7739 .setOperandDead(3);
7740 }
7741
7742 BuildMI(BB&: BodyBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::SI_WATERFALL_LOOP)).addMBB(MBB: &LoopBB);
7743}
7744
7745// Build a waterfall loop around \p MI, replacing the VGPR \p ScalarOp register
7746// with SGPRs by iterating over all unique values across all lanes.
7747// Returns the loop basic block that now contains \p MI.
7748static MachineBasicBlock *
7749generateWaterFallLoop(const SIInstrInfo &TII, MachineInstr &MI,
7750 ArrayRef<MachineOperand *> ScalarOps,
7751 MachineDominatorTree *MDT,
7752 MachineBasicBlock::iterator Begin = nullptr,
7753 MachineBasicBlock::iterator End = nullptr,
7754 ArrayRef<Register> PhySGPRs = {}) {
7755 assert((PhySGPRs.empty() || PhySGPRs.size() == ScalarOps.size()) &&
7756 "Physical SGPRs must be empty or match the number of scalar operands");
7757 MachineBasicBlock &MBB = *MI.getParent();
7758 MachineFunction &MF = *MBB.getParent();
7759 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
7760 const SIRegisterInfo *TRI = ST.getRegisterInfo();
7761 MachineRegisterInfo &MRI = MF.getRegInfo();
7762 if (!Begin.isValid())
7763 Begin = &MI;
7764 if (!End.isValid()) {
7765 End = &MI;
7766 ++End;
7767 }
7768 const DebugLoc &DL = MI.getDebugLoc();
7769 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
7770 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
7771
7772 // Save SCC. Waterfall Loop may overwrite SCC.
7773 Register SaveSCCReg;
7774
7775 // FIXME: We should maintain SCC liveness while doing the FixSGPRCopies walk
7776 // rather than unlimited scan everywhere
7777 bool SCCNotDead =
7778 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::SCC, Before: MI,
7779 Neighborhood: std::numeric_limits<unsigned>::max()) !=
7780 MachineBasicBlock::LQR_Dead;
7781 if (SCCNotDead) {
7782 SaveSCCReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7783 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SaveSCCReg)
7784 .addImm(Val: 1)
7785 .addImm(Val: 0);
7786 }
7787
7788 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
7789
7790 // Save the EXEC mask
7791 BuildMI(BB&: MBB, I: Begin, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: SaveExec).addReg(RegNo: LMC.ExecReg);
7792
7793 // Killed uses in the instruction we are waterfalling around will be
7794 // incorrect due to the added control-flow.
7795 MachineBasicBlock::iterator AfterMI = MI;
7796 ++AfterMI;
7797 for (auto I = Begin; I != AfterMI; I++) {
7798 for (auto &MO : I->all_uses())
7799 MRI.clearKillFlags(Reg: MO.getReg());
7800 }
7801
7802 // To insert the loop we need to split the block. Move everything after this
7803 // point to a new block, and insert a new empty block between the two.
7804 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
7805 MachineBasicBlock *BodyBB = MF.CreateMachineBasicBlock();
7806 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
7807 MachineFunction::iterator MBBI(MBB);
7808 ++MBBI;
7809
7810 MF.insert(MBBI, MBB: LoopBB);
7811 MF.insert(MBBI, MBB: BodyBB);
7812 MF.insert(MBBI, MBB: RemainderBB);
7813
7814 LoopBB->addSuccessor(Succ: BodyBB);
7815 BodyBB->addSuccessor(Succ: LoopBB);
7816 BodyBB->addSuccessor(Succ: RemainderBB);
7817
7818 // Move Begin to MI to the BodyBB, and the remainder of the block to
7819 // RemainderBB.
7820 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
7821 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: End, To: MBB.end());
7822 BodyBB->splice(Where: BodyBB->begin(), Other: &MBB, From: Begin, To: MBB.end());
7823
7824 MBB.addSuccessor(Succ: LoopBB);
7825
7826 // Update dominators. We know that MBB immediately dominates LoopBB, that
7827 // LoopBB immediately dominates BodyBB, and BodyBB immediately dominates
7828 // RemainderBB. RemainderBB immediately dominates all of the successors
7829 // transferred to it from MBB that MBB used to properly dominate.
7830 if (MDT) {
7831 MDT->addNewBlock(BB: LoopBB, DomBB: &MBB);
7832 MDT->addNewBlock(BB: BodyBB, DomBB: LoopBB);
7833 MDT->addNewBlock(BB: RemainderBB, DomBB: BodyBB);
7834 for (auto &Succ : RemainderBB->successors()) {
7835 if (MDT->properlyDominates(A: &MBB, B: Succ)) {
7836 MDT->changeImmediateDominator(BB: Succ, NewBB: RemainderBB);
7837 }
7838 }
7839 }
7840
7841 emitLoadScalarOpsFromVGPRLoop(TII, MRI, PredBB&: MBB, LoopBB&: *LoopBB, BodyBB&: *BodyBB, DL, ScalarOps,
7842 PhySGPRs);
7843
7844 MachineBasicBlock::iterator First = RemainderBB->begin();
7845 // Restore SCC
7846 if (SCCNotDead) {
7847 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_LG_U32))
7848 .addReg(RegNo: SaveSCCReg)
7849 .addImm(Val: 0);
7850 }
7851
7852 // Restore the EXEC mask
7853 BuildMI(BB&: *RemainderBB, I: First, MIMD: DL, MCID: TII.get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
7854 .addReg(RegNo: SaveExec);
7855 return BodyBB;
7856}
7857
7858// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
7859static std::tuple<unsigned, unsigned>
7860extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
7861 MachineBasicBlock &MBB = *MI.getParent();
7862 MachineFunction &MF = *MBB.getParent();
7863 MachineRegisterInfo &MRI = MF.getRegInfo();
7864
7865 // Extract the ptr from the resource descriptor.
7866 unsigned RsrcPtr =
7867 TII.buildExtractSubReg(MI, MRI, SuperReg: Rsrc, SuperRC: &AMDGPU::VReg_128RegClass,
7868 SubIdx: AMDGPU::sub0_sub1, SubRC: &AMDGPU::VReg_64RegClass);
7869
7870 // Create an empty resource descriptor
7871 Register Zero64 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
7872 Register SRsrcFormatLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7873 Register SRsrcFormatHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
7874 Register NewSRsrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
7875 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
7876
7877 // Zero64 = 0
7878 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: Zero64)
7879 .addImm(Val: 0);
7880
7881 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
7882 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatLo)
7883 .addImm(Val: Lo_32(Value: RsrcDataFormat));
7884
7885 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
7886 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: SRsrcFormatHi)
7887 .addImm(Val: Hi_32(Value: RsrcDataFormat));
7888
7889 // NewSRsrc = {Zero64, SRsrcFormat}
7890 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewSRsrc)
7891 .addReg(RegNo: Zero64)
7892 .addImm(Val: AMDGPU::sub0_sub1)
7893 .addReg(RegNo: SRsrcFormatLo)
7894 .addImm(Val: AMDGPU::sub2)
7895 .addReg(RegNo: SRsrcFormatHi)
7896 .addImm(Val: AMDGPU::sub3);
7897
7898 return std::tuple(RsrcPtr, NewSRsrc);
7899}
7900
7901MachineBasicBlock *
7902SIInstrInfo::legalizeOperands(MachineInstr &MI,
7903 MachineDominatorTree *MDT) const {
7904 MachineFunction &MF = *MI.getMF();
7905 MachineRegisterInfo &MRI = MF.getRegInfo();
7906 MachineBasicBlock *CreatedBB = nullptr;
7907
7908 // Legalize True16
7909 if (ST.useRealTrue16Insts())
7910 legalizeOperandsVALUt16(Inst&: MI, MRI);
7911
7912 // Legalize VOP2
7913 if (isVOP2(MI) || isVOPC(MI)) {
7914 legalizeOperandsVOP2(MRI, MI);
7915 return CreatedBB;
7916 }
7917
7918 // Legalize VOP3
7919 if (isVOP3(MI)) {
7920 legalizeOperandsVOP3(MRI, MI);
7921 return CreatedBB;
7922 }
7923
7924 // Legalize SMRD
7925 if (isSMRD(MI)) {
7926 legalizeOperandsSMRD(MRI, MI);
7927 return CreatedBB;
7928 }
7929
7930 // Legalize FLAT
7931 if (isFLAT(MI)) {
7932 legalizeOperandsFLAT(MRI, MI);
7933 return CreatedBB;
7934 }
7935
7936 // Legalize PHI
7937 // The register class of the operands must be the same type as the register
7938 // class of the output.
7939 if (MI.getOpcode() == AMDGPU::PHI) {
7940 const TargetRegisterClass *VRC = getOpRegClass(MI, OpNo: 0);
7941 assert(!RI.isSGPRClass(VRC));
7942
7943 // Update all the operands so they have the same type.
7944 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7945 MachineOperand &Op = MI.getOperand(i: I);
7946 if (!Op.isReg() || !Op.getReg().isVirtual())
7947 continue;
7948
7949 // MI is a PHI instruction.
7950 MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
7951 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
7952
7953 // Avoid creating no-op copies with the same src and dst reg class. These
7954 // confuse some of the machine passes.
7955 legalizeGenericOperand(InsertMBB&: *InsertBB, I: Insert, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7956 }
7957 }
7958
7959 // REG_SEQUENCE doesn't really require operand legalization, but if one has a
7960 // VGPR dest type and SGPR sources, insert copies so all operands are
7961 // VGPRs. This seems to help operand folding / the register coalescer.
7962 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
7963 MachineBasicBlock *MBB = MI.getParent();
7964 const TargetRegisterClass *DstRC = getOpRegClass(MI, OpNo: 0);
7965 if (RI.hasVGPRs(RC: DstRC)) {
7966 // Update all the operands so they are VGPR register classes. These may
7967 // not be the same register class because REG_SEQUENCE supports mixing
7968 // subregister index types e.g. sub0_sub1 + sub2 + sub3
7969 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
7970 MachineOperand &Op = MI.getOperand(i: I);
7971 if (!Op.isReg() || !Op.getReg().isVirtual())
7972 continue;
7973
7974 const TargetRegisterClass *OpRC = MRI.getRegClass(Reg: Op.getReg());
7975 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(SRC: OpRC);
7976 if (VRC == OpRC)
7977 continue;
7978
7979 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC: VRC, Op, MRI, DL: MI.getDebugLoc());
7980 }
7981 }
7982
7983 return CreatedBB;
7984 }
7985
7986 // Legalize INSERT_SUBREG
7987 // src0 must have the same register class as dst
7988 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
7989 Register Dst = MI.getOperand(i: 0).getReg();
7990 Register Src0 = MI.getOperand(i: 1).getReg();
7991 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: Dst);
7992 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0);
7993 if (DstRC != Src0RC) {
7994 MachineBasicBlock *MBB = MI.getParent();
7995 MachineOperand &Op = MI.getOperand(i: 1);
7996 legalizeGenericOperand(InsertMBB&: *MBB, I: MI, DstRC, Op, MRI, DL: MI.getDebugLoc());
7997 }
7998 return CreatedBB;
7999 }
8000
8001 // Legalize SI_INIT_M0
8002 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
8003 MachineOperand &Src = MI.getOperand(i: 0);
8004 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
8005 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
8006 return CreatedBB;
8007 }
8008
8009 // Legalize S_BITREPLICATE, S_QUADMASK and S_WQM
8010 if (MI.getOpcode() == AMDGPU::S_BITREPLICATE_B64_B32 ||
8011 MI.getOpcode() == AMDGPU::S_QUADMASK_B32 ||
8012 MI.getOpcode() == AMDGPU::S_QUADMASK_B64 ||
8013 MI.getOpcode() == AMDGPU::S_WQM_B32 ||
8014 MI.getOpcode() == AMDGPU::S_WQM_B64 ||
8015 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U32 ||
8016 MI.getOpcode() == AMDGPU::S_INVERSE_BALLOT_U64) {
8017 MachineOperand &Src = MI.getOperand(i: 1);
8018 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
8019 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
8020 return CreatedBB;
8021 }
8022
8023 // Legalize MIMG/VIMAGE/VSAMPLE and MUBUF/MTBUF for shaders.
8024 //
8025 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
8026 // scratch memory access. In both cases, the legalization never involves
8027 // conversion to the addr64 form.
8028 if (isImage(MI) || (AMDGPU::isGraphics(CC: MF.getFunction().getCallingConv()) &&
8029 (isMUBUF(MI) || isMTBUF(MI)))) {
8030 AMDGPU::OpName RSrcOpName = (isVIMAGE(MI) || isVSAMPLE(MI))
8031 ? AMDGPU::OpName::rsrc
8032 : AMDGPU::OpName::srsrc;
8033 MachineOperand *SRsrc = getNamedOperand(MI, OperandName: RSrcOpName);
8034 if (SRsrc && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SRsrc->getReg())))
8035 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SRsrc}, MDT);
8036
8037 AMDGPU::OpName SampOpName =
8038 isMIMG(MI) ? AMDGPU::OpName::ssamp : AMDGPU::OpName::samp;
8039 MachineOperand *SSamp = getNamedOperand(MI, OperandName: SampOpName);
8040 if (SSamp && !RI.isSGPRClass(RC: MRI.getRegClass(Reg: SSamp->getReg())))
8041 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {SSamp}, MDT);
8042
8043 return CreatedBB;
8044 }
8045
8046 // Legalize SI_CALL
8047 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
8048 MachineOperand *Dest = &MI.getOperand(i: 0);
8049 if (!RI.isSGPRClass(RC: MRI.getRegClass(Reg: Dest->getReg()))) {
8050 createWaterFallForSiCall(MI: &MI, MDT, ScalarOps: {Dest});
8051 }
8052 }
8053
8054 // Legalize s_sleep_var.
8055 if (MI.getOpcode() == AMDGPU::S_SLEEP_VAR) {
8056 const DebugLoc &DL = MI.getDebugLoc();
8057 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8058 int Src0Idx =
8059 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
8060 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
8061 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: Reg)
8062 .add(MO: Src0);
8063 Src0.ChangeToRegister(Reg, isDef: false);
8064 return nullptr;
8065 }
8066
8067 // Legalize TENSOR_LOAD_TO_LDS_d2/_d4, TENSOR_STORE_FROM_LDS_d2/_d4. All their
8068 // operands are scalar.
8069 if (MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d2 ||
8070 MI.getOpcode() == AMDGPU::TENSOR_LOAD_TO_LDS_d4 ||
8071 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d2 ||
8072 MI.getOpcode() == AMDGPU::TENSOR_STORE_FROM_LDS_d4) {
8073 for (MachineOperand &Src : MI.explicit_operands()) {
8074 if (Src.isReg() && RI.hasVectorRegisters(RC: MRI.getRegClass(Reg: Src.getReg())))
8075 Src.setReg(readlaneVGPRToSGPR(SrcReg: Src.getReg(), UseMI&: MI, MRI));
8076 }
8077 return CreatedBB;
8078 }
8079
8080 // Legalize MUBUF instructions.
8081 bool isSoffsetLegal = true;
8082 int SoffsetIdx =
8083 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::soffset);
8084 if (SoffsetIdx != -1) {
8085 MachineOperand *Soffset = &MI.getOperand(i: SoffsetIdx);
8086 if (Soffset->isReg() && Soffset->getReg().isVirtual() &&
8087 !RI.isSGPRClass(RC: MRI.getRegClass(Reg: Soffset->getReg()))) {
8088 isSoffsetLegal = false;
8089 }
8090 }
8091
8092 bool isRsrcLegal = true;
8093 int RsrcIdx =
8094 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::srsrc);
8095 if (RsrcIdx != -1) {
8096 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
8097 if (Rsrc->isReg() && !RI.isSGPRReg(MRI, Reg: Rsrc->getReg()))
8098 isRsrcLegal = false;
8099 }
8100
8101 // The operands are legal.
8102 if (isRsrcLegal && isSoffsetLegal)
8103 return CreatedBB;
8104
8105 if (!isRsrcLegal) {
8106 // Legalize a VGPR Rsrc
8107 //
8108 // If the instruction is _ADDR64, we can avoid a waterfall by extracting
8109 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
8110 // a zero-value SRsrc.
8111 //
8112 // If the instruction is _OFFSET (both idxen and offen disabled), and we
8113 // support ADDR64 instructions, we can convert to ADDR64 and do the same as
8114 // above.
8115 //
8116 // Otherwise we are on non-ADDR64 hardware, and/or we have
8117 // idxen/offen/bothen and we fall back to a waterfall loop.
8118
8119 MachineOperand *Rsrc = &MI.getOperand(i: RsrcIdx);
8120 MachineBasicBlock &MBB = *MI.getParent();
8121
8122 MachineOperand *VAddr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
8123 if (VAddr && AMDGPU::getIfAddr64Inst(Opcode: MI.getOpcode()) != -1) {
8124 // This is already an ADDR64 instruction so we need to add the pointer
8125 // extracted from the resource descriptor to the current value of VAddr.
8126 Register NewVAddrLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8127 Register NewVAddrHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8128 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
8129
8130 const auto *BoolXExecRC = RI.getWaveMaskRegClass();
8131 Register CondReg0 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
8132 Register CondReg1 = MRI.createVirtualRegister(RegClass: BoolXExecRC);
8133
8134 unsigned RsrcPtr, NewSRsrc;
8135 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
8136
8137 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
8138 const DebugLoc &DL = MI.getDebugLoc();
8139 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: NewVAddrLo)
8140 .addDef(RegNo: CondReg0)
8141 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
8142 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub0)
8143 .addImm(Val: 0);
8144
8145 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
8146 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: NewVAddrHi)
8147 .addDef(RegNo: CondReg1, Flags: RegState::Dead)
8148 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
8149 .addReg(RegNo: VAddr->getReg(), Flags: {}, SubReg: AMDGPU::sub1)
8150 .addReg(RegNo: CondReg0)
8151 .addImm(Val: 0);
8152
8153 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8154 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVAddr)
8155 .addReg(RegNo: NewVAddrLo)
8156 .addImm(Val: AMDGPU::sub0)
8157 .addReg(RegNo: NewVAddrHi)
8158 .addImm(Val: AMDGPU::sub1);
8159
8160 VAddr->setReg(NewVAddr);
8161 Rsrc->setReg(NewSRsrc);
8162 } else if (!VAddr && ST.hasAddr64()) {
8163 // This instructions is the _OFFSET variant, so we need to convert it to
8164 // ADDR64.
8165 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
8166 "FIXME: Need to emit flat atomics here");
8167
8168 unsigned RsrcPtr, NewSRsrc;
8169 std::tie(args&: RsrcPtr, args&: NewSRsrc) = extractRsrcPtr(TII: *this, MI, Rsrc&: *Rsrc);
8170
8171 Register NewVAddr = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
8172 MachineOperand *VData = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata);
8173 MachineOperand *Offset = getNamedOperand(MI, OperandName: AMDGPU::OpName::offset);
8174 MachineOperand *SOffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8175 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(Opcode: MI.getOpcode());
8176
8177 // Atomics with return have an additional tied operand and are
8178 // missing some of the special bits.
8179 MachineOperand *VDataIn = getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata_in);
8180 MachineInstr *Addr64;
8181
8182 if (!VDataIn) {
8183 // Regular buffer load / store.
8184 MachineInstrBuilder MIB =
8185 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
8186 .add(MO: *VData)
8187 .addReg(RegNo: NewVAddr)
8188 .addReg(RegNo: NewSRsrc)
8189 .add(MO: *SOffset)
8190 .add(MO: *Offset);
8191
8192 if (const MachineOperand *CPol =
8193 getNamedOperand(MI, OperandName: AMDGPU::OpName::cpol)) {
8194 MIB.addImm(Val: CPol->getImm());
8195 }
8196
8197 if (const MachineOperand *TFE =
8198 getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe)) {
8199 MIB.addImm(Val: TFE->getImm());
8200 }
8201
8202 MIB.addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::swz));
8203
8204 MIB.cloneMemRefs(OtherMI: MI);
8205 Addr64 = MIB;
8206 } else {
8207 // Atomics with return.
8208 Addr64 = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: Addr64Opcode))
8209 .add(MO: *VData)
8210 .add(MO: *VDataIn)
8211 .addReg(RegNo: NewVAddr)
8212 .addReg(RegNo: NewSRsrc)
8213 .add(MO: *SOffset)
8214 .add(MO: *Offset)
8215 .addImm(Val: getNamedImmOperand(MI, OperandName: AMDGPU::OpName::cpol))
8216 .cloneMemRefs(OtherMI: MI);
8217 }
8218
8219 MI.removeFromParent();
8220
8221 // NewVaddr = {NewVaddrHi, NewVaddrLo}
8222 BuildMI(BB&: MBB, I: Addr64, MIMD: Addr64->getDebugLoc(), MCID: get(Opcode: AMDGPU::REG_SEQUENCE),
8223 DestReg: NewVAddr)
8224 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub0)
8225 .addImm(Val: AMDGPU::sub0)
8226 .addReg(RegNo: RsrcPtr, Flags: {}, SubReg: AMDGPU::sub1)
8227 .addImm(Val: AMDGPU::sub1);
8228 } else {
8229 // Legalize a VGPR Rsrc and soffset together.
8230 if (!isSoffsetLegal) {
8231 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8232 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc, Soffset}, MDT);
8233 return CreatedBB;
8234 }
8235 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Rsrc}, MDT);
8236 return CreatedBB;
8237 }
8238 }
8239
8240 // Legalize a VGPR soffset.
8241 if (!isSoffsetLegal) {
8242 MachineOperand *Soffset = getNamedOperand(MI, OperandName: AMDGPU::OpName::soffset);
8243 CreatedBB = generateWaterFallLoop(TII: *this, MI, ScalarOps: {Soffset}, MDT);
8244 return CreatedBB;
8245 }
8246 return CreatedBB;
8247}
8248
8249void SIInstrWorklist::insert(MachineInstr *MI) {
8250 if (InSet.insert(Ptr: MI).second)
8251 InstrList.push_back(Elt: MI);
8252 // Add MBUF instructiosn to deferred list.
8253 int RsrcIdx =
8254 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::srsrc);
8255 if (RsrcIdx != -1) {
8256 DeferredList.insert(X: MI);
8257 }
8258}
8259
8260bool SIInstrWorklist::isDeferred(MachineInstr *MI) {
8261 return DeferredList.contains(key: MI);
8262}
8263
8264// Legalize size mismatches between 16bit and 32bit registers in v2s copy
8265// lowering (change sgpr to vgpr).
8266// This is mainly caused by 16bit SALU and 16bit VALU using reg with different
8267// size. Need to legalize the size of the operands during the vgpr lowering
8268// chain. This can be removed after we have sgpr16 in place
8269void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI, unsigned OpIdx,
8270 MachineRegisterInfo &MRI) const {
8271 if (!ST.useRealTrue16Insts())
8272 return;
8273
8274 unsigned Opcode = MI.getOpcode();
8275 MachineBasicBlock *MBB = MI.getParent();
8276 // Legalize operands and check for size mismatch
8277 if (OpIdx >= MI.getNumExplicitOperands() ||
8278 OpIdx >= get(Opcode).getNumOperands() ||
8279 get(Opcode).operands()[OpIdx].RegClass == -1)
8280 return;
8281
8282 MachineOperand &Op = MI.getOperand(i: OpIdx);
8283 if (!Op.isReg() || !Op.getReg().isVirtual() || Op.isDef())
8284 return;
8285
8286 const TargetRegisterClass *CurrRC = MRI.getRegClass(Reg: Op.getReg());
8287 if (!RI.isVGPRClass(RC: CurrRC))
8288 return;
8289
8290 int16_t RCID = getOpRegClassID(OpInfo: get(Opcode).operands()[OpIdx]);
8291 const TargetRegisterClass *ExpectedRC = RI.getRegClass(i: RCID);
8292 if (RI.getMatchingSuperRegClass(A: CurrRC, B: ExpectedRC, Idx: AMDGPU::lo16)) {
8293 // Default to the lo16 only if the subregister is not specified.
8294 if (Op.getSubReg() == AMDGPU::NoSubRegister)
8295 Op.setSubReg(AMDGPU::lo16);
8296 return;
8297 }
8298
8299 const TargetRegisterClass *CurrSRC =
8300 RI.getSubRegisterClass(CurrRC, Op.getSubReg());
8301 if (RI.getMatchingSuperRegClass(A: ExpectedRC, B: CurrSRC, Idx: AMDGPU::lo16)) {
8302 const DebugLoc &DL = MI.getDebugLoc();
8303 Register NewDstReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8304 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
8305 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
8306 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
8307 .addReg(RegNo: Op.getReg(), Flags: {}, SubReg: Op.getSubReg())
8308 .addImm(Val: AMDGPU::lo16)
8309 .addReg(RegNo: Undef)
8310 .addImm(Val: AMDGPU::hi16);
8311 Op.setReg(NewDstReg);
8312 Op.setSubReg(AMDGPU::NoSubRegister);
8313 }
8314}
8315void SIInstrInfo::legalizeOperandsVALUt16(MachineInstr &MI,
8316 MachineRegisterInfo &MRI) const {
8317 for (unsigned OpIdx = 0; OpIdx < MI.getNumExplicitOperands(); OpIdx++)
8318 legalizeOperandsVALUt16(MI, OpIdx, MRI);
8319}
8320
8321void SIInstrInfo::createWaterFallForSiCall(MachineInstr *MI,
8322 MachineDominatorTree *MDT,
8323 ArrayRef<MachineOperand *> ScalarOps,
8324 ArrayRef<Register> PhySGPRs) const {
8325 assert(MI->getOpcode() == AMDGPU::SI_CALL_ISEL &&
8326 "This only handle waterfall for SI_CALL_ISEL");
8327 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and
8328 // following copies, we also need to move copies from and to physical
8329 // registers into the loop block.
8330 // Also move the copies to physical registers into the loop block
8331 MachineBasicBlock &MBB = *MI->getParent();
8332 MachineBasicBlock::iterator Start(MI);
8333 while (Start->getOpcode() != AMDGPU::ADJCALLSTACKUP)
8334 --Start;
8335 MachineBasicBlock::iterator End(MI);
8336 while (End->getOpcode() != AMDGPU::ADJCALLSTACKDOWN)
8337 ++End;
8338
8339 // Also include following copies of the return value
8340 ++End;
8341 while (End != MBB.end() && End->isCopy() &&
8342 MI->definesRegister(Reg: End->getOperand(i: 1).getReg(), TRI: &RI))
8343 ++End;
8344
8345 generateWaterFallLoop(TII: *this, MI&: *MI, ScalarOps, MDT, Begin: Start, End, PhySGPRs);
8346}
8347
8348void SIInstrInfo::moveToVALU(SIInstrWorklist &Worklist,
8349 MachineDominatorTree *MDT) const {
8350 DenseMap<MachineInstr *, V2PhysSCopyInfo> WaterFalls;
8351 DenseMap<MachineInstr *, bool> V2SPhyCopiesToErase;
8352 while (!Worklist.empty()) {
8353 MachineInstr &Inst = *Worklist.top();
8354 Worklist.erase_top();
8355 // Skip MachineInstr in the deferred list.
8356 if (Worklist.isDeferred(MI: &Inst))
8357 continue;
8358 moveToVALUImpl(Worklist, MDT, Inst, WaterFalls, V2SPhyCopiesToErase);
8359 }
8360
8361 // Deferred list of instructions will be processed once
8362 // all the MachineInstr in the worklist are done.
8363 for (MachineInstr *Inst : Worklist.getDeferredList()) {
8364 moveToVALUImpl(Worklist, MDT, Inst&: *Inst, WaterFalls, V2SPhyCopiesToErase);
8365 assert(Worklist.empty() &&
8366 "Deferred MachineInstr are not supposed to re-populate worklist");
8367 }
8368
8369 for (auto &Entry : WaterFalls) {
8370 if (Entry.first->getOpcode() == AMDGPU::SI_CALL_ISEL)
8371 createWaterFallForSiCall(MI: Entry.first, MDT, ScalarOps: Entry.second.MOs,
8372 PhySGPRs: Entry.second.SGPRs);
8373 }
8374
8375 for (std::pair<MachineInstr *, bool> Entry : V2SPhyCopiesToErase)
8376 if (Entry.second)
8377 Entry.first->eraseFromParent();
8378}
8379void SIInstrInfo::createReadFirstLaneFromCopyToPhysReg(
8380 MachineRegisterInfo &MRI, Register DstReg, MachineInstr &Inst) const {
8381 // If it's a copy of a VGPR to a physical SGPR, insert a V_READFIRSTLANE and
8382 // hope for the best.
8383 const TargetRegisterClass *DstRC = RI.getRegClassForReg(MRI, Reg: DstReg);
8384 ArrayRef<int16_t> SubRegIndices = RI.getRegSplitParts(RC: DstRC, EltSize: 4);
8385 if (SubRegIndices.size() <= 1) {
8386 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8387 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8388 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8389 .add(MO: Inst.getOperand(i: 1));
8390 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY),
8391 DestReg: DstReg)
8392 .addReg(RegNo: NewDst);
8393 } else {
8394 SmallVector<Register, 8> DstRegs;
8395 for (int16_t Indice : SubRegIndices) {
8396 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
8397 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8398 MCID: get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: NewDst)
8399 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: Indice);
8400
8401 DstRegs.push_back(Elt: NewDst);
8402 }
8403 MachineInstrBuilder MIB =
8404 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
8405 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
8406 for (unsigned i = 0; i < SubRegIndices.size(); ++i) {
8407 MIB.addReg(RegNo: DstRegs[i]);
8408 MIB.addImm(Val: RI.getSubRegFromChannel(Channel: i));
8409 }
8410 }
8411}
8412
8413void SIInstrInfo::handleCopyToPhysHelper(
8414 SIInstrWorklist &Worklist, Register DstReg, MachineInstr &Inst,
8415 MachineRegisterInfo &MRI,
8416 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8417 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8418 if (DstReg == AMDGPU::M0) {
8419 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8420 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8421 return;
8422 }
8423 Register SrcReg = Inst.getOperand(i: 1).getReg();
8424 MachineBasicBlock::iterator I = Inst.getIterator();
8425 MachineBasicBlock::iterator E = Inst.getParent()->end();
8426 // Only search current block since phyreg's def & use cannot cross
8427 // blocks when MF.NoPhi = false.
8428 while (++I != E) {
8429 // For SI_CALL_ISEL users, replace the phys SGPR with the VGPR source
8430 // and record the operand for later waterfall loop generation.
8431 if (I->getOpcode() == AMDGPU::SI_CALL_ISEL) {
8432 MachineInstr *UseMI = &*I;
8433 for (unsigned i = 0; i < UseMI->getNumOperands(); ++i) {
8434 if (UseMI->getOperand(i).isReg() &&
8435 UseMI->getOperand(i).getReg() == DstReg) {
8436 MachineOperand *MO = &UseMI->getOperand(i);
8437 MO->setReg(SrcReg);
8438 V2PhysSCopyInfo &V2SCopyInfo = WaterFalls[UseMI];
8439 V2SCopyInfo.MOs.push_back(Elt: MO);
8440 V2SCopyInfo.SGPRs.push_back(Elt: DstReg);
8441 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8442 }
8443 }
8444 } else if (I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG &&
8445 I->getOperand(i: 0).isReg() &&
8446 I->getOperand(i: 0).getReg() == DstReg) {
8447 createReadFirstLaneFromCopyToPhysReg(MRI, DstReg, Inst);
8448 V2SPhyCopiesToErase.try_emplace(Key: &Inst, Args: true);
8449 } else if (I->readsRegister(Reg: DstReg, TRI: &RI)) {
8450 // COPY cannot be erased if other type of inst uses it.
8451 V2SPhyCopiesToErase[&Inst] = false;
8452 }
8453 if (I->findRegisterDefOperand(Reg: DstReg, TRI: &RI))
8454 break;
8455 }
8456}
8457
8458void SIInstrInfo::moveToVALUImpl(
8459 SIInstrWorklist &Worklist, MachineDominatorTree *MDT, MachineInstr &Inst,
8460 DenseMap<MachineInstr *, V2PhysSCopyInfo> &WaterFalls,
8461 DenseMap<MachineInstr *, bool> &V2SPhyCopiesToErase) const {
8462
8463 MachineBasicBlock *MBB = Inst.getParent();
8464 if (!MBB)
8465 return;
8466 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8467 unsigned Opcode = Inst.getOpcode();
8468 unsigned NewOpcode = getVALUOp(MI: Inst);
8469 const DebugLoc &DL = Inst.getDebugLoc();
8470
8471 // Handle some special cases
8472 switch (Opcode) {
8473 default:
8474 break;
8475 case AMDGPU::S_ADD_I32:
8476 case AMDGPU::S_SUB_I32: {
8477 // FIXME: The u32 versions currently selected use the carry.
8478 bool Changed;
8479 MachineBasicBlock *CreatedBBTmp = nullptr;
8480 std::tie(args&: Changed, args&: CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT);
8481 if (Changed)
8482 return;
8483
8484 // Default handling
8485 break;
8486 }
8487
8488 case AMDGPU::S_MUL_U64:
8489 if (ST.useVMulU64Inst()) {
8490 NewOpcode = AMDGPU::V_MUL_U64_e64;
8491 break;
8492 }
8493 // Split s_mul_u64 in 32-bit vector multiplications.
8494 splitScalarSMulU64(Worklist, Inst, MDT);
8495 Inst.eraseFromParent();
8496 return;
8497
8498 case AMDGPU::S_MUL_U64_U32_PSEUDO:
8499 case AMDGPU::S_MUL_I64_I32_PSEUDO:
8500 // This is a special case of s_mul_u64 where all the operands are either
8501 // zero extended or sign extended.
8502 splitScalarSMulPseudo(Worklist, Inst, MDT);
8503 Inst.eraseFromParent();
8504 return;
8505
8506 case AMDGPU::S_AND_B64:
8507 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_AND_B32, MDT);
8508 Inst.eraseFromParent();
8509 return;
8510
8511 case AMDGPU::S_OR_B64:
8512 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_OR_B32, MDT);
8513 Inst.eraseFromParent();
8514 return;
8515
8516 case AMDGPU::S_XOR_B64:
8517 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XOR_B32, MDT);
8518 Inst.eraseFromParent();
8519 return;
8520
8521 case AMDGPU::S_NAND_B64:
8522 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NAND_B32, MDT);
8523 Inst.eraseFromParent();
8524 return;
8525
8526 case AMDGPU::S_NOR_B64:
8527 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOR_B32, MDT);
8528 Inst.eraseFromParent();
8529 return;
8530
8531 case AMDGPU::S_XNOR_B64:
8532 if (ST.hasDLInsts())
8533 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_XNOR_B32, MDT);
8534 else
8535 splitScalar64BitXnor(Worklist, Inst, MDT);
8536 Inst.eraseFromParent();
8537 return;
8538
8539 case AMDGPU::S_ANDN2_B64:
8540 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ANDN2_B32, MDT);
8541 Inst.eraseFromParent();
8542 return;
8543
8544 case AMDGPU::S_ORN2_B64:
8545 splitScalar64BitBinaryOp(Worklist, Inst, Opcode: AMDGPU::S_ORN2_B32, MDT);
8546 Inst.eraseFromParent();
8547 return;
8548
8549 case AMDGPU::S_BREV_B64:
8550 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_BREV_B32, Swap: true);
8551 Inst.eraseFromParent();
8552 return;
8553
8554 case AMDGPU::S_NOT_B64:
8555 splitScalar64BitUnaryOp(Worklist, Inst, Opcode: AMDGPU::S_NOT_B32);
8556 Inst.eraseFromParent();
8557 return;
8558
8559 case AMDGPU::S_BCNT1_I32_B64:
8560 splitScalar64BitBCNT(Worklist, Inst);
8561 Inst.eraseFromParent();
8562 return;
8563
8564 case AMDGPU::S_BFE_I64:
8565 splitScalar64BitBFE(Worklist, Inst);
8566 Inst.eraseFromParent();
8567 return;
8568
8569 case AMDGPU::S_FLBIT_I32_B64:
8570 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBH_U32_e32);
8571 Inst.eraseFromParent();
8572 return;
8573 case AMDGPU::S_FF1_I32_B64:
8574 splitScalar64BitCountOp(Worklist, Inst, Opcode: AMDGPU::V_FFBL_B32_e32);
8575 Inst.eraseFromParent();
8576 return;
8577
8578 case AMDGPU::S_LSHL_B32:
8579 if (ST.hasOnlyRevVALUShifts()) {
8580 NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
8581 swapOperands(Inst);
8582 }
8583 break;
8584 case AMDGPU::S_ASHR_I32:
8585 if (ST.hasOnlyRevVALUShifts()) {
8586 NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
8587 swapOperands(Inst);
8588 }
8589 break;
8590 case AMDGPU::S_LSHR_B32:
8591 if (ST.hasOnlyRevVALUShifts()) {
8592 NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
8593 swapOperands(Inst);
8594 }
8595 break;
8596 case AMDGPU::S_LSHL_B64:
8597 if (ST.hasOnlyRevVALUShifts()) {
8598 NewOpcode = ST.getGeneration() >= AMDGPUSubtarget::GFX12
8599 ? AMDGPU::V_LSHLREV_B64_pseudo_e64
8600 : AMDGPU::V_LSHLREV_B64_e64;
8601 swapOperands(Inst);
8602 }
8603 break;
8604 case AMDGPU::S_ASHR_I64:
8605 if (ST.hasOnlyRevVALUShifts()) {
8606 NewOpcode = AMDGPU::V_ASHRREV_I64_e64;
8607 swapOperands(Inst);
8608 }
8609 break;
8610 case AMDGPU::S_LSHR_B64:
8611 if (ST.hasOnlyRevVALUShifts()) {
8612 NewOpcode = AMDGPU::V_LSHRREV_B64_e64;
8613 swapOperands(Inst);
8614 }
8615 break;
8616
8617 case AMDGPU::S_ABS_I32:
8618 lowerScalarAbs(Worklist, Inst);
8619 Inst.eraseFromParent();
8620 return;
8621
8622 case AMDGPU::S_ABSDIFF_I32:
8623 lowerScalarAbsDiff(Worklist, Inst);
8624 Inst.eraseFromParent();
8625 return;
8626
8627 case AMDGPU::S_CBRANCH_SCC0:
8628 case AMDGPU::S_CBRANCH_SCC1: {
8629 // Clear unused bits of vcc
8630 Register CondReg = Inst.getOperand(i: 1).getReg();
8631 bool IsSCC = CondReg == AMDGPU::SCC;
8632 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
8633 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: LMC.AndOpc), DestReg: LMC.VccReg)
8634 .addReg(RegNo: LMC.ExecReg)
8635 .addReg(RegNo: IsSCC ? LMC.VccReg : CondReg)
8636 .setOperandDead(3); // implicit-def $scc
8637 Inst.removeOperand(OpNo: 1);
8638 } break;
8639
8640 case AMDGPU::S_BFE_U64:
8641 case AMDGPU::S_BFM_B64:
8642 llvm_unreachable("Moving this op to VALU not implemented");
8643
8644 case AMDGPU::S_PACK_LL_B32_B16:
8645 case AMDGPU::S_PACK_LH_B32_B16:
8646 case AMDGPU::S_PACK_HL_B32_B16:
8647 case AMDGPU::S_PACK_HH_B32_B16:
8648 movePackToVALU(Worklist, MRI, Inst);
8649 Inst.eraseFromParent();
8650 return;
8651
8652 case AMDGPU::S_XNOR_B32:
8653 lowerScalarXnor(Worklist, Inst);
8654 Inst.eraseFromParent();
8655 return;
8656
8657 case AMDGPU::S_NAND_B32:
8658 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8659 Inst.eraseFromParent();
8660 return;
8661
8662 case AMDGPU::S_NOR_B32:
8663 splitScalarNotBinop(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8664 Inst.eraseFromParent();
8665 return;
8666
8667 case AMDGPU::S_ANDN2_B32:
8668 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_AND_B32);
8669 Inst.eraseFromParent();
8670 return;
8671
8672 case AMDGPU::S_ORN2_B32:
8673 splitScalarBinOpN2(Worklist, Inst, Opcode: AMDGPU::S_OR_B32);
8674 Inst.eraseFromParent();
8675 return;
8676
8677 // TODO: remove as soon as everything is ready
8678 // to replace VGPR to SGPR copy with V_READFIRSTLANEs.
8679 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO
8680 // can only be selected from the uniform SDNode.
8681 case AMDGPU::S_ADD_CO_PSEUDO:
8682 case AMDGPU::S_SUB_CO_PSEUDO: {
8683 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
8684 ? AMDGPU::V_ADDC_U32_e64
8685 : AMDGPU::V_SUBB_U32_e64;
8686 const auto *CarryRC = RI.getWaveMaskRegClass();
8687
8688 Register CarryInReg = Inst.getOperand(i: 4).getReg();
8689 if (!MRI.constrainRegClass(Reg: CarryInReg, RC: CarryRC)) {
8690 Register NewCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
8691 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCarryReg)
8692 .addReg(RegNo: CarryInReg);
8693 }
8694
8695 Register CarryOutReg = Inst.getOperand(i: 1).getReg();
8696
8697 Register DestReg = MRI.createVirtualRegister(RegClass: RI.getEquivalentVGPRClass(
8698 SRC: MRI.getRegClass(Reg: Inst.getOperand(i: 0).getReg())));
8699 MachineInstr *CarryOp =
8700 BuildMI(BB&: *MBB, I: &Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: Opc), DestReg)
8701 .addReg(RegNo: CarryOutReg, Flags: RegState::Define)
8702 .add(MO: Inst.getOperand(i: 2))
8703 .add(MO: Inst.getOperand(i: 3))
8704 .addReg(RegNo: CarryInReg)
8705 .addImm(Val: 0);
8706 legalizeOperands(MI&: *CarryOp);
8707 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: DestReg);
8708 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8709 Inst.eraseFromParent();
8710 }
8711 return;
8712 case AMDGPU::S_UADDO_PSEUDO:
8713 case AMDGPU::S_USUBO_PSEUDO: {
8714 MachineOperand &Dest0 = Inst.getOperand(i: 0);
8715 MachineOperand &Dest1 = Inst.getOperand(i: 1);
8716 MachineOperand &Src0 = Inst.getOperand(i: 2);
8717 MachineOperand &Src1 = Inst.getOperand(i: 3);
8718
8719 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
8720 ? AMDGPU::V_ADD_CO_U32_e64
8721 : AMDGPU::V_SUB_CO_U32_e64;
8722 const TargetRegisterClass *NewRC =
8723 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest0.getReg()));
8724 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8725 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
8726 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
8727 .add(MO: Src0)
8728 .add(MO: Src1)
8729 .addImm(Val: 0); // clamp bit
8730
8731 legalizeOperands(MI&: *NewInstr, MDT);
8732 MRI.replaceRegWith(FromReg: Dest0.getReg(), ToReg: DestReg);
8733 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8734 Inst.eraseFromParent();
8735 }
8736 return;
8737 case AMDGPU::S_LSHL1_ADD_U32:
8738 case AMDGPU::S_LSHL2_ADD_U32:
8739 case AMDGPU::S_LSHL3_ADD_U32:
8740 case AMDGPU::S_LSHL4_ADD_U32: {
8741 MachineOperand &Dest = Inst.getOperand(i: 0);
8742 MachineOperand &Src0 = Inst.getOperand(i: 1);
8743 MachineOperand &Src1 = Inst.getOperand(i: 2);
8744 unsigned ShiftAmt = (Opcode == AMDGPU::S_LSHL1_ADD_U32 ? 1
8745 : Opcode == AMDGPU::S_LSHL2_ADD_U32 ? 2
8746 : Opcode == AMDGPU::S_LSHL3_ADD_U32 ? 3
8747 : 4);
8748
8749 const TargetRegisterClass *NewRC =
8750 RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg()));
8751 Register DestReg = MRI.createVirtualRegister(RegClass: NewRC);
8752 MachineInstr *NewInstr =
8753 BuildMI(BB&: *MBB, I: &Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_ADD_U32_e64), DestReg)
8754 .add(MO: Src0)
8755 .addImm(Val: ShiftAmt)
8756 .add(MO: Src1);
8757
8758 legalizeOperands(MI&: *NewInstr, MDT);
8759 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: DestReg);
8760 addUsersToMoveToVALUWorklist(Reg: DestReg, MRI, Worklist);
8761 Inst.eraseFromParent();
8762 }
8763 return;
8764 case AMDGPU::S_CSELECT_B32:
8765 case AMDGPU::S_CSELECT_B64:
8766 lowerSelect(Worklist, Inst, MDT);
8767 Inst.eraseFromParent();
8768 return;
8769 case AMDGPU::S_CMP_EQ_I32:
8770 case AMDGPU::S_CMP_LG_I32:
8771 case AMDGPU::S_CMP_GT_I32:
8772 case AMDGPU::S_CMP_GE_I32:
8773 case AMDGPU::S_CMP_LT_I32:
8774 case AMDGPU::S_CMP_LE_I32:
8775 case AMDGPU::S_CMP_EQ_U32:
8776 case AMDGPU::S_CMP_LG_U32:
8777 case AMDGPU::S_CMP_GT_U32:
8778 case AMDGPU::S_CMP_GE_U32:
8779 case AMDGPU::S_CMP_LT_U32:
8780 case AMDGPU::S_CMP_LE_U32:
8781 case AMDGPU::S_CMP_EQ_U64:
8782 case AMDGPU::S_CMP_LG_U64:
8783 case AMDGPU::S_CMP_LT_F32:
8784 case AMDGPU::S_CMP_EQ_F32:
8785 case AMDGPU::S_CMP_LE_F32:
8786 case AMDGPU::S_CMP_GT_F32:
8787 case AMDGPU::S_CMP_LG_F32:
8788 case AMDGPU::S_CMP_GE_F32:
8789 case AMDGPU::S_CMP_O_F32:
8790 case AMDGPU::S_CMP_U_F32:
8791 case AMDGPU::S_CMP_NGE_F32:
8792 case AMDGPU::S_CMP_NLG_F32:
8793 case AMDGPU::S_CMP_NGT_F32:
8794 case AMDGPU::S_CMP_NLE_F32:
8795 case AMDGPU::S_CMP_NEQ_F32:
8796 case AMDGPU::S_CMP_NLT_F32: {
8797 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8798 auto NewInstr =
8799 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8800 .setMIFlags(Inst.getFlags());
8801 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src0_modifiers) >=
8802 0) {
8803 NewInstr
8804 .addImm(Val: 0) // src0_modifiers
8805 .add(MO: Inst.getOperand(i: 0)) // src0
8806 .addImm(Val: 0) // src1_modifiers
8807 .add(MO: Inst.getOperand(i: 1)) // src1
8808 .addImm(Val: 0); // clamp
8809 } else {
8810 NewInstr.add(MO: Inst.getOperand(i: 0)).add(MO: Inst.getOperand(i: 1));
8811 }
8812 legalizeOperands(MI&: *NewInstr, MDT);
8813 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8814 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8815 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8816 Inst.eraseFromParent();
8817 return;
8818 }
8819 case AMDGPU::S_CMP_LT_F16:
8820 case AMDGPU::S_CMP_EQ_F16:
8821 case AMDGPU::S_CMP_LE_F16:
8822 case AMDGPU::S_CMP_GT_F16:
8823 case AMDGPU::S_CMP_LG_F16:
8824 case AMDGPU::S_CMP_GE_F16:
8825 case AMDGPU::S_CMP_O_F16:
8826 case AMDGPU::S_CMP_U_F16:
8827 case AMDGPU::S_CMP_NGE_F16:
8828 case AMDGPU::S_CMP_NLG_F16:
8829 case AMDGPU::S_CMP_NGT_F16:
8830 case AMDGPU::S_CMP_NLE_F16:
8831 case AMDGPU::S_CMP_NEQ_F16:
8832 case AMDGPU::S_CMP_NLT_F16: {
8833 Register CondReg = MRI.createVirtualRegister(RegClass: RI.getWaveMaskRegClass());
8834 auto NewInstr =
8835 BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode), DestReg: CondReg)
8836 .setMIFlags(Inst.getFlags());
8837 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
8838 NewInstr
8839 .addImm(Val: 0) // src0_modifiers
8840 .add(MO: Inst.getOperand(i: 0)) // src0
8841 .addImm(Val: 0) // src1_modifiers
8842 .add(MO: Inst.getOperand(i: 1)) // src1
8843 .addImm(Val: 0); // clamp
8844 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8845 NewInstr.addImm(Val: 0); // op_sel0
8846 } else {
8847 NewInstr
8848 .add(MO: Inst.getOperand(i: 0))
8849 .add(MO: Inst.getOperand(i: 1));
8850 }
8851 legalizeOperands(MI&: *NewInstr, MDT);
8852 int SCCIdx = Inst.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
8853 const MachineOperand &SCCOp = Inst.getOperand(i: SCCIdx);
8854 addSCCDefUsersToVALUWorklist(Op: SCCOp, SCCDefInst&: Inst, Worklist, NewCond: CondReg);
8855 Inst.eraseFromParent();
8856 return;
8857 }
8858 case AMDGPU::S_CVT_HI_F32_F16: {
8859 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8860 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8861 if (ST.useRealTrue16Insts()) {
8862 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: TmpReg)
8863 .add(MO: Inst.getOperand(i: 1));
8864 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8865 .addImm(Val: 0) // src0_modifiers
8866 .addReg(RegNo: TmpReg, Flags: {}, SubReg: AMDGPU::hi16)
8867 .addImm(Val: 0) // clamp
8868 .addImm(Val: 0) // omod
8869 .addImm(Val: 0); // op_sel0
8870 } else {
8871 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
8872 .addImm(Val: 16)
8873 .add(MO: Inst.getOperand(i: 1));
8874 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8875 .addImm(Val: 0) // src0_modifiers
8876 .addReg(RegNo: TmpReg)
8877 .addImm(Val: 0) // clamp
8878 .addImm(Val: 0); // omod
8879 }
8880
8881 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8882 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8883 Inst.eraseFromParent();
8884 return;
8885 }
8886 case AMDGPU::S_MINIMUM_F32:
8887 case AMDGPU::S_MAXIMUM_F32: {
8888 Register NewDst = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
8889 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8890 .addImm(Val: 0) // src0_modifiers
8891 .add(MO: Inst.getOperand(i: 1))
8892 .addImm(Val: 0) // src1_modifiers
8893 .add(MO: Inst.getOperand(i: 2))
8894 .addImm(Val: 0) // clamp
8895 .addImm(Val: 0); // omod
8896 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8897
8898 legalizeOperands(MI&: *NewInstr, MDT);
8899 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8900 Inst.eraseFromParent();
8901 return;
8902 }
8903 case AMDGPU::S_MINIMUM_F16:
8904 case AMDGPU::S_MAXIMUM_F16: {
8905 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8906 ? &AMDGPU::VGPR_16RegClass
8907 : &AMDGPU::VGPR_32RegClass);
8908 MachineInstr *NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8909 .addImm(Val: 0) // src0_modifiers
8910 .add(MO: Inst.getOperand(i: 1))
8911 .addImm(Val: 0) // src1_modifiers
8912 .add(MO: Inst.getOperand(i: 2))
8913 .addImm(Val: 0) // clamp
8914 .addImm(Val: 0) // omod
8915 .addImm(Val: 0); // opsel0
8916 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8917 legalizeOperands(MI&: *NewInstr, MDT);
8918 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8919 Inst.eraseFromParent();
8920 return;
8921 }
8922 case AMDGPU::V_S_EXP_F16_e64:
8923 case AMDGPU::V_S_LOG_F16_e64:
8924 case AMDGPU::V_S_RCP_F16_e64:
8925 case AMDGPU::V_S_RSQ_F16_e64:
8926 case AMDGPU::V_S_SQRT_F16_e64: {
8927 Register NewDst = MRI.createVirtualRegister(RegClass: ST.useRealTrue16Insts()
8928 ? &AMDGPU::VGPR_16RegClass
8929 : &AMDGPU::VGPR_32RegClass);
8930 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: NewOpcode), DestReg: NewDst)
8931 .add(MO: Inst.getOperand(i: 1)) // src0_modifiers
8932 .add(MO: Inst.getOperand(i: 2))
8933 .add(MO: Inst.getOperand(i: 3)) // clamp
8934 .add(MO: Inst.getOperand(i: 4)) // omod
8935 .setMIFlags(Inst.getFlags());
8936 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::op_sel))
8937 NewInstr.addImm(Val: 0); // opsel0
8938 MRI.replaceRegWith(FromReg: Inst.getOperand(i: 0).getReg(), ToReg: NewDst);
8939 legalizeOperands(MI&: *NewInstr, MDT);
8940 addUsersToMoveToVALUWorklist(Reg: NewDst, MRI, Worklist);
8941 Inst.eraseFromParent();
8942 return;
8943 }
8944 }
8945
8946 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
8947 // We cannot move this instruction to the VALU, so we should try to
8948 // legalize its operands instead.
8949 legalizeOperands(MI&: Inst, MDT);
8950 return;
8951 }
8952 // Handle converting generic instructions like COPY-to-SGPR into
8953 // COPY-to-VGPR.
8954 if (NewOpcode == Opcode) {
8955 Register DstReg = Inst.getOperand(i: 0).getReg();
8956 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
8957
8958 if (Inst.isCopy() && DstReg.isPhysical() &&
8959 Inst.getOperand(i: 1).getReg().isVirtual()) {
8960 handleCopyToPhysHelper(Worklist, DstReg, Inst, MRI, WaterFalls,
8961 V2SPhyCopiesToErase);
8962 return;
8963 }
8964
8965 if (Inst.isCopy() && Inst.getOperand(i: 1).getReg().isVirtual()) {
8966 Register NewDstReg = Inst.getOperand(i: 1).getReg();
8967 const TargetRegisterClass *SrcRC = RI.getRegClassForReg(MRI, Reg: NewDstReg);
8968 if (const TargetRegisterClass *CommonRC =
8969 RI.getCommonSubClass(A: NewDstRC, B: SrcRC)) {
8970 // Instead of creating a copy where src and dst are the same register
8971 // class, we just replace all uses of dst with src. These kinds of
8972 // copies interfere with the heuristics MachineSink uses to decide
8973 // whether or not to split a critical edge. Since the pass assumes
8974 // that copies will end up as machine instructions and not be
8975 // eliminated.
8976 addUsersToMoveToVALUWorklist(Reg: DstReg, MRI, Worklist);
8977 unsigned SrcSubReg = Inst.getOperand(i: 1).getSubReg();
8978 bool IsUndef = Inst.getOperand(i: 1).isUndef();
8979 for (MachineOperand &UseMO :
8980 make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
8981 UseMO.setSubReg(
8982 RI.composeSubRegIndices(a: SrcSubReg, b: UseMO.getSubReg()));
8983 UseMO.setReg(NewDstReg);
8984 if (IsUndef)
8985 UseMO.setIsUndef();
8986 }
8987 MRI.clearKillFlags(Reg: NewDstReg);
8988
8989 if (!MRI.constrainRegClass(Reg: NewDstReg, RC: CommonRC))
8990 llvm_unreachable("failed to constrain register");
8991
8992 Inst.eraseFromParent();
8993
8994 for (MachineOperand &UseMO :
8995 make_early_inc_range(Range: MRI.use_operands(Reg: NewDstReg))) {
8996 MachineInstr &UseMI = *UseMO.getParent();
8997
8998 // Legalize t16 operands since replaceReg is called after
8999 // addUsersToVALU.
9000 legalizeOperandsVALUt16(MI&: UseMI, MRI);
9001
9002 unsigned OpIdx = UseMI.getOperandNo(I: &UseMO);
9003 if (const TargetRegisterClass *OpRC =
9004 getRegClass(MCID: UseMI.getDesc(), OpNum: OpIdx))
9005 MRI.constrainRegClass(Reg: NewDstReg, RC: OpRC);
9006 }
9007
9008 return;
9009 }
9010 }
9011
9012 // If this is a v2s copy between 16bit and 32bit reg,
9013 // replace vgpr copy to reg_sequence/extract_subreg
9014 // This can be remove after we have sgpr16 in place
9015 if (ST.useRealTrue16Insts() && Inst.isCopy() &&
9016 Inst.getOperand(i: 1).getReg().isVirtual() &&
9017 RI.isVGPR(MRI, Reg: Inst.getOperand(i: 1).getReg())) {
9018 const TargetRegisterClass *SrcRegRC = getOpRegClass(MI: Inst, OpNo: 1);
9019 if (RI.getMatchingSuperRegClass(A: NewDstRC, B: SrcRegRC, Idx: AMDGPU::lo16)) {
9020 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9021 Register Undef = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
9022 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
9023 MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
9024 BuildMI(BB&: *Inst.getParent(), I: &Inst, MIMD: Inst.getDebugLoc(),
9025 MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDstReg)
9026 .addReg(RegNo: Inst.getOperand(i: 1).getReg())
9027 .addImm(Val: AMDGPU::lo16)
9028 .addReg(RegNo: Undef)
9029 .addImm(Val: AMDGPU::hi16);
9030 Inst.eraseFromParent();
9031 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9032 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9033 return;
9034 } else if (RI.getMatchingSuperRegClass(A: SrcRegRC, B: NewDstRC,
9035 Idx: AMDGPU::lo16)) {
9036 Inst.getOperand(i: 1).setSubReg(AMDGPU::lo16);
9037 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9038 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9039 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9040 return;
9041 }
9042 }
9043
9044 Register NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9045 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9046 legalizeOperands(MI&: Inst, MDT);
9047 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9048 return;
9049 }
9050
9051 // Use the new VALU Opcode.
9052 auto NewInstr = BuildMI(BB&: *MBB, I&: Inst, MIMD: Inst.getDebugLoc(), MCID: get(Opcode: NewOpcode))
9053 .setMIFlags(Inst.getFlags());
9054 if (isVOP3(Opcode: NewOpcode) && !isVOP3(Opcode)) {
9055 // Intersperse VOP3 modifiers among the SALU operands.
9056 NewInstr->addOperand(Op: Inst.getOperand(i: 0));
9057 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
9058 Name: AMDGPU::OpName::src0_modifiers) >= 0)
9059 NewInstr.addImm(Val: 0);
9060 if (AMDGPU::hasNamedOperand(Opcode: NewOpcode, NamedIdx: AMDGPU::OpName::src0)) {
9061 const MachineOperand &Src = Inst.getOperand(i: 1);
9062 NewInstr->addOperand(Op: Src);
9063 }
9064
9065 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
9066 // We are converting these to a BFE, so we need to add the missing
9067 // operands for the size and offset.
9068 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
9069 NewInstr.addImm(Val: 0);
9070 NewInstr.addImm(Val: Size);
9071 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
9072 // The VALU version adds the second operand to the result, so insert an
9073 // extra 0 operand.
9074 NewInstr.addImm(Val: 0);
9075 } else if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
9076 const MachineOperand &OffsetWidthOp = Inst.getOperand(i: 2);
9077 // If we need to move this to VGPRs, we need to unpack the second
9078 // operand back into the 2 separate ones for bit offset and width.
9079 assert(OffsetWidthOp.isImm() &&
9080 "Scalar BFE is only implemented for constant width and offset");
9081 uint32_t Imm = OffsetWidthOp.getImm();
9082
9083 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9084 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9085 NewInstr.addImm(Val: Offset);
9086 NewInstr.addImm(Val: BitWidth);
9087 } else {
9088 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
9089 Name: AMDGPU::OpName::src1_modifiers) >= 0)
9090 NewInstr.addImm(Val: 0);
9091 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src1) >= 0)
9092 NewInstr->addOperand(Op: Inst.getOperand(i: 2));
9093 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode,
9094 Name: AMDGPU::OpName::src2_modifiers) >= 0)
9095 NewInstr.addImm(Val: 0);
9096 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::src2) >= 0)
9097 NewInstr->addOperand(Op: Inst.getOperand(i: 3));
9098 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::clamp) >= 0)
9099 NewInstr.addImm(Val: 0);
9100 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::omod) >= 0)
9101 NewInstr.addImm(Val: 0);
9102 if (AMDGPU::getNamedOperandIdx(Opcode: NewOpcode, Name: AMDGPU::OpName::op_sel) >= 0)
9103 NewInstr.addImm(Val: 0);
9104 }
9105 } else {
9106 // Just copy the SALU operands.
9107 for (const MachineOperand &Op : Inst.explicit_operands())
9108 NewInstr->addOperand(Op);
9109 }
9110
9111 // Remove any references to SCC. Vector instructions can't read from it, and
9112 // We're just about to add the implicit use / defs of VCC, and we don't want
9113 // both.
9114 bool DeadSCCDef = false;
9115 for (MachineOperand &Op : Inst.implicit_operands()) {
9116 if (Op.getReg() == AMDGPU::SCC) {
9117 // Only propagate through live-def of SCC.
9118 if (Op.isDef()) {
9119 if (Op.isDead())
9120 DeadSCCDef = true;
9121 else
9122 addSCCDefUsersToVALUWorklist(Op, SCCDefInst&: Inst, Worklist);
9123 continue;
9124 }
9125
9126 addSCCDefsToVALUWorklist(SCCUseInst: NewInstr, Worklist);
9127 }
9128 }
9129 Inst.eraseFromParent();
9130 Register NewDstReg;
9131 if (NewInstr->getOperand(i: 0).isReg() && NewInstr->getOperand(i: 0).isDef()) {
9132 Register DstReg = NewInstr->getOperand(i: 0).getReg();
9133 assert(DstReg.isVirtual());
9134 // Update the destination register class.
9135 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst: *NewInstr);
9136 assert(NewDstRC);
9137 NewDstReg = MRI.createVirtualRegister(RegClass: NewDstRC);
9138 MRI.replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
9139 }
9140 fixImplicitOperands(MI&: *NewInstr);
9141
9142 if (DeadSCCDef) {
9143 // A scalar op with a dead SCC def lowers to a VALU op whose VCC def will
9144 // also be dead.
9145 if (MachineOperand *VCCDef =
9146 NewInstr->findRegisterDefOperand(Reg: RI.getVCC(), TRI: &RI))
9147 VCCDef->setIsDead();
9148 }
9149
9150 // Legalize the operands
9151 legalizeOperands(MI&: *NewInstr, MDT);
9152 if (NewDstReg)
9153 addUsersToMoveToVALUWorklist(Reg: NewDstReg, MRI, Worklist);
9154}
9155
9156// Add/sub require special handling to deal with carry outs.
9157std::pair<bool, MachineBasicBlock *>
9158SIInstrInfo::moveScalarAddSub(SIInstrWorklist &Worklist, MachineInstr &Inst,
9159 MachineDominatorTree *MDT) const {
9160 if (ST.hasAddNoCarryInsts()) {
9161 // Assume there is no user of scc since we don't select this in that case.
9162 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
9163 // is used.
9164
9165 MachineBasicBlock &MBB = *Inst.getParent();
9166 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9167
9168 Register OldDstReg = Inst.getOperand(i: 0).getReg();
9169 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9170
9171 unsigned Opc = Inst.getOpcode();
9172 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
9173
9174 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
9175 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
9176
9177 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
9178 Inst.removeOperand(OpNo: 3);
9179
9180 Inst.setDesc(get(Opcode: NewOpc));
9181 Inst.addOperand(Op: MachineOperand::CreateImm(Val: 0)); // clamp bit
9182 Inst.addImplicitDefUseOperands(MF&: *MBB.getParent());
9183 MRI.replaceRegWith(FromReg: OldDstReg, ToReg: ResultReg);
9184 MachineBasicBlock *NewBB = legalizeOperands(MI&: Inst, MDT);
9185
9186 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9187 return std::pair(true, NewBB);
9188 }
9189
9190 return std::pair(false, nullptr);
9191}
9192
9193void SIInstrInfo::lowerSelect(SIInstrWorklist &Worklist, MachineInstr &Inst,
9194 MachineDominatorTree *MDT) const {
9195
9196 MachineBasicBlock &MBB = *Inst.getParent();
9197 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9198 MachineBasicBlock::iterator MII = Inst;
9199 const DebugLoc &DL = Inst.getDebugLoc();
9200
9201 MachineOperand &Dest = Inst.getOperand(i: 0);
9202 MachineOperand &Src0 = Inst.getOperand(i: 1);
9203 MachineOperand &Src1 = Inst.getOperand(i: 2);
9204 MachineOperand &Cond = Inst.getOperand(i: 3);
9205
9206 Register CondReg = Cond.getReg();
9207 bool IsSCC = (CondReg == AMDGPU::SCC);
9208
9209 // Remove S_CSELECT instructions that we previously inserted to feed the SCC
9210 // condition output from S_CMP into the SGPR condition input of V_CNDMASK. If
9211 // the S_CMP has been promoted to V_CMP then we can feed its SGPR condition
9212 // output directly into the V_CNDMASK.
9213 if (!IsSCC && Src0.isImm() && (Src0.getImm() == -1) && Src1.isImm() &&
9214 (Src1.getImm() == 0)) {
9215 for (MachineOperand &UseMO :
9216 make_early_inc_range(Range: MRI.use_nodbg_operands(Reg: Dest.getReg()))) {
9217 MachineInstr &UseMI = *UseMO.getParent();
9218 switch (UseMI.getOpcode()) {
9219 case AMDGPU::V_CNDMASK_B16_fake16_e32:
9220 case AMDGPU::V_CNDMASK_B16_fake16_e64:
9221 case AMDGPU::V_CNDMASK_B16_t16_e32:
9222 case AMDGPU::V_CNDMASK_B16_t16_e64:
9223 case AMDGPU::V_CNDMASK_B32_e32:
9224 case AMDGPU::V_CNDMASK_B32_e64:
9225 case AMDGPU::V_CNDMASK_B64_PSEUDO:
9226 if (UseMO.isImplicit() ||
9227 &UseMO == getNamedOperand(MI&: UseMI, OperandName: AMDGPU::OpName::src2))
9228 UseMO.setReg(CondReg);
9229 }
9230 }
9231 if (MRI.use_nodbg_empty(RegNo: Dest.getReg()))
9232 return;
9233 }
9234
9235 Register NewCondReg = CondReg;
9236 if (IsSCC) {
9237 const TargetRegisterClass *TC = RI.getWaveMaskRegClass();
9238 NewCondReg = MRI.createVirtualRegister(RegClass: TC);
9239
9240 // Now look for the closest SCC def if it is a copy
9241 // replacing the CondReg with the COPY source register
9242 bool CopyFound = false;
9243 for (MachineInstr &CandI :
9244 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(Inst)),
9245 y: Inst.getParent()->rend())) {
9246 if (CandI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) !=
9247 -1) {
9248 if (CandI.isCopy() && CandI.getOperand(i: 0).getReg() == AMDGPU::SCC) {
9249 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::COPY), DestReg: NewCondReg)
9250 .addReg(RegNo: CandI.getOperand(i: 1).getReg());
9251 CopyFound = true;
9252 }
9253 break;
9254 }
9255 }
9256 if (!CopyFound) {
9257 // SCC def is not a copy
9258 // Insert a trivial select instead of creating a copy, because a copy from
9259 // SCC would semantically mean just copying a single bit, but we may need
9260 // the result to be a vector condition mask that needs preserving.
9261 unsigned Opcode =
9262 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
9263 auto NewSelect =
9264 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewCondReg).addImm(Val: -1).addImm(Val: 0);
9265 NewSelect->getOperand(i: 3).setIsUndef(Cond.isUndef());
9266 }
9267 }
9268
9269 Register NewDestReg = MRI.createVirtualRegister(
9270 RegClass: RI.getEquivalentVGPRClass(SRC: MRI.getRegClass(Reg: Dest.getReg())));
9271 MachineInstr *NewInst;
9272 if (Inst.getOpcode() == AMDGPU::S_CSELECT_B32) {
9273 NewInst = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: NewDestReg)
9274 .addImm(Val: 0)
9275 .add(MO: Src1) // False
9276 .addImm(Val: 0)
9277 .add(MO: Src0) // True
9278 .addReg(RegNo: NewCondReg);
9279 } else {
9280 NewInst =
9281 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO), DestReg: NewDestReg)
9282 .add(MO: Src1) // False
9283 .add(MO: Src0) // True
9284 .addReg(RegNo: NewCondReg);
9285 }
9286 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDestReg);
9287 legalizeOperands(MI&: *NewInst, MDT);
9288 addUsersToMoveToVALUWorklist(Reg: NewDestReg, MRI, Worklist);
9289}
9290
9291void SIInstrInfo::lowerScalarAbs(SIInstrWorklist &Worklist,
9292 MachineInstr &Inst) const {
9293 MachineBasicBlock &MBB = *Inst.getParent();
9294 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9295 MachineBasicBlock::iterator MII = Inst;
9296 const DebugLoc &DL = Inst.getDebugLoc();
9297
9298 MachineOperand &Dest = Inst.getOperand(i: 0);
9299 MachineOperand &Src = Inst.getOperand(i: 1);
9300 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9301 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9302
9303 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9304 unsigned SubOp =
9305 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9306
9307 MachineInstrBuilder Sub =
9308 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: Src.getReg());
9309 if (HasCarryOut)
9310 Sub.setOperandDead(3); // Dead vcc
9311
9312 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9313 .addReg(RegNo: Src.getReg())
9314 .addReg(RegNo: TmpReg);
9315
9316 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9317 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9318}
9319
9320void SIInstrInfo::lowerScalarAbsDiff(SIInstrWorklist &Worklist,
9321 MachineInstr &Inst) const {
9322 MachineBasicBlock &MBB = *Inst.getParent();
9323 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9324 MachineBasicBlock::iterator MII = Inst;
9325 const DebugLoc &DL = Inst.getDebugLoc();
9326
9327 MachineOperand &Dest = Inst.getOperand(i: 0);
9328 MachineOperand &Src1 = Inst.getOperand(i: 1);
9329 MachineOperand &Src2 = Inst.getOperand(i: 2);
9330 Register SubResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9331 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9332 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9333
9334 bool HasCarryOut = !ST.hasAddNoCarryInsts();
9335 unsigned SubOp =
9336 HasCarryOut ? AMDGPU::V_SUB_CO_U32_e32 : AMDGPU::V_SUB_U32_e32;
9337
9338 MachineInstrBuilder Sub1 = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: SubResultReg)
9339 .addReg(RegNo: Src1.getReg())
9340 .addReg(RegNo: Src2.getReg());
9341
9342 MachineInstrBuilder Sub2 =
9343 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: SubOp), DestReg: TmpReg).addImm(Val: 0).addReg(RegNo: SubResultReg);
9344
9345 if (HasCarryOut) {
9346 Sub1.setOperandDead(3); // Dead vcc
9347 Sub2.setOperandDead(3); // Dead vcc
9348 }
9349
9350 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MAX_I32_e64), DestReg: ResultReg)
9351 .addReg(RegNo: SubResultReg)
9352 .addReg(RegNo: TmpReg);
9353
9354 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9355 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9356}
9357
9358void SIInstrInfo::lowerScalarXnor(SIInstrWorklist &Worklist,
9359 MachineInstr &Inst) const {
9360 MachineBasicBlock &MBB = *Inst.getParent();
9361 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9362 MachineBasicBlock::iterator MII = Inst;
9363 const DebugLoc &DL = Inst.getDebugLoc();
9364
9365 MachineOperand &Dest = Inst.getOperand(i: 0);
9366 MachineOperand &Src0 = Inst.getOperand(i: 1);
9367 MachineOperand &Src1 = Inst.getOperand(i: 2);
9368
9369 if (ST.hasDLInsts()) {
9370 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9371 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src0, MRI, DL);
9372 legalizeGenericOperand(InsertMBB&: MBB, I: MII, DstRC: &AMDGPU::VGPR_32RegClass, Op&: Src1, MRI, DL);
9373
9374 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_XNOR_B32_e64), DestReg: NewDest)
9375 .add(MO: Src0)
9376 .add(MO: Src1);
9377
9378 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9379 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9380 } else {
9381 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
9382 // invert either source and then perform the XOR. If either source is a
9383 // scalar register, then we can leave the inversion on the scalar unit to
9384 // achieve a better distribution of scalar and vector instructions.
9385 bool Src0IsSGPR = Src0.isReg() &&
9386 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src0.getReg()));
9387 bool Src1IsSGPR = Src1.isReg() &&
9388 RI.isSGPRClass(RC: MRI.getRegClass(Reg: Src1.getReg()));
9389 MachineInstr *Xor;
9390 Register Temp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9391 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9392
9393 // Build a pair of scalar instructions and add them to the work list.
9394 // The next iteration over the work list will lower these to the vector
9395 // unit as necessary.
9396 if (Src0IsSGPR) {
9397 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src0);
9398 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9399 .addReg(RegNo: Temp)
9400 .add(MO: Src1);
9401 } else if (Src1IsSGPR) {
9402 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Temp).add(MO: Src1);
9403 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: NewDest)
9404 .add(MO: Src0)
9405 .addReg(RegNo: Temp);
9406 } else {
9407 Xor = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B32), DestReg: Temp)
9408 .add(MO: Src0)
9409 .add(MO: Src1);
9410 MachineInstr *Not =
9411 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest).addReg(RegNo: Temp);
9412 Worklist.insert(MI: Not);
9413 }
9414
9415 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9416
9417 Worklist.insert(MI: Xor);
9418
9419 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9420 }
9421}
9422
9423void SIInstrInfo::splitScalarNotBinop(SIInstrWorklist &Worklist,
9424 MachineInstr &Inst,
9425 unsigned Opcode) const {
9426 MachineBasicBlock &MBB = *Inst.getParent();
9427 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9428 MachineBasicBlock::iterator MII = Inst;
9429 const DebugLoc &DL = Inst.getDebugLoc();
9430
9431 MachineOperand &Dest = Inst.getOperand(i: 0);
9432 MachineOperand &Src0 = Inst.getOperand(i: 1);
9433 MachineOperand &Src1 = Inst.getOperand(i: 2);
9434
9435 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9436 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
9437
9438 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: Interm)
9439 .add(MO: Src0)
9440 .add(MO: Src1);
9441
9442 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: NewDest)
9443 .addReg(RegNo: Interm);
9444
9445 Worklist.insert(MI: &Op);
9446 Worklist.insert(MI: &Not);
9447
9448 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9449 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9450}
9451
9452void SIInstrInfo::splitScalarBinOpN2(SIInstrWorklist &Worklist,
9453 MachineInstr &Inst,
9454 unsigned Opcode) const {
9455 MachineBasicBlock &MBB = *Inst.getParent();
9456 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9457 MachineBasicBlock::iterator MII = Inst;
9458 const DebugLoc &DL = Inst.getDebugLoc();
9459
9460 MachineOperand &Dest = Inst.getOperand(i: 0);
9461 MachineOperand &Src0 = Inst.getOperand(i: 1);
9462 MachineOperand &Src1 = Inst.getOperand(i: 2);
9463
9464 Register NewDest = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9465 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
9466
9467 MachineInstr &Not = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B32), DestReg: Interm)
9468 .add(MO: Src1);
9469
9470 MachineInstr &Op = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode), DestReg: NewDest)
9471 .add(MO: Src0)
9472 .addReg(RegNo: Interm);
9473
9474 Worklist.insert(MI: &Not);
9475 Worklist.insert(MI: &Op);
9476
9477 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9478 addUsersToMoveToVALUWorklist(Reg: NewDest, MRI, Worklist);
9479}
9480
9481void SIInstrInfo::splitScalar64BitUnaryOp(SIInstrWorklist &Worklist,
9482 MachineInstr &Inst, unsigned Opcode,
9483 bool Swap) const {
9484 MachineBasicBlock &MBB = *Inst.getParent();
9485 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9486
9487 MachineOperand &Dest = Inst.getOperand(i: 0);
9488 MachineOperand &Src0 = Inst.getOperand(i: 1);
9489 const DebugLoc &DL = Inst.getDebugLoc();
9490
9491 MachineBasicBlock::iterator MII = Inst;
9492
9493 const MCInstrDesc &InstDesc = get(Opcode);
9494 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9495 MRI.getRegClass(Reg: Src0.getReg()) :
9496 &AMDGPU::SGPR_32RegClass;
9497
9498 const TargetRegisterClass *Src0SubRC =
9499 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9500
9501 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9502 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9503
9504 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9505 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9506 const TargetRegisterClass *NewDestSubRC =
9507 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9508
9509 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9510 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0).add(MO: SrcReg0Sub0);
9511
9512 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9513 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9514
9515 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9516 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1).add(MO: SrcReg0Sub1);
9517
9518 if (Swap)
9519 std::swap(a&: DestSub0, b&: DestSub1);
9520
9521 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9522 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9523 .addReg(RegNo: DestSub0)
9524 .addImm(Val: AMDGPU::sub0)
9525 .addReg(RegNo: DestSub1)
9526 .addImm(Val: AMDGPU::sub1);
9527
9528 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9529
9530 Worklist.insert(MI: &LoHalf);
9531 Worklist.insert(MI: &HiHalf);
9532
9533 // We don't need to legalizeOperands here because for a single operand, src0
9534 // will support any kind of input.
9535
9536 // Move all users of this moved value.
9537 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9538}
9539
9540// There is not a vector equivalent of s_mul_u64. For this reason, we need to
9541// split the s_mul_u64 in 32-bit vector multiplications.
9542void SIInstrInfo::splitScalarSMulU64(SIInstrWorklist &Worklist,
9543 MachineInstr &Inst,
9544 MachineDominatorTree *MDT) const {
9545 MachineBasicBlock &MBB = *Inst.getParent();
9546 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9547
9548 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9549 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9550 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9551
9552 MachineOperand &Dest = Inst.getOperand(i: 0);
9553 MachineOperand &Src0 = Inst.getOperand(i: 1);
9554 MachineOperand &Src1 = Inst.getOperand(i: 2);
9555 const DebugLoc &DL = Inst.getDebugLoc();
9556 MachineBasicBlock::iterator MII = Inst;
9557
9558 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9559 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9560 const TargetRegisterClass *Src0SubRC =
9561 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9562 if (RI.isSGPRClass(RC: Src0SubRC))
9563 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9564 const TargetRegisterClass *Src1SubRC =
9565 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9566 if (RI.isSGPRClass(RC: Src1SubRC))
9567 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9568
9569 // First, we extract the low 32-bit and high 32-bit values from each of the
9570 // operands.
9571 MachineOperand Op0L =
9572 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9573 MachineOperand Op1L =
9574 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9575 MachineOperand Op0H =
9576 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9577 MachineOperand Op1H =
9578 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9579
9580 // The multilication is done as follows:
9581 //
9582 // Op1H Op1L
9583 // * Op0H Op0L
9584 // --------------------
9585 // Op1H*Op0L Op1L*Op0L
9586 // + Op1H*Op0H Op1L*Op0H
9587 // -----------------------------------------
9588 // (Op1H*Op0L + Op1L*Op0H + carry) Op1L*Op0L
9589 //
9590 // We drop Op1H*Op0H because the result of the multiplication is a 64-bit
9591 // value and that would overflow.
9592 // The low 32-bit value is Op1L*Op0L.
9593 // The high 32-bit value is Op1H*Op0L + Op1L*Op0H + carry (from Op1L*Op0L).
9594
9595 Register Op1L_Op0H_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9596 MachineInstr *Op1L_Op0H =
9597 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1L_Op0H_Reg)
9598 .add(MO: Op1L)
9599 .add(MO: Op0H);
9600
9601 Register Op1H_Op0L_Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9602 MachineInstr *Op1H_Op0L =
9603 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: Op1H_Op0L_Reg)
9604 .add(MO: Op1H)
9605 .add(MO: Op0L);
9606
9607 Register CarryReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9608 MachineInstr *Carry =
9609 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: CarryReg)
9610 .add(MO: Op1L)
9611 .add(MO: Op0L);
9612
9613 MachineInstr *LoHalf =
9614 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9615 .add(MO: Op1L)
9616 .add(MO: Op0L);
9617
9618 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9619 MachineInstr *Add = BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: AddReg)
9620 .addReg(RegNo: Op1L_Op0H_Reg)
9621 .addReg(RegNo: Op1H_Op0L_Reg);
9622
9623 MachineInstr *HiHalf =
9624 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg: DestSub1)
9625 .addReg(RegNo: AddReg)
9626 .addReg(RegNo: CarryReg);
9627
9628 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9629 .addReg(RegNo: DestSub0)
9630 .addImm(Val: AMDGPU::sub0)
9631 .addReg(RegNo: DestSub1)
9632 .addImm(Val: AMDGPU::sub1);
9633
9634 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9635
9636 // Try to legalize the operands in case we need to swap the order to keep it
9637 // valid.
9638 legalizeOperands(MI&: *Op1L_Op0H, MDT);
9639 legalizeOperands(MI&: *Op1H_Op0L, MDT);
9640 legalizeOperands(MI&: *Carry, MDT);
9641 legalizeOperands(MI&: *LoHalf, MDT);
9642 legalizeOperands(MI&: *Add, MDT);
9643 legalizeOperands(MI&: *HiHalf, MDT);
9644
9645 // Move all users of this moved value.
9646 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9647}
9648
9649// Lower S_MUL_U64_U32_PSEUDO/S_MUL_I64_I32_PSEUDO in two 32-bit vector
9650// multiplications.
9651void SIInstrInfo::splitScalarSMulPseudo(SIInstrWorklist &Worklist,
9652 MachineInstr &Inst,
9653 MachineDominatorTree *MDT) const {
9654 MachineBasicBlock &MBB = *Inst.getParent();
9655 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9656
9657 Register FullDestReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9658 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9659 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9660
9661 MachineOperand &Dest = Inst.getOperand(i: 0);
9662 MachineOperand &Src0 = Inst.getOperand(i: 1);
9663 MachineOperand &Src1 = Inst.getOperand(i: 2);
9664 const DebugLoc &DL = Inst.getDebugLoc();
9665 MachineBasicBlock::iterator MII = Inst;
9666
9667 const TargetRegisterClass *Src0RC = MRI.getRegClass(Reg: Src0.getReg());
9668 const TargetRegisterClass *Src1RC = MRI.getRegClass(Reg: Src1.getReg());
9669 const TargetRegisterClass *Src0SubRC =
9670 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9671 if (RI.isSGPRClass(RC: Src0SubRC))
9672 Src0SubRC = RI.getEquivalentVGPRClass(SRC: Src0SubRC);
9673 const TargetRegisterClass *Src1SubRC =
9674 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9675 if (RI.isSGPRClass(RC: Src1SubRC))
9676 Src1SubRC = RI.getEquivalentVGPRClass(SRC: Src1SubRC);
9677
9678 // First, we extract the low 32-bit and high 32-bit values from each of the
9679 // operands.
9680 MachineOperand Op0L =
9681 buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9682 MachineOperand Op1L =
9683 buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9684
9685 unsigned Opc = Inst.getOpcode();
9686 unsigned NewOpc = Opc == AMDGPU::S_MUL_U64_U32_PSEUDO
9687 ? AMDGPU::V_MUL_HI_U32_e64
9688 : AMDGPU::V_MUL_HI_I32_e64;
9689 MachineInstr *HiHalf =
9690 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: NewOpc), DestReg: DestSub1).add(MO: Op1L).add(MO: Op0L);
9691
9692 MachineInstr *LoHalf =
9693 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MUL_LO_U32_e64), DestReg: DestSub0)
9694 .add(MO: Op1L)
9695 .add(MO: Op0L);
9696
9697 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9698 .addReg(RegNo: DestSub0)
9699 .addImm(Val: AMDGPU::sub0)
9700 .addReg(RegNo: DestSub1)
9701 .addImm(Val: AMDGPU::sub1);
9702
9703 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9704
9705 // Try to legalize the operands in case we need to swap the order to keep it
9706 // valid.
9707 legalizeOperands(MI&: *HiHalf, MDT);
9708 legalizeOperands(MI&: *LoHalf, MDT);
9709
9710 // Move all users of this moved value.
9711 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9712}
9713
9714void SIInstrInfo::splitScalar64BitBinaryOp(SIInstrWorklist &Worklist,
9715 MachineInstr &Inst, unsigned Opcode,
9716 MachineDominatorTree *MDT) const {
9717 MachineBasicBlock &MBB = *Inst.getParent();
9718 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9719
9720 MachineOperand &Dest = Inst.getOperand(i: 0);
9721 MachineOperand &Src0 = Inst.getOperand(i: 1);
9722 MachineOperand &Src1 = Inst.getOperand(i: 2);
9723 const DebugLoc &DL = Inst.getDebugLoc();
9724
9725 MachineBasicBlock::iterator MII = Inst;
9726
9727 const MCInstrDesc &InstDesc = get(Opcode);
9728 const TargetRegisterClass *Src0RC = Src0.isReg() ?
9729 MRI.getRegClass(Reg: Src0.getReg()) :
9730 &AMDGPU::SGPR_32RegClass;
9731
9732 const TargetRegisterClass *Src0SubRC =
9733 RI.getSubRegisterClass(Src0RC, AMDGPU::sub0);
9734 const TargetRegisterClass *Src1RC = Src1.isReg() ?
9735 MRI.getRegClass(Reg: Src1.getReg()) :
9736 &AMDGPU::SGPR_32RegClass;
9737
9738 const TargetRegisterClass *Src1SubRC =
9739 RI.getSubRegisterClass(Src1RC, AMDGPU::sub0);
9740
9741 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9742 SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
9743 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9744 SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
9745 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src0, SuperRC: Src0RC,
9746 SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
9747 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src1, SuperRC: Src1RC,
9748 SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
9749
9750 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9751 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(SRC: DestRC);
9752 const TargetRegisterClass *NewDestSubRC =
9753 RI.getSubRegisterClass(NewDestRC, AMDGPU::sub0);
9754
9755 Register DestSub0 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9756 MachineInstr &LoHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub0)
9757 .add(MO: SrcReg0Sub0)
9758 .add(MO: SrcReg1Sub0);
9759
9760 Register DestSub1 = MRI.createVirtualRegister(RegClass: NewDestSubRC);
9761 MachineInstr &HiHalf = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: DestSub1)
9762 .add(MO: SrcReg0Sub1)
9763 .add(MO: SrcReg1Sub1);
9764
9765 Register FullDestReg = MRI.createVirtualRegister(RegClass: NewDestRC);
9766 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: FullDestReg)
9767 .addReg(RegNo: DestSub0)
9768 .addImm(Val: AMDGPU::sub0)
9769 .addReg(RegNo: DestSub1)
9770 .addImm(Val: AMDGPU::sub1);
9771
9772 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: FullDestReg);
9773
9774 Worklist.insert(MI: &LoHalf);
9775 Worklist.insert(MI: &HiHalf);
9776
9777 // Move all users of this moved value.
9778 addUsersToMoveToVALUWorklist(Reg: FullDestReg, MRI, Worklist);
9779}
9780
9781void SIInstrInfo::splitScalar64BitXnor(SIInstrWorklist &Worklist,
9782 MachineInstr &Inst,
9783 MachineDominatorTree *MDT) const {
9784 MachineBasicBlock &MBB = *Inst.getParent();
9785 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9786
9787 MachineOperand &Dest = Inst.getOperand(i: 0);
9788 MachineOperand &Src0 = Inst.getOperand(i: 1);
9789 MachineOperand &Src1 = Inst.getOperand(i: 2);
9790 const DebugLoc &DL = Inst.getDebugLoc();
9791
9792 MachineBasicBlock::iterator MII = Inst;
9793
9794 const TargetRegisterClass *DestRC = MRI.getRegClass(Reg: Dest.getReg());
9795
9796 Register Interm = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
9797
9798 MachineOperand* Op0;
9799 MachineOperand* Op1;
9800
9801 if (Src0.isReg() && RI.isSGPRReg(MRI, Reg: Src0.getReg())) {
9802 Op0 = &Src0;
9803 Op1 = &Src1;
9804 } else {
9805 Op0 = &Src1;
9806 Op1 = &Src0;
9807 }
9808
9809 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_NOT_B64), DestReg: Interm)
9810 .add(MO: *Op0);
9811
9812 Register NewDest = MRI.createVirtualRegister(RegClass: DestRC);
9813
9814 MachineInstr &Xor = *BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::S_XOR_B64), DestReg: NewDest)
9815 .addReg(RegNo: Interm)
9816 .add(MO: *Op1);
9817
9818 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: NewDest);
9819
9820 Worklist.insert(MI: &Xor);
9821}
9822
9823void SIInstrInfo::splitScalar64BitBCNT(SIInstrWorklist &Worklist,
9824 MachineInstr &Inst) const {
9825 MachineBasicBlock &MBB = *Inst.getParent();
9826 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9827
9828 MachineBasicBlock::iterator MII = Inst;
9829 const DebugLoc &DL = Inst.getDebugLoc();
9830
9831 MachineOperand &Dest = Inst.getOperand(i: 0);
9832 MachineOperand &Src = Inst.getOperand(i: 1);
9833
9834 const MCInstrDesc &InstDesc = get(Opcode: AMDGPU::V_BCNT_U32_B32_e64);
9835 const TargetRegisterClass *SrcRC = Src.isReg() ?
9836 MRI.getRegClass(Reg: Src.getReg()) :
9837 &AMDGPU::SGPR_32RegClass;
9838
9839 Register MidReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9840 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9841
9842 const TargetRegisterClass *SrcSubRC =
9843 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9844
9845 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9846 SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9847 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC,
9848 SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9849
9850 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg).add(MO: SrcRegSub0).addImm(Val: 0);
9851
9852 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: ResultReg).add(MO: SrcRegSub1).addReg(RegNo: MidReg);
9853
9854 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9855
9856 // We don't need to legalize operands here. src0 for either instruction can be
9857 // an SGPR, and the second input is unused or determined here.
9858 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9859}
9860
9861void SIInstrInfo::splitScalar64BitBFE(SIInstrWorklist &Worklist,
9862 MachineInstr &Inst) const {
9863 MachineBasicBlock &MBB = *Inst.getParent();
9864 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9865 MachineBasicBlock::iterator MII = Inst;
9866 const DebugLoc &DL = Inst.getDebugLoc();
9867
9868 MachineOperand &Dest = Inst.getOperand(i: 0);
9869 uint32_t Imm = Inst.getOperand(i: 2).getImm();
9870 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
9871 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
9872
9873 (void) Offset;
9874
9875 // Only sext_inreg cases handled.
9876 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
9877 Offset == 0 && "Not implemented");
9878
9879 if (BitWidth < 32) {
9880 Register MidRegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9881 Register MidRegHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9882 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9883
9884 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFE_I32_e64), DestReg: MidRegLo)
9885 .addReg(RegNo: Inst.getOperand(i: 1).getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9886 .addImm(Val: 0)
9887 .addImm(Val: BitWidth);
9888
9889 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e32), DestReg: MidRegHi)
9890 .addImm(Val: 31)
9891 .addReg(RegNo: MidRegLo);
9892
9893 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9894 .addReg(RegNo: MidRegLo)
9895 .addImm(Val: AMDGPU::sub0)
9896 .addReg(RegNo: MidRegHi)
9897 .addImm(Val: AMDGPU::sub1);
9898
9899 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9900 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9901 return;
9902 }
9903
9904 MachineOperand &Src = Inst.getOperand(i: 1);
9905 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9906 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_64RegClass);
9907
9908 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ASHRREV_I32_e64), DestReg: TmpReg)
9909 .addImm(Val: 31)
9910 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0);
9911
9912 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: ResultReg)
9913 .addReg(RegNo: Src.getReg(), Flags: {}, SubReg: AMDGPU::sub0)
9914 .addImm(Val: AMDGPU::sub0)
9915 .addReg(RegNo: TmpReg)
9916 .addImm(Val: AMDGPU::sub1);
9917
9918 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
9919 addUsersToMoveToVALUWorklist(Reg: ResultReg, MRI, Worklist);
9920}
9921
9922void SIInstrInfo::splitScalar64BitCountOp(SIInstrWorklist &Worklist,
9923 MachineInstr &Inst, unsigned Opcode,
9924 MachineDominatorTree *MDT) const {
9925 // (S_FLBIT_I32_B64 hi:lo) ->
9926 // -> (umin (V_FFBH_U32_e32 hi), (or (V_FFBH_U32_e32 lo), 32))
9927 // (S_FF1_I32_B64 hi:lo) ->
9928 // ->(umin (or (V_FFBL_B32_e32 hi), 32) (V_FFBL_B32_e32 lo))
9929
9930 MachineBasicBlock &MBB = *Inst.getParent();
9931 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9932 MachineBasicBlock::iterator MII = Inst;
9933 const DebugLoc &DL = Inst.getDebugLoc();
9934
9935 MachineOperand &Dest = Inst.getOperand(i: 0);
9936 MachineOperand &Src = Inst.getOperand(i: 1);
9937
9938 const MCInstrDesc &InstDesc = get(Opcode);
9939
9940 bool IsCtlz = Opcode == AMDGPU::V_FFBH_U32_e32;
9941
9942 const TargetRegisterClass *SrcRC =
9943 Src.isReg() ? MRI.getRegClass(Reg: Src.getReg()) : &AMDGPU::SGPR_32RegClass;
9944 const TargetRegisterClass *SrcSubRC =
9945 RI.getSubRegisterClass(SrcRC, AMDGPU::sub0);
9946
9947 MachineOperand SrcRegSub0 =
9948 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
9949 MachineOperand SrcRegSub1 =
9950 buildExtractSubRegOrImm(MII, MRI, Op: Src, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
9951
9952 Register MidReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9953 Register MidReg2 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9954 Register MidReg3 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9955 Register MidReg4 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
9956
9957 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg1).add(MO: SrcRegSub0);
9958
9959 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: InstDesc, DestReg: MidReg2).add(MO: SrcRegSub1);
9960
9961 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_OR_B32_e32), DestReg: MidReg3)
9962 .addImm(Val: 32)
9963 .addReg(RegNo: IsCtlz ? MidReg1 : MidReg2);
9964
9965 BuildMI(BB&: MBB, I: MII, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MIN_U32_e64), DestReg: MidReg4)
9966 .addReg(RegNo: MidReg3)
9967 .addReg(RegNo: IsCtlz ? MidReg2 : MidReg1);
9968
9969 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: MidReg4);
9970
9971 addUsersToMoveToVALUWorklist(Reg: MidReg4, MRI, Worklist);
9972}
9973
9974void SIInstrInfo::addUsersToMoveToVALUWorklist(
9975 Register DstReg, MachineRegisterInfo &MRI,
9976 SIInstrWorklist &Worklist) const {
9977 for (MachineOperand &MO : make_early_inc_range(Range: MRI.use_operands(Reg: DstReg))) {
9978 MachineInstr &UseMI = *MO.getParent();
9979
9980 unsigned OpNo = 0;
9981
9982 switch (UseMI.getOpcode()) {
9983 case AMDGPU::COPY:
9984 case AMDGPU::WQM:
9985 case AMDGPU::SOFT_WQM:
9986 case AMDGPU::STRICT_WWM:
9987 case AMDGPU::STRICT_WQM:
9988 case AMDGPU::REG_SEQUENCE:
9989 case AMDGPU::PHI:
9990 case AMDGPU::INSERT_SUBREG:
9991 break;
9992 default:
9993 OpNo = MO.getOperandNo();
9994 break;
9995 }
9996
9997 const TargetRegisterClass *OpRC = getOpRegClass(MI: UseMI, OpNo);
9998 MRI.constrainRegClass(Reg: DstReg, RC: OpRC);
9999
10000 if (!RI.hasVectorRegisters(RC: OpRC))
10001 Worklist.insert(MI: &UseMI);
10002 else
10003 // Legalization could change user list.
10004 legalizeOperandsVALUt16(MI&: UseMI, OpIdx: OpNo, MRI);
10005 }
10006}
10007
10008void SIInstrInfo::movePackToVALU(SIInstrWorklist &Worklist,
10009 MachineRegisterInfo &MRI,
10010 MachineInstr &Inst) const {
10011 Register ResultReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10012 MachineBasicBlock *MBB = Inst.getParent();
10013 MachineOperand &Src0 = Inst.getOperand(i: 1);
10014 MachineOperand &Src1 = Inst.getOperand(i: 2);
10015 const DebugLoc &DL = Inst.getDebugLoc();
10016
10017 if (ST.useRealTrue16Insts()) {
10018 Register SrcReg0, SrcReg1;
10019 if (!Src0.isReg() || !RI.isVGPR(MRI, Reg: Src0.getReg())) {
10020 SrcReg0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10021 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
10022 MCID: get(Opcode: Src0.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg0)
10023 .add(MO: Src0);
10024 } else {
10025 SrcReg0 = Src0.getReg();
10026 }
10027
10028 if (!Src1.isReg() || !RI.isVGPR(MRI, Reg: Src1.getReg())) {
10029 SrcReg1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10030 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL,
10031 MCID: get(Opcode: Src1.isImm() ? AMDGPU::V_MOV_B32_e32 : AMDGPU::COPY), DestReg: SrcReg1)
10032 .add(MO: Src1);
10033 } else {
10034 SrcReg1 = Src1.getReg();
10035 }
10036
10037 bool isSrc0Reg16 = MRI.constrainRegClass(Reg: SrcReg0, RC: &AMDGPU::VGPR_16RegClass);
10038 bool isSrc1Reg16 = MRI.constrainRegClass(Reg: SrcReg1, RC: &AMDGPU::VGPR_16RegClass);
10039
10040 auto NewMI = BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ResultReg);
10041 switch (Inst.getOpcode()) {
10042 case AMDGPU::S_PACK_LL_B32_B16:
10043 NewMI
10044 .addReg(RegNo: SrcReg0, Flags: {},
10045 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10046 .addImm(Val: AMDGPU::lo16)
10047 .addReg(RegNo: SrcReg1, Flags: {},
10048 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10049 .addImm(Val: AMDGPU::hi16);
10050 break;
10051 case AMDGPU::S_PACK_LH_B32_B16:
10052 NewMI
10053 .addReg(RegNo: SrcReg0, Flags: {},
10054 SubReg: isSrc0Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10055 .addImm(Val: AMDGPU::lo16)
10056 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
10057 .addImm(Val: AMDGPU::hi16);
10058 break;
10059 case AMDGPU::S_PACK_HL_B32_B16:
10060 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
10061 .addImm(Val: AMDGPU::lo16)
10062 .addReg(RegNo: SrcReg1, Flags: {},
10063 SubReg: isSrc1Reg16 ? AMDGPU::NoSubRegister : AMDGPU::lo16)
10064 .addImm(Val: AMDGPU::hi16);
10065 break;
10066 case AMDGPU::S_PACK_HH_B32_B16:
10067 NewMI.addReg(RegNo: SrcReg0, Flags: {}, SubReg: AMDGPU::hi16)
10068 .addImm(Val: AMDGPU::lo16)
10069 .addReg(RegNo: SrcReg1, Flags: {}, SubReg: AMDGPU::hi16)
10070 .addImm(Val: AMDGPU::hi16);
10071 break;
10072 default:
10073 llvm_unreachable("unhandled s_pack_* instruction");
10074 }
10075
10076 MachineOperand &Dest = Inst.getOperand(i: 0);
10077 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10078 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
10079 return;
10080 }
10081
10082 switch (Inst.getOpcode()) {
10083 case AMDGPU::S_PACK_LL_B32_B16: {
10084 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10085 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10086
10087 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
10088 // 0.
10089 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
10090 .addImm(Val: 0xffff);
10091
10092 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: TmpReg)
10093 .addReg(RegNo: ImmReg)
10094 .add(MO: Src0);
10095
10096 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
10097 .add(MO: Src1)
10098 .addImm(Val: 16)
10099 .addReg(RegNo: TmpReg);
10100 break;
10101 }
10102 case AMDGPU::S_PACK_LH_B32_B16: {
10103 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10104 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
10105 .addImm(Val: 0xffff);
10106 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_BFI_B32_e64), DestReg: ResultReg)
10107 .addReg(RegNo: ImmReg)
10108 .add(MO: Src0)
10109 .add(MO: Src1);
10110 break;
10111 }
10112 case AMDGPU::S_PACK_HL_B32_B16: {
10113 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10114 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
10115 .addImm(Val: 16)
10116 .add(MO: Src0);
10117 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: ResultReg)
10118 .add(MO: Src1)
10119 .addImm(Val: 16)
10120 .addReg(RegNo: TmpReg);
10121 break;
10122 }
10123 case AMDGPU::S_PACK_HH_B32_B16: {
10124 Register ImmReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10125 Register TmpReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
10126 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: TmpReg)
10127 .addImm(Val: 16)
10128 .add(MO: Src0);
10129 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: ImmReg)
10130 .addImm(Val: 0xffff0000);
10131 BuildMI(BB&: *MBB, I&: Inst, MIMD: DL, MCID: get(Opcode: AMDGPU::V_AND_OR_B32_e64), DestReg: ResultReg)
10132 .add(MO: Src1)
10133 .addReg(RegNo: ImmReg)
10134 .addReg(RegNo: TmpReg);
10135 break;
10136 }
10137 default:
10138 llvm_unreachable("unhandled s_pack_* instruction");
10139 }
10140
10141 MachineOperand &Dest = Inst.getOperand(i: 0);
10142 MRI.replaceRegWith(FromReg: Dest.getReg(), ToReg: ResultReg);
10143 addUsersToMoveToVALUWorklist(DstReg: ResultReg, MRI, Worklist);
10144}
10145
10146void SIInstrInfo::addSCCDefUsersToVALUWorklist(const MachineOperand &Op,
10147 MachineInstr &SCCDefInst,
10148 SIInstrWorklist &Worklist,
10149 Register NewCond) const {
10150
10151 // Ensure that def inst defines SCC, which is still live.
10152 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() &&
10153 !Op.isDead() && Op.getParent() == &SCCDefInst);
10154 SmallVector<MachineInstr *, 4> CopyToDelete;
10155 // This assumes that all the users of SCC are in the same block
10156 // as the SCC def.
10157 for (MachineInstr &MI : // Skip the def inst itself.
10158 make_range(x: std::next(x: MachineBasicBlock::iterator(SCCDefInst)),
10159 y: SCCDefInst.getParent()->end())) {
10160 // Check if SCC is used first.
10161 int SCCIdx = MI.findRegisterUseOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isKill: false);
10162 if (SCCIdx != -1) {
10163 if (MI.isCopy()) {
10164 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10165 Register DestReg = MI.getOperand(i: 0).getReg();
10166
10167 MRI.replaceRegWith(FromReg: DestReg, ToReg: NewCond);
10168 CopyToDelete.push_back(Elt: &MI);
10169 } else {
10170
10171 if (NewCond.isValid())
10172 MI.getOperand(i: SCCIdx).setReg(NewCond);
10173
10174 Worklist.insert(MI: &MI);
10175 }
10176 }
10177 // Exit if we find another SCC def.
10178 if (MI.findRegisterDefOperandIdx(Reg: AMDGPU::SCC, TRI: &RI, isDead: false, Overlap: false) != -1)
10179 break;
10180 }
10181 for (auto &Copy : CopyToDelete)
10182 Copy->eraseFromParent();
10183}
10184
10185// Instructions that use SCC may be converted to VALU instructions. When that
10186// happens, the SCC register is changed to VCC_LO. The instruction that defines
10187// SCC must be changed to an instruction that defines VCC. This function makes
10188// sure that the instruction that defines SCC is added to the moveToVALU
10189// worklist.
10190void SIInstrInfo::addSCCDefsToVALUWorklist(MachineInstr *SCCUseInst,
10191 SIInstrWorklist &Worklist) const {
10192 // Look for a preceding instruction that either defines VCC or SCC. If VCC
10193 // then there is nothing to do because the defining instruction has been
10194 // converted to a VALU already. If SCC then that instruction needs to be
10195 // converted to a VALU.
10196 for (MachineInstr &MI :
10197 make_range(x: std::next(x: MachineBasicBlock::reverse_iterator(SCCUseInst)),
10198 y: SCCUseInst->getParent()->rend())) {
10199 if (MI.modifiesRegister(Reg: AMDGPU::VCC, TRI: &RI))
10200 break;
10201 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
10202 Worklist.insert(MI: &MI);
10203 break;
10204 }
10205 }
10206}
10207
10208const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
10209 const MachineInstr &Inst) const {
10210 const TargetRegisterClass *NewDstRC = getOpRegClass(MI: Inst, OpNo: 0);
10211
10212 switch (Inst.getOpcode()) {
10213 // For target instructions, getOpRegClass just returns the virtual register
10214 // class associated with the operand, so we need to find an equivalent VGPR
10215 // register class in order to move the instruction to the VALU.
10216 case AMDGPU::COPY:
10217 case AMDGPU::PHI:
10218 case AMDGPU::REG_SEQUENCE:
10219 case AMDGPU::INSERT_SUBREG:
10220 case AMDGPU::WQM:
10221 case AMDGPU::SOFT_WQM:
10222 case AMDGPU::STRICT_WWM:
10223 case AMDGPU::STRICT_WQM: {
10224 const TargetRegisterClass *SrcRC = getOpRegClass(MI: Inst, OpNo: 1);
10225 if (RI.isAGPRClass(RC: SrcRC)) {
10226 if (RI.isAGPRClass(RC: NewDstRC))
10227 return nullptr;
10228
10229 switch (Inst.getOpcode()) {
10230 case AMDGPU::PHI:
10231 case AMDGPU::REG_SEQUENCE:
10232 case AMDGPU::INSERT_SUBREG:
10233 NewDstRC = RI.getEquivalentAGPRClass(SRC: NewDstRC);
10234 break;
10235 default:
10236 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
10237 }
10238
10239 if (!NewDstRC)
10240 return nullptr;
10241 } else {
10242 if (!RI.isSGPRClass(RC: NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass)
10243 return nullptr;
10244
10245 NewDstRC = RI.getEquivalentVGPRClass(SRC: NewDstRC);
10246 if (!NewDstRC)
10247 return nullptr;
10248 }
10249
10250 return NewDstRC;
10251 }
10252 default:
10253 return NewDstRC;
10254 }
10255}
10256
10257// Find the one SGPR operand we are allowed to use.
10258Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
10259 int OpIndices[3]) const {
10260 const MCInstrDesc &Desc = MI.getDesc();
10261
10262 // Find the one SGPR operand we are allowed to use.
10263 //
10264 // First we need to consider the instruction's operand requirements before
10265 // legalizing. Some operands are required to be SGPRs, such as implicit uses
10266 // of VCC, but we are still bound by the constant bus requirement to only use
10267 // one.
10268 //
10269 // If the operand's class is an SGPR, we can never move it.
10270
10271 Register SGPRReg = findImplicitSGPRRead(MI);
10272 if (SGPRReg)
10273 return SGPRReg;
10274
10275 Register UsedSGPRs[3] = {Register()};
10276 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
10277
10278 for (unsigned i = 0; i < 3; ++i) {
10279 int Idx = OpIndices[i];
10280 if (Idx == -1)
10281 break;
10282
10283 const MachineOperand &MO = MI.getOperand(i: Idx);
10284 if (!MO.isReg())
10285 continue;
10286
10287 // Is this operand statically required to be an SGPR based on the operand
10288 // constraints?
10289 const TargetRegisterClass *OpRC =
10290 RI.getRegClass(i: getOpRegClassID(OpInfo: Desc.operands()[Idx]));
10291 bool IsRequiredSGPR = RI.isSGPRClass(RC: OpRC);
10292 if (IsRequiredSGPR)
10293 return MO.getReg();
10294
10295 // If this could be a VGPR or an SGPR, Check the dynamic register class.
10296 Register Reg = MO.getReg();
10297 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
10298 if (RI.isSGPRClass(RC: RegRC))
10299 UsedSGPRs[i] = Reg;
10300 }
10301
10302 // We don't have a required SGPR operand, so we have a bit more freedom in
10303 // selecting operands to move.
10304
10305 // Try to select the most used SGPR. If an SGPR is equal to one of the
10306 // others, we choose that.
10307 //
10308 // e.g.
10309 // V_FMA_F32 v0, s0, s0, s0 -> No moves
10310 // V_FMA_F32 v0, s0, s1, s0 -> Move s1
10311
10312 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
10313 // prefer those.
10314
10315 if (UsedSGPRs[0]) {
10316 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
10317 SGPRReg = UsedSGPRs[0];
10318 }
10319
10320 if (!SGPRReg && UsedSGPRs[1]) {
10321 if (UsedSGPRs[1] == UsedSGPRs[2])
10322 SGPRReg = UsedSGPRs[1];
10323 }
10324
10325 return SGPRReg;
10326}
10327
10328MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI,
10329 AMDGPU::OpName OperandName) const {
10330 if (OperandName == AMDGPU::OpName::NUM_OPERAND_NAMES)
10331 return nullptr;
10332
10333 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OperandName);
10334 if (Idx == -1)
10335 return nullptr;
10336
10337 return &MI.getOperand(i: Idx);
10338}
10339
10340uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const {
10341 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
10342 int64_t Format = ST.getGeneration() >= AMDGPUSubtarget::GFX11
10343 ? (int64_t)AMDGPU::UfmtGFX11::UFMT_32_FLOAT
10344 : (int64_t)AMDGPU::UfmtGFX10::UFMT_32_FLOAT;
10345 return (Format << 44) |
10346 (1ULL << 56) | // RESOURCE_LEVEL = 1
10347 (3ULL << 60); // OOB_SELECT = 3
10348 }
10349
10350 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
10351 if (ST.isAmdHsaOS()) {
10352 // Set ATC = 1. GFX9 doesn't have this bit.
10353 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10354 RsrcDataFormat |= (1ULL << 56);
10355
10356 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
10357 // BTW, it disables TC L2 and therefore decreases performance.
10358 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
10359 RsrcDataFormat |= (2ULL << 59);
10360 }
10361
10362 return RsrcDataFormat;
10363}
10364
10365uint64_t SIInstrInfo::getScratchRsrcWords23() const {
10366 uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
10367 AMDGPU::RSRC_TID_ENABLE |
10368 0xffffffff; // Size;
10369
10370 // GFX9 doesn't have ELEMENT_SIZE.
10371 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
10372 uint64_t EltSizeValue = Log2_32(Value: ST.getMaxPrivateElementSize(ForBufferRSrc: true)) - 1;
10373 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
10374 }
10375
10376 // IndexStride = 64 / 32.
10377 uint64_t IndexStride = ST.isWave64() ? 3 : 2;
10378 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
10379
10380 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
10381 // Clear them unless we want a huge stride.
10382 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
10383 ST.getGeneration() <= AMDGPUSubtarget::GFX9)
10384 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
10385
10386 return Rsrc23;
10387}
10388
10389bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const {
10390 unsigned Opc = MI.getOpcode();
10391
10392 return isSMRD(Opcode: Opc);
10393}
10394
10395bool SIInstrInfo::isHighLatencyDef(int Opc) const {
10396 return get(Opcode: Opc).mayLoad() &&
10397 (isMUBUF(Opcode: Opc) || isMTBUF(Opcode: Opc) || isMIMG(Opcode: Opc) || isFLAT(Opcode: Opc));
10398}
10399
10400Register SIInstrInfo::isStackAccess(const MachineInstr &MI, int &FrameIndex,
10401 TypeSize &MemBytes) const {
10402 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr);
10403 if (!Addr || !Addr->isFI())
10404 return Register();
10405
10406 assert(!MI.memoperands_empty() &&
10407 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS);
10408
10409 FrameIndex = Addr->getIndex();
10410
10411 int VDataIdx =
10412 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
10413 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: VDataIdx));
10414 return MI.getOperand(i: VDataIdx).getReg();
10415}
10416
10417Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex,
10418 TypeSize &MemBytes) const {
10419 const MachineOperand *Addr = getNamedOperand(MI, OperandName: AMDGPU::OpName::addr);
10420 assert(Addr && Addr->isFI());
10421 FrameIndex = Addr->getIndex();
10422
10423 int DataIdx =
10424 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::data);
10425 MemBytes = TypeSize::getFixed(ExactSize: getOpSize(Opcode: MI.getOpcode(), OpNo: DataIdx));
10426 return MI.getOperand(i: DataIdx).getReg();
10427}
10428
10429Register SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
10430 int &FrameIndex,
10431 TypeSize &MemBytes) const {
10432 if (!MI.mayLoad())
10433 return Register();
10434
10435 if (isMUBUF(MI) || isVGPRSpill(MI))
10436 return isStackAccess(MI, FrameIndex, MemBytes);
10437
10438 if (isSGPRSpill(MI))
10439 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10440
10441 return Register();
10442}
10443
10444Register SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
10445 int &FrameIndex,
10446 TypeSize &MemBytes) const {
10447 if (!MI.mayStore())
10448 return Register();
10449
10450 if (isMUBUF(MI) || isVGPRSpill(MI))
10451 return isStackAccess(MI, FrameIndex, MemBytes);
10452
10453 if (isSGPRSpill(MI))
10454 return isSGPRStackAccess(MI, FrameIndex, MemBytes);
10455
10456 return Register();
10457}
10458
10459unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
10460 unsigned Opc = MI.getOpcode();
10461 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: Opc);
10462 unsigned DescSize = Desc.getSize();
10463
10464 // If we have a definitive size, we can use it. Otherwise we need to inspect
10465 // the operands to know the size.
10466 if (isFixedSize(MI)) {
10467 unsigned Size = DescSize;
10468
10469 // If we hit the buggy offset, an extra nop will be inserted in MC so
10470 // estimate the worst case.
10471 if (MI.isBranch() && ST.hasOffset3fBug())
10472 Size += 4;
10473
10474 return Size;
10475 }
10476
10477 // Instructions may have a 32-bit literal encoded after them. Check
10478 // operands that could ever be literals.
10479 if (isVALU(MI, /*AllowLDSDMA=*/false) || isSALU(MI)) {
10480 if (isDPP(MI))
10481 return DescSize;
10482 bool HasLiteral = false;
10483 unsigned LiteralSize = 4;
10484 for (int I = 0, E = MI.getNumExplicitOperands(); I != E; ++I) {
10485 const MachineOperand &Op = MI.getOperand(i: I);
10486 const MCOperandInfo &OpInfo = Desc.operands()[I];
10487 if (!Op.isReg() && !isInlineConstant(MO: Op, OpInfo)) {
10488 HasLiteral = true;
10489 if (ST.has64BitLiterals()) {
10490 switch (OpInfo.OperandType) {
10491 default:
10492 break;
10493 case AMDGPU::OPERAND_REG_IMM_FP64:
10494 case AMDGPU::OPERAND_REG_IMM_V2FP64:
10495 if (!AMDGPU::isValid32BitLiteral(Val: Op.getImm(), IsFP64: true))
10496 LiteralSize = 8;
10497 break;
10498 case AMDGPU::OPERAND_REG_IMM_INT64:
10499 case AMDGPU::OPERAND_REG_IMM_V2INT64:
10500 // A 32-bit literal is only valid when the value fits in BOTH signed
10501 // and unsigned 32-bit ranges [0, 2^31-1], matching the MC code
10502 // emitter's getLit64Encoding logic. This is because of the lack of
10503 // abilility to tell signedness of the literal, therefore we need to
10504 // be conservative and assume values outside this range require a
10505 // 64-bit literal encoding (8 bytes).
10506 if (!Op.isImm() || !isInt<32>(x: Op.getImm()) ||
10507 !isUInt<32>(x: Op.getImm()))
10508 LiteralSize = 8;
10509 break;
10510 }
10511 }
10512 break;
10513 }
10514 }
10515 return HasLiteral ? DescSize + LiteralSize : DescSize;
10516 }
10517
10518 // Check whether we have extra NSA words.
10519 if (isMIMG(MI)) {
10520 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
10521 if (VAddr0Idx < 0)
10522 return 8;
10523
10524 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::srsrc);
10525 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4);
10526 }
10527
10528 switch (Opc) {
10529 case TargetOpcode::BUNDLE:
10530 return getInstBundleSize(MI);
10531 case TargetOpcode::INLINEASM:
10532 case TargetOpcode::INLINEASM_BR: {
10533 const MachineFunction *MF = MI.getMF();
10534 const char *AsmStr = MI.getOperand(i: 0).getSymbolName();
10535 return getInlineAsmLength(Str: AsmStr, MAI: MF->getTarget().getMCAsmInfo(), STI: &ST);
10536 }
10537 default:
10538 if (MI.isMetaInstruction())
10539 return 0;
10540
10541 // If D16 Pseudo inst, get correct MC code size
10542 const auto *D16Info = AMDGPU::getT16D16Helper(T16Op: Opc);
10543 if (D16Info) {
10544 // Assume d16_lo/hi inst are always in same size
10545 unsigned LoInstOpcode = D16Info->LoOp;
10546 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: LoInstOpcode);
10547 DescSize = Desc.getSize();
10548 }
10549
10550 // If FMA Pseudo inst, get correct MC code size
10551 if (Opc == AMDGPU::V_FMA_MIX_F16_t16 || Opc == AMDGPU::V_FMA_MIX_BF16_t16) {
10552 // All potential lowerings are the same size; arbitrarily pick one.
10553 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opcode: AMDGPU::V_FMA_MIXLO_F16);
10554 DescSize = Desc.getSize();
10555 }
10556
10557 return DescSize;
10558 }
10559}
10560
10561TargetInstrInfo::InstSizeVerifyMode
10562SIInstrInfo::getInstSizeVerifyMode(const MachineInstr &MI) const {
10563 if (MI.isBranch() && ST.hasOffset3fBug())
10564 return InstSizeVerifyMode::NoVerify;
10565 return InstSizeVerifyMode::ExactSize;
10566}
10567
10568ArrayRef<std::pair<int, const char *>>
10569SIInstrInfo::getSerializableTargetIndices() const {
10570 static const std::pair<int, const char *> TargetIndices[] = {
10571 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"},
10572 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"},
10573 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"},
10574 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"},
10575 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}};
10576 return ArrayRef(TargetIndices);
10577}
10578
10579/// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The
10580/// post-RA version of misched uses CreateTargetMIHazardRecognizer.
10581ScheduleHazardRecognizer *
10582SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
10583 const ScheduleDAG *DAG) const {
10584 return new GCNHazardRecognizer(DAG->MF);
10585}
10586
10587/// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer
10588/// pass.
10589ScheduleHazardRecognizer *
10590SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF,
10591 MachineLoopInfo *MLI) const {
10592 return new GCNHazardRecognizer(
10593 MF, GCNHazardRecognizer::OperatingMode::HazardRecognizerMode, MLI);
10594}
10595
10596// Called during:
10597// - pre-RA scheduling and post-RA scheduling
10598ScheduleHazardRecognizer *
10599SIInstrInfo::CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
10600 const ScheduleDAGMI *DAG) const {
10601 // Borrowed from Arm Target
10602 // We would like to restrict this hazard recognizer to only
10603 // post-RA scheduling; we can tell that we're post-RA because we don't
10604 // track VRegLiveness.
10605 if (!DAG->hasVRegLiveness())
10606 return new GCNHazardRecognizer(DAG->MF);
10607 return TargetInstrInfo::CreateTargetMIHazardRecognizer(II, DAG);
10608}
10609
10610std::pair<unsigned, unsigned>
10611SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10612 return std::pair(TF & MO_MASK, TF & ~MO_MASK);
10613}
10614
10615ArrayRef<std::pair<unsigned, const char *>>
10616SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10617 static const std::pair<unsigned, const char *> TargetFlags[] = {
10618 {MO_GOTPCREL, "amdgpu-gotprel"},
10619 {MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo"},
10620 {MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi"},
10621 {MO_GOTPCREL64, "amdgpu-gotprel64"},
10622 {MO_REL32_LO, "amdgpu-rel32-lo"},
10623 {MO_REL32_HI, "amdgpu-rel32-hi"},
10624 {MO_REL64, "amdgpu-rel64"},
10625 {MO_ABS32_LO, "amdgpu-abs32-lo"},
10626 {MO_ABS32_HI, "amdgpu-abs32-hi"},
10627 {MO_ABS64, "amdgpu-abs64"},
10628 };
10629
10630 return ArrayRef(TargetFlags);
10631}
10632
10633ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10634SIInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10635 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10636 {
10637 {MONoClobber, "amdgpu-noclobber"},
10638 {MOLastUse, "amdgpu-last-use"},
10639 {MOCooperative, "amdgpu-cooperative"},
10640 {MOThreadPrivate, "amdgpu-thread-private"},
10641 };
10642
10643 return ArrayRef(TargetFlags);
10644}
10645
10646unsigned SIInstrInfo::getLiveRangeSplitOpcode(Register SrcReg,
10647 const MachineFunction &MF) const {
10648 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
10649 assert(SrcReg.isVirtual());
10650 if (MFI->checkFlag(Reg: SrcReg, Flag: AMDGPU::VirtRegFlag::WWM_REG))
10651 return AMDGPU::WWM_COPY;
10652
10653 return AMDGPU::COPY;
10654}
10655
10656bool SIInstrInfo::canAddToBBProlog(const MachineInstr &MI) const {
10657 uint32_t Opcode = MI.getOpcode();
10658 // Check if it is SGPR spill or wwm-register spill Opcode.
10659 if (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode))
10660 return true;
10661
10662 const MachineFunction *MF = MI.getMF();
10663 const MachineRegisterInfo &MRI = MF->getRegInfo();
10664 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
10665
10666 // See if this is Liverange split instruction inserted for SGPR or
10667 // wwm-register. The implicit def inserted for wwm-registers should also be
10668 // included as they can appear at the bb begin.
10669 bool IsLRSplitInst = MI.getFlag(Flag: MachineInstr::LRSplit);
10670 if (!IsLRSplitInst && Opcode != AMDGPU::IMPLICIT_DEF)
10671 return false;
10672
10673 Register Reg = MI.getOperand(i: 0).getReg();
10674 if (RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg)))
10675 return IsLRSplitInst;
10676
10677 return MFI->isWWMReg(Reg);
10678}
10679
10680bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
10681 Register Reg) const {
10682 // We need to handle instructions which may be inserted during register
10683 // allocation to handle the prolog. The initial prolog instruction may have
10684 // been separated from the start of the block by spills and copies inserted
10685 // needed by the prolog. However, the insertions for scalar registers can
10686 // always be placed at the BB top as they are independent of the exec mask
10687 // value.
10688 bool IsNullOrVectorRegister = true;
10689 if (Reg) {
10690 const MachineFunction *MF = MI.getMF();
10691 const MachineRegisterInfo &MRI = MF->getRegInfo();
10692 IsNullOrVectorRegister = !RI.isSGPRClass(RC: RI.getRegClassForReg(MRI, Reg));
10693 }
10694
10695 return IsNullOrVectorRegister &&
10696 (canAddToBBProlog(MI) ||
10697 (!MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
10698 MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &RI)));
10699}
10700
10701MachineInstrBuilder
10702SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10703 MachineBasicBlock::iterator I,
10704 const DebugLoc &DL,
10705 Register DestReg) const {
10706 if (ST.hasAddNoCarryInsts())
10707 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e64), DestReg);
10708
10709 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
10710 Register UnusedCarry = MRI.createVirtualRegister(RegClass: RI.getBoolRC());
10711 MRI.setRegAllocationHint(VReg: UnusedCarry, Type: 0, PrefReg: RI.getVCC());
10712
10713 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10714 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10715}
10716
10717MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
10718 MachineBasicBlock::iterator I,
10719 const DebugLoc &DL,
10720 Register DestReg,
10721 RegScavenger &RS) const {
10722 if (ST.hasAddNoCarryInsts())
10723 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_U32_e32), DestReg);
10724
10725 // If available, prefer to use vcc.
10726 Register UnusedCarry = !RS.isRegUsed(Reg: AMDGPU::VCC)
10727 ? Register(RI.getVCC())
10728 : RS.scavengeRegisterBackwards(
10729 RC: *RI.getBoolRC(), To: I, /* RestoreAfter */ false,
10730 SPAdj: 0, /* AllowSpill */ false);
10731
10732 // TODO: Users need to deal with this.
10733 if (!UnusedCarry.isValid())
10734 return MachineInstrBuilder();
10735
10736 return BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg)
10737 .addReg(RegNo: UnusedCarry, Flags: RegState::Define | RegState::Dead);
10738}
10739
10740bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
10741 switch (Opcode) {
10742 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
10743 case AMDGPU::SI_KILL_I1_TERMINATOR:
10744 return true;
10745 default:
10746 return false;
10747 }
10748}
10749
10750const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const {
10751 switch (Opcode) {
10752 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
10753 return get(Opcode: AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR);
10754 case AMDGPU::SI_KILL_I1_PSEUDO:
10755 return get(Opcode: AMDGPU::SI_KILL_I1_TERMINATOR);
10756 default:
10757 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO");
10758 }
10759}
10760
10761bool SIInstrInfo::isLegalMUBUFImmOffset(unsigned Imm) const {
10762 return Imm <= getMaxMUBUFImmOffset(ST);
10763}
10764
10765unsigned SIInstrInfo::getMaxMUBUFImmOffset(const GCNSubtarget &ST) {
10766 // GFX12 field is non-negative 24-bit signed byte offset.
10767 const unsigned OffsetBits =
10768 ST.getGeneration() >= AMDGPUSubtarget::GFX12 ? 23 : 12;
10769 return (1 << OffsetBits) - 1;
10770}
10771
10772void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const {
10773 if (!ST.isWave32())
10774 return;
10775
10776 if (MI.isInlineAsm())
10777 return;
10778
10779 if (MI.getNumOperands() < MI.getDesc().getNumOperands())
10780 return;
10781
10782 for (auto &Op : MI.implicit_operands()) {
10783 if (Op.isReg() && Op.getReg() == AMDGPU::VCC)
10784 Op.setReg(AMDGPU::VCC_LO);
10785 }
10786}
10787
10788bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const {
10789 if (!isSMRD(MI))
10790 return false;
10791
10792 // Check that it is using a buffer resource.
10793 int Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::sbase);
10794 if (Idx == -1) // e.g. s_memtime
10795 return false;
10796
10797 const int16_t RCID = getOpRegClassID(OpInfo: MI.getDesc().operands()[Idx]);
10798 return RI.getRegClass(i: RCID)->hasSubClassEq(RC: &AMDGPU::SGPR_128RegClass);
10799}
10800
10801// Given Imm, split it into the values to put into the SOffset and ImmOffset
10802// fields in an MUBUF instruction. Return false if it is not possible (due to a
10803// hardware bug needing a workaround).
10804//
10805// The required alignment ensures that individual address components remain
10806// aligned if they are aligned to begin with. It also ensures that additional
10807// offsets within the given alignment can be added to the resulting ImmOffset.
10808bool SIInstrInfo::splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset,
10809 uint32_t &ImmOffset, Align Alignment) const {
10810 const uint64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST);
10811 const uint32_t MaxImm = alignDown(Value: MaxOffset, Align: Alignment.value());
10812 uint32_t Overflow = 0;
10813
10814 if (Imm > MaxImm) {
10815 if (Imm <= MaxImm + 64) {
10816 // Use an SOffset inline constant for 4..64
10817 Overflow = Imm - MaxImm;
10818 Imm = MaxImm;
10819 } else {
10820 // Try to keep the same value in SOffset for adjacent loads, so that
10821 // the corresponding register contents can be re-used.
10822 //
10823 // Load values with all low-bits (except for alignment bits) set into
10824 // SOffset, so that a larger range of values can be covered using
10825 // s_movk_i32.
10826 //
10827 // Atomic operations fail to work correctly when individual address
10828 // components are unaligned, even if their sum is aligned.
10829 uint32_t High = (Imm + Alignment.value()) & ~MaxOffset;
10830 uint32_t Low = (Imm + Alignment.value()) & MaxOffset;
10831 Imm = Low;
10832 Overflow = High - Alignment.value();
10833 }
10834 }
10835
10836 if (Overflow > 0) {
10837 // There is a hardware bug in SI and CI which prevents address clamping in
10838 // MUBUF instructions from working correctly with SOffsets. The immediate
10839 // offset is unaffected.
10840 if (ST.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS)
10841 return false;
10842
10843 // It is not possible to set immediate in SOffset field on some targets.
10844 if (ST.hasRestrictedSOffset())
10845 return false;
10846 }
10847
10848 ImmOffset = Imm;
10849 SOffset = Overflow;
10850 return true;
10851}
10852
10853// Depending on the used address space and instructions, some immediate offsets
10854// are allowed and some are not.
10855// Pre-GFX12, flat instruction offsets can only be non-negative, global and
10856// scratch instruction offsets can also be negative. On GFX12, offsets can be
10857// negative for all variants.
10858//
10859// There are several bugs related to these offsets:
10860// On gfx10.1, flat instructions that go into the global address space cannot
10861// use an offset.
10862//
10863// For scratch instructions, the address can be either an SGPR or a VGPR.
10864// The following offsets can be used, depending on the architecture (x means
10865// cannot be used):
10866// +----------------------------+------+------+
10867// | Address-Mode | SGPR | VGPR |
10868// +----------------------------+------+------+
10869// | gfx9 | | |
10870// | negative, 4-aligned offset | x | ok |
10871// | negative, unaligned offset | x | ok |
10872// +----------------------------+------+------+
10873// | gfx10 | | |
10874// | negative, 4-aligned offset | ok | ok |
10875// | negative, unaligned offset | ok | x |
10876// +----------------------------+------+------+
10877// | gfx10.3 | | |
10878// | negative, 4-aligned offset | ok | ok |
10879// | negative, unaligned offset | ok | ok |
10880// +----------------------------+------+------+
10881//
10882// This function ignores the addressing mode, so if an offset cannot be used in
10883// one addressing mode, it is considered illegal.
10884bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace,
10885 AMDGPU::FlatAddrSpace FlatVariant) const {
10886 // TODO: Should 0 be special cased?
10887 if (!ST.hasFlatInstOffsets())
10888 return false;
10889
10890 using AMDGPU::FlatAddrSpace;
10891 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == FlatAddrSpace::FLAT &&
10892 (AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
10893 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS))
10894 return false;
10895
10896 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10897 FlatVariant == FlatAddrSpace::FlatScratch && Offset < 0 &&
10898 (Offset % 4) != 0) {
10899 return false;
10900 }
10901
10902 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10903 unsigned N = AMDGPU::getNumFlatOffsetBits(ST);
10904 return isIntN(N, x: Offset) && (AllowNegative || Offset >= 0);
10905}
10906
10907// See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not.
10908std::pair<int64_t, int64_t>
10909SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace,
10910 AMDGPU::FlatAddrSpace FlatVariant) const {
10911 int64_t RemainderOffset = COffsetVal;
10912 int64_t ImmField = 0;
10913
10914 bool AllowNegative = allowNegativeFlatOffset(FlatVariant);
10915 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST) - 1;
10916
10917 if (AllowNegative) {
10918 // Use signed division by a power of two to truncate towards 0.
10919 int64_t D = 1LL << NumBits;
10920 RemainderOffset = (COffsetVal / D) * D;
10921 ImmField = COffsetVal - RemainderOffset;
10922
10923 if (ST.hasNegativeUnalignedScratchOffsetBug() &&
10924 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch && ImmField < 0 &&
10925 (ImmField % 4) != 0) {
10926 // Make ImmField a multiple of 4
10927 RemainderOffset += ImmField % 4;
10928 ImmField -= ImmField % 4;
10929 }
10930 } else if (COffsetVal >= 0) {
10931 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(N: NumBits);
10932 RemainderOffset = COffsetVal - ImmField;
10933 }
10934
10935 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant));
10936 assert(RemainderOffset + ImmField == COffsetVal);
10937 return {ImmField, RemainderOffset};
10938}
10939
10940bool SIInstrInfo::allowNegativeFlatOffset(
10941 AMDGPU::FlatAddrSpace FlatVariant) const {
10942 if (ST.hasNegativeScratchOffsetBug() &&
10943 FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch)
10944 return false;
10945
10946 return FlatVariant != AMDGPU::FlatAddrSpace::FLAT || AMDGPU::isGFX12Plus(STI: ST);
10947}
10948
10949static unsigned subtargetEncodingFamily(const GCNSubtarget &ST) {
10950 switch (ST.getGeneration()) {
10951 default:
10952 break;
10953 case AMDGPUSubtarget::SOUTHERN_ISLANDS:
10954 case AMDGPUSubtarget::SEA_ISLANDS:
10955 return SIEncodingFamily::SI;
10956 case AMDGPUSubtarget::VOLCANIC_ISLANDS:
10957 // The GFX80 encoding family only contains buffer instructions with unpacked
10958 // D16 data; pseudoToMCOpcode falls back on VI for everything else.
10959 // TODO: remove this when we discard GFX80 encoding.
10960 return ST.hasUnpackedD16VMem() ? SIEncodingFamily::GFX80
10961 : SIEncodingFamily::VI;
10962 case AMDGPUSubtarget::GFX9:
10963 return SIEncodingFamily::VI;
10964 case AMDGPUSubtarget::GFX10:
10965 return SIEncodingFamily::GFX10;
10966 case AMDGPUSubtarget::GFX11:
10967 return ST.hasGFX11_7Insts() ? SIEncodingFamily::GFX1170
10968 : SIEncodingFamily::GFX11;
10969 case AMDGPUSubtarget::GFX12:
10970 return ST.hasGFX1250Insts() ? SIEncodingFamily::GFX1250
10971 : SIEncodingFamily::GFX12;
10972 case AMDGPUSubtarget::GFX13:
10973 return SIEncodingFamily::GFX13;
10974 }
10975 llvm_unreachable("Unknown subtarget generation!");
10976}
10977
10978bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
10979 switch(MCOp) {
10980 // These opcodes use indirect register addressing so
10981 // they need special handling by codegen (currently missing).
10982 // Therefore it is too risky to allow these opcodes
10983 // to be selected by dpp combiner or sdwa peepholer.
10984 case AMDGPU::V_MOVRELS_B32_dpp_gfx10:
10985 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
10986 case AMDGPU::V_MOVRELD_B32_dpp_gfx10:
10987 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10:
10988 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10:
10989 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
10990 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10:
10991 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
10992 return true;
10993 default:
10994 return false;
10995 }
10996}
10997
10998#define GENERATE_RENAMED_GFX9_CASES(OPCODE) \
10999 case OPCODE##_dpp: \
11000 case OPCODE##_e32: \
11001 case OPCODE##_e64:
11002
11003static bool isRenamedInGFX9(int Opcode) {
11004 switch (Opcode) {
11005 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
11006 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
11007 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
11008 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
11009 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
11010 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
11011 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
11012 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
11013 GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
11014 case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
11015 case AMDGPU::V_FMA_F16_gfx9_e64:
11016 case AMDGPU::V_INTERP_P2_F16:
11017 case AMDGPU::V_MAD_F16_e64:
11018 case AMDGPU::V_MAD_U16_e64:
11019 case AMDGPU::V_MAD_I16_e64:
11020 return true;
11021 default:
11022 return false;
11023 }
11024}
11025
11026int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
11027 assert(Opcode == (int)SIInstrInfo::getNonSoftWaitcntOpcode(Opcode) &&
11028 "SIInsertWaitcnts should have promoted soft waitcnt instructions!");
11029
11030 unsigned Gen = subtargetEncodingFamily(ST);
11031
11032 if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
11033 Gen = SIEncodingFamily::GFX9;
11034
11035 if (SIInstrFlags::isSDWA(O: get(Opcode))) {
11036 switch (ST.getGeneration()) {
11037 default:
11038 Gen = SIEncodingFamily::SDWA;
11039 break;
11040 case AMDGPUSubtarget::GFX9:
11041 Gen = SIEncodingFamily::SDWA9;
11042 break;
11043 case AMDGPUSubtarget::GFX10:
11044 Gen = SIEncodingFamily::SDWA10;
11045 break;
11046 }
11047 }
11048
11049 if (isMAI(Opcode)) {
11050 int MFMAOp = AMDGPU::getMFMAEarlyClobberOp(Opcode);
11051 if (MFMAOp != -1)
11052 Opcode = MFMAOp;
11053 }
11054
11055 int32_t MCOp = AMDGPU::getMCOpcode(Opcode, Gen);
11056
11057 // Only buffer instructions with unpacked D16 data have a GFX80 encoding.
11058 // Anything else on such a subtarget uses the plain VI encoding.
11059 // TODO: remove this when we discard GFX80 encoding.
11060 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && Gen == SIEncodingFamily::GFX80)
11061 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::VI);
11062
11063 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX11_7Insts())
11064 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX11);
11065
11066 if (MCOp == AMDGPU::INSTRUCTION_LIST_END && ST.hasGFX1250Insts())
11067 MCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX12);
11068
11069 // -1 means that Opcode is already a native instruction.
11070 if (MCOp == -1)
11071 return Opcode;
11072
11073 if (ST.hasGFX90AInsts()) {
11074 uint32_t NMCOp = AMDGPU::INSTRUCTION_LIST_END;
11075 if (ST.hasGFX940Insts())
11076 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX940);
11077 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11078 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX90A);
11079 if (NMCOp == AMDGPU::INSTRUCTION_LIST_END)
11080 NMCOp = AMDGPU::getMCOpcode(Opcode, Gen: SIEncodingFamily::GFX9);
11081 if (NMCOp != AMDGPU::INSTRUCTION_LIST_END)
11082 MCOp = NMCOp;
11083 }
11084
11085 // INSTRUCTION_LIST_END means that Opcode is a pseudo instruction that has no
11086 // encoding in the given subtarget generation.
11087 if (MCOp == AMDGPU::INSTRUCTION_LIST_END)
11088 return -1;
11089
11090 if (isAsmOnlyOpcode(MCOp))
11091 return -1;
11092
11093 return MCOp;
11094}
11095
11096static
11097TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) {
11098 assert(RegOpnd.isReg());
11099 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() :
11100 getRegSubRegPair(O: RegOpnd);
11101}
11102
11103TargetInstrInfo::RegSubRegPair
11104llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) {
11105 assert(MI.isRegSequence());
11106 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I)
11107 if (MI.getOperand(i: 1 + 2 * I + 1).getImm() == SubReg) {
11108 auto &RegOp = MI.getOperand(i: 1 + 2 * I);
11109 return getRegOrUndef(RegOpnd: RegOp);
11110 }
11111 return TargetInstrInfo::RegSubRegPair();
11112}
11113
11114// Try to find the definition of reg:subreg in subreg-manipulation pseudos
11115// Following a subreg of reg:subreg isn't supported
11116static bool followSubRegDef(MachineInstr &MI,
11117 TargetInstrInfo::RegSubRegPair &RSR) {
11118 if (!RSR.SubReg)
11119 return false;
11120 switch (MI.getOpcode()) {
11121 default: break;
11122 case AMDGPU::REG_SEQUENCE:
11123 RSR = getRegSequenceSubReg(MI, SubReg: RSR.SubReg);
11124 return true;
11125 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg
11126 case AMDGPU::INSERT_SUBREG:
11127 if (RSR.SubReg == (unsigned)MI.getOperand(i: 3).getImm())
11128 // inserted the subreg we're looking for
11129 RSR = getRegOrUndef(RegOpnd: MI.getOperand(i: 2));
11130 else { // the subreg in the rest of the reg
11131 auto R1 = getRegOrUndef(RegOpnd: MI.getOperand(i: 1));
11132 if (R1.SubReg) // subreg of subreg isn't supported
11133 return false;
11134 RSR.Reg = R1.Reg;
11135 }
11136 return true;
11137 }
11138 return false;
11139}
11140
11141MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
11142 const MachineRegisterInfo &MRI) {
11143 assert(MRI.isSSA());
11144 if (!P.Reg.isVirtual())
11145 return nullptr;
11146
11147 auto RSR = P;
11148 auto *DefInst = MRI.getVRegDef(Reg: RSR.Reg);
11149 while (auto *MI = DefInst) {
11150 DefInst = nullptr;
11151 switch (MI->getOpcode()) {
11152 case AMDGPU::COPY:
11153 case AMDGPU::V_MOV_B32_e32: {
11154 auto &Op1 = MI->getOperand(i: 1);
11155 if (Op1.isReg() && Op1.getReg().isVirtual()) {
11156 if (Op1.isUndef())
11157 return nullptr;
11158 RSR = getRegSubRegPair(O: Op1);
11159 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
11160 }
11161 break;
11162 }
11163 default:
11164 if (followSubRegDef(MI&: *MI, RSR)) {
11165 if (!RSR.Reg)
11166 return nullptr;
11167 DefInst = MRI.getVRegDef(Reg: RSR.Reg);
11168 }
11169 }
11170 if (!DefInst)
11171 return MI;
11172 }
11173 return nullptr;
11174}
11175
11176bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI,
11177 Register VReg,
11178 const MachineInstr &DefMI,
11179 const MachineInstr &UseMI) {
11180 assert(MRI.isSSA() && "Must be run on SSA");
11181
11182 auto *TRI = MRI.getTargetRegisterInfo();
11183 auto *DefBB = DefMI.getParent();
11184
11185 // Don't bother searching between blocks, although it is possible this block
11186 // doesn't modify exec.
11187 if (UseMI.getParent() != DefBB)
11188 return true;
11189
11190 const int MaxInstScan = 20;
11191 int NumInst = 0;
11192
11193 // Stop scan at the use.
11194 auto E = UseMI.getIterator();
11195 for (auto I = std::next(x: DefMI.getIterator()); I != E; ++I) {
11196 if (I->isDebugInstr())
11197 continue;
11198
11199 if (++NumInst > MaxInstScan)
11200 return true;
11201
11202 if (I->modifiesRegister(Reg: AMDGPU::EXEC, TRI))
11203 return true;
11204 }
11205
11206 return false;
11207}
11208
11209bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
11210 Register VReg,
11211 const MachineInstr &DefMI) {
11212 assert(MRI.isSSA() && "Must be run on SSA");
11213
11214 auto *TRI = MRI.getTargetRegisterInfo();
11215 auto *DefBB = DefMI.getParent();
11216
11217 const int MaxUseScan = 10;
11218 int NumUse = 0;
11219
11220 for (auto &Use : MRI.use_nodbg_operands(Reg: VReg)) {
11221 auto &UseInst = *Use.getParent();
11222 // Don't bother searching between blocks, although it is possible this block
11223 // doesn't modify exec.
11224 if (UseInst.getParent() != DefBB || UseInst.isPHI())
11225 return true;
11226
11227 if (++NumUse > MaxUseScan)
11228 return true;
11229 }
11230
11231 if (NumUse == 0)
11232 return false;
11233
11234 const int MaxInstScan = 20;
11235 int NumInst = 0;
11236
11237 // Stop scan when we have seen all the uses.
11238 for (auto I = std::next(x: DefMI.getIterator()); ; ++I) {
11239 assert(I != DefBB->end());
11240
11241 if (I->isDebugInstr())
11242 continue;
11243
11244 if (++NumInst > MaxInstScan)
11245 return true;
11246
11247 for (const MachineOperand &Op : I->operands()) {
11248 // We don't check reg masks here as they're used only on calls:
11249 // 1. EXEC is only considered const within one BB
11250 // 2. Call should be a terminator instruction if present in a BB
11251
11252 if (!Op.isReg())
11253 continue;
11254
11255 Register Reg = Op.getReg();
11256 if (Op.isUse()) {
11257 if (Reg == VReg && --NumUse == 0)
11258 return false;
11259 } else if (TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::EXEC))
11260 return true;
11261 }
11262 }
11263}
11264
11265MachineInstr *SIInstrInfo::createPHIDestinationCopy(
11266 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
11267 const DebugLoc &DL, Register Src, Register Dst) const {
11268 auto Cur = MBB.begin();
11269 if (Cur != MBB.end())
11270 do {
11271 if (!Cur->isPHI() && Cur->readsRegister(Reg: Dst, /*TRI=*/nullptr))
11272 return BuildMI(BB&: MBB, I: Cur, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Dst).addReg(RegNo: Src);
11273 ++Cur;
11274 } while (Cur != MBB.end() && Cur != LastPHIIt);
11275
11276 return TargetInstrInfo::createPHIDestinationCopy(MBB, InsPt: LastPHIIt, DL, Src,
11277 Dst);
11278}
11279
11280MachineInstr *SIInstrInfo::createPHISourceCopy(
11281 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
11282 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const {
11283 if (InsPt != MBB.end() &&
11284 (InsPt->getOpcode() == AMDGPU::SI_IF ||
11285 InsPt->getOpcode() == AMDGPU::SI_ELSE ||
11286 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
11287 InsPt->definesRegister(Reg: Src, /*TRI=*/nullptr)) {
11288 InsPt++;
11289 return BuildMI(BB&: MBB, I: InsPt, MIMD: DL,
11290 MCID: get(Opcode: AMDGPU::LaneMaskConstants::get(ST).MovTermOpc), DestReg: Dst)
11291 .addReg(RegNo: Src, Flags: {}, SubReg: SrcSubReg)
11292 .addReg(RegNo: AMDGPU::EXEC, Flags: RegState::Implicit);
11293 }
11294 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
11295 Dst);
11296}
11297
11298bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
11299
11300bool SIInstrInfo::hasRAWDependency(const MachineInstr &FirstMI,
11301 const MachineInstr &SecondMI) const {
11302 for (const auto &Use : SecondMI.all_uses()) {
11303 if (Use.isReg() && FirstMI.modifiesRegister(Reg: Use.getReg(), TRI: &RI))
11304 return true;
11305 }
11306 return false;
11307}
11308
11309/// If OpX is multicycle, anti-dependencies are not allowed.
11310/// isDPMACCInstruction was not designed for VOPD, but it is fit for the
11311/// purpose.
11312bool llvm::SIInstrInfo::isVOPDAntidependencyAllowed(
11313 const MachineInstr &OpX) const {
11314 return !AMDGPU::isDPMACCInstruction(Opc: OpX.getOpcode());
11315}
11316
11317MachineInstr *
11318SIInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
11319 ArrayRef<unsigned> Ops, int FrameIndex,
11320 MachineInstr *&CopyMI, LiveIntervals *LIS,
11321 VirtRegMap *VRM) const {
11322 // This is a bit of a hack (copied from AArch64). Consider this instruction:
11323 //
11324 // %0:sreg_32 = COPY $m0
11325 //
11326 // We explicitly chose SReg_32 for the virtual register so such a copy might
11327 // be eliminated by RegisterCoalescer. However, that may not be possible, and
11328 // %0 may even spill. We can't spill $m0 normally (it would require copying to
11329 // a numbered SGPR anyway), and since it is in the SReg_32 register class,
11330 // TargetInstrInfo::foldMemoryOperand() is going to try.
11331 // A similar issue also exists with spilling and reloading $exec registers.
11332 //
11333 // To prevent that, constrain the %0 register class here.
11334 if (isFullCopyInstr(MI)) {
11335 Register DstReg = MI.getOperand(i: 0).getReg();
11336 Register SrcReg = MI.getOperand(i: 1).getReg();
11337 if ((DstReg.isVirtual() || SrcReg.isVirtual()) &&
11338 (DstReg.isVirtual() != SrcReg.isVirtual())) {
11339 MachineRegisterInfo &MRI = MF.getRegInfo();
11340 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg;
11341 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VirtReg);
11342 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_32RegClass)) {
11343 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_32_XM0_XEXECRegClass);
11344 return nullptr;
11345 }
11346 if (RC->hasSuperClassEq(RC: &AMDGPU::SReg_64RegClass)) {
11347 MRI.constrainRegClass(Reg: VirtReg, RC: &AMDGPU::SReg_64_XEXECRegClass);
11348 return nullptr;
11349 }
11350 }
11351 }
11352
11353 return nullptr;
11354}
11355
11356unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
11357 const MachineInstr &MI,
11358 unsigned *PredCost) const {
11359 if (MI.isBundle()) {
11360 MachineBasicBlock::const_instr_iterator I(MI.getIterator());
11361 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
11362 unsigned Lat = 0, Count = 0;
11363 for (++I; I != E && I->isBundledWithPred(); ++I) {
11364 ++Count;
11365 Lat = std::max(a: Lat, b: SchedModel.computeInstrLatency(MI: &*I));
11366 }
11367 return Lat + Count - 1;
11368 }
11369
11370 return SchedModel.computeInstrLatency(MI: &MI);
11371}
11372
11373unsigned SIInstrInfo::getBlockingCycles(const MachineInstr &MI) const {
11374 if (!ST.hasGFX1250VALUBlockingCycles())
11375 return 0;
11376 return getGFX1250BlockingCyclesTable(MI);
11377}
11378
11379unsigned
11380SIInstrInfo::getGFX1250BlockingCyclesTable(const MachineInstr &MI) const {
11381 if (const auto *Entry = AMDGPU::getGFX1250BlockingCyclesInfo(Opcode: MI.getOpcode()))
11382 return Entry->GFX1250BlockingCycles;
11383 return 0;
11384}
11385
11386const MachineOperand &
11387SIInstrInfo::getCalleeOperand(const MachineInstr &MI) const {
11388 if (const MachineOperand *CallAddrOp =
11389 getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
11390 return *CallAddrOp;
11391 return TargetInstrInfo::getCalleeOperand(MI);
11392}
11393
11394ValueUniformity
11395SIInstrInfo::getGenericValueUniformity(const MachineInstr &MI) const {
11396 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11397 unsigned Opcode = MI.getOpcode();
11398
11399 auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
11400 Register Dst = MI.getOperand(i: 0).getReg();
11401 Register Src = MI.getOperand(i: 1).getReg();
11402 LLT DstTy = MRI.getType(Reg: Dst);
11403 LLT SrcTy = MRI.getType(Reg: Src);
11404 unsigned DstAS = DstTy.getAddressSpace();
11405 unsigned SrcAS = SrcTy.getAddressSpace();
11406 return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
11407 DstAS == AMDGPUAS::FLAT_ADDRESS &&
11408 ST.hasGloballyAddressableScratch()
11409 ? ValueUniformity::NeverUniform
11410 : ValueUniformity::Default;
11411 };
11412
11413 // If the target supports globally addressable scratch, the mapping from
11414 // scratch memory to the flat aperture changes therefore an address space cast
11415 // is no longer uniform.
11416 if (Opcode == TargetOpcode::G_ADDRSPACE_CAST)
11417 return HandleAddrSpaceCast(MI);
11418
11419 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI)) {
11420 auto IID = GI->getIntrinsicID();
11421 if (AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID))
11422 return ValueUniformity::NeverUniform;
11423 if (AMDGPU::isIntrinsicAlwaysUniform(IntrID: IID))
11424 return ValueUniformity::AlwaysUniform;
11425
11426 switch (IID) {
11427 case Intrinsic::amdgcn_if:
11428 case Intrinsic::amdgcn_else:
11429 // FIXME: Uniform if second result
11430 break;
11431 }
11432
11433 return ValueUniformity::Default;
11434 }
11435
11436 // Loads from the private and flat address spaces are divergent, because
11437 // threads can execute the load instruction with the same inputs and get
11438 // different results.
11439 //
11440 // All other loads are not divergent, because if threads issue loads with the
11441 // same arguments, they will always get the same result.
11442 if (Opcode == AMDGPU::G_LOAD || Opcode == AMDGPU::G_ZEXTLOAD ||
11443 Opcode == AMDGPU::G_SEXTLOAD) {
11444 if (MI.memoperands_empty())
11445 return ValueUniformity::NeverUniform; // conservative assumption
11446
11447 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11448 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11449 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11450 })) {
11451 // At least one MMO in a non-global address space.
11452 return ValueUniformity::NeverUniform;
11453 }
11454 return ValueUniformity::Default;
11455 }
11456
11457 if (SIInstrInfo::isGenericAtomicRMWOpcode(Opc: Opcode) ||
11458 Opcode == AMDGPU::G_ATOMIC_CMPXCHG ||
11459 Opcode == AMDGPU::G_ATOMIC_CMPXCHG_WITH_SUCCESS ||
11460 AMDGPU::isGenericAtomic(Opc: Opcode)) {
11461 return ValueUniformity::NeverUniform;
11462 }
11463
11464 // Result is computed from uniform SP and uniform wave-wide max size.
11465 if (Opcode == TargetOpcode::G_DYN_STACKALLOC)
11466 return ValueUniformity::AlwaysUniform;
11467
11468 if (Opcode == AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)
11469 return ValueUniformity::NeverUniform;
11470
11471 return ValueUniformity::Default;
11472}
11473
11474const MIRFormatter *SIInstrInfo::getMIRFormatter() const {
11475 if (!Formatter)
11476 Formatter = std::make_unique<AMDGPUMIRFormatter>(args: ST);
11477 return Formatter.get();
11478}
11479
11480ValueUniformity SIInstrInfo::getValueUniformity(const MachineInstr &MI) const {
11481
11482 if (isNeverUniform(MI))
11483 return ValueUniformity::NeverUniform;
11484
11485 unsigned opcode = MI.getOpcode();
11486 if (opcode == AMDGPU::V_READLANE_B32 ||
11487 opcode == AMDGPU::V_READFIRSTLANE_B32 ||
11488 opcode == AMDGPU::SI_RESTORE_S32_FROM_VGPR)
11489 return ValueUniformity::AlwaysUniform;
11490
11491 // If any of defs is divergent, report as NeverUniform. isUniformReg will
11492 // calculate in more detail for each def from its reg class, if available.
11493 if (MI.isInlineAsm()) {
11494 for (const MachineOperand &MO : MI.operands()) {
11495 if (!MO.isReg() || !MO.isDef())
11496 continue;
11497 const TargetRegisterClass *RC =
11498 MI.getRegClassConstraint(OpIdx: MO.getOperandNo(), TII: this, TRI: &RI);
11499 if (!RC || !RI.isSGPRClass(RC))
11500 return ValueUniformity::NeverUniform;
11501 }
11502 }
11503
11504 if (isCopyInstr(MI)) {
11505 const MachineOperand &srcOp = MI.getOperand(i: 1);
11506 if (srcOp.isReg() && srcOp.getReg().isPhysical()) {
11507 const TargetRegisterClass *regClass =
11508 RI.getPhysRegBaseClass(Reg: srcOp.getReg());
11509 return RI.isSGPRClass(RC: regClass) ? ValueUniformity::AlwaysUniform
11510 : ValueUniformity::NeverUniform;
11511 }
11512 return ValueUniformity::Default;
11513 }
11514
11515 // GMIR handling
11516 if (MI.isPreISelOpcode())
11517 return SIInstrInfo::getGenericValueUniformity(MI);
11518
11519 // Atomics are divergent because they are executed sequentially: when an
11520 // atomic operation refers to the same address in each thread, then each
11521 // thread after the first sees the value written by the previous thread as
11522 // original value.
11523
11524 if (isAtomic(MI))
11525 return ValueUniformity::NeverUniform;
11526
11527 // Loads from the private and flat address spaces are divergent, because
11528 // threads can execute the load instruction with the same inputs and get
11529 // different results.
11530 if (isFLAT(MI) && MI.mayLoad()) {
11531 if (MI.memoperands_empty())
11532 return ValueUniformity::NeverUniform; // conservative assumption
11533
11534 if (llvm::any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *mmo) {
11535 return mmo->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
11536 mmo->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS;
11537 })) {
11538 // At least one MMO in a non-global address space.
11539 return ValueUniformity::NeverUniform;
11540 }
11541
11542 return ValueUniformity::Default;
11543 }
11544
11545 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
11546 const AMDGPURegisterBankInfo *RBI = ST.getRegBankInfo();
11547
11548 // FIXME: It's conceptually broken to report this for an instruction, and not
11549 // a specific def operand. For inline asm in particular, there could be mixed
11550 // uniform and divergent results.
11551 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
11552 const MachineOperand &SrcOp = MI.getOperand(i: I);
11553 if (!SrcOp.isReg())
11554 continue;
11555
11556 Register Reg = SrcOp.getReg();
11557 if (!Reg || !SrcOp.readsReg())
11558 continue;
11559
11560 // If RegBank is null, this is unassigned or an unallocatable special
11561 // register, which are all scalars.
11562 const RegisterBank *RegBank = RBI->getRegBank(Reg, MRI, TRI: RI);
11563 if (RegBank && RegBank->getID() != AMDGPU::SGPRRegBankID)
11564 return ValueUniformity::NeverUniform;
11565 }
11566
11567 // TODO: Uniformity check condtions above can be rearranged for more
11568 // redability
11569
11570 // TODO: amdgcn.{ballot, [if]cmp} should be AlwaysUniform, but they are
11571 // currently turned into no-op COPYs by SelectionDAG ISel and are
11572 // therefore no longer recognizable.
11573
11574 return ValueUniformity::Default;
11575}
11576
11577unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) {
11578 switch (MF.getFunction().getCallingConv()) {
11579 case CallingConv::AMDGPU_PS:
11580 return 1;
11581 case CallingConv::AMDGPU_VS:
11582 return 2;
11583 case CallingConv::AMDGPU_GS:
11584 return 3;
11585 case CallingConv::AMDGPU_HS:
11586 case CallingConv::AMDGPU_LS:
11587 case CallingConv::AMDGPU_ES: {
11588 const Function &F = MF.getFunction();
11589 F.getContext().diagnose(DI: DiagnosticInfoUnsupported(
11590 F, "ds_ordered_count unsupported for this calling conv"));
11591 [[fallthrough]];
11592 }
11593 case CallingConv::AMDGPU_CS:
11594 case CallingConv::AMDGPU_KERNEL:
11595 case CallingConv::C:
11596 case CallingConv::Fast:
11597 default:
11598 // Assume other calling conventions are various compute callable functions
11599 return 0;
11600 }
11601}
11602
11603bool SIInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
11604 Register &SrcReg2, int64_t &CmpMask,
11605 int64_t &CmpValue) const {
11606 if (!MI.getOperand(i: 0).isReg() || MI.getOperand(i: 0).getSubReg())
11607 return false;
11608
11609 switch (MI.getOpcode()) {
11610 default:
11611 break;
11612 case AMDGPU::S_CMP_EQ_U32:
11613 case AMDGPU::S_CMP_EQ_I32:
11614 case AMDGPU::S_CMP_LG_U32:
11615 case AMDGPU::S_CMP_LG_I32:
11616 case AMDGPU::S_CMP_LT_U32:
11617 case AMDGPU::S_CMP_LT_I32:
11618 case AMDGPU::S_CMP_GT_U32:
11619 case AMDGPU::S_CMP_GT_I32:
11620 case AMDGPU::S_CMP_LE_U32:
11621 case AMDGPU::S_CMP_LE_I32:
11622 case AMDGPU::S_CMP_GE_U32:
11623 case AMDGPU::S_CMP_GE_I32:
11624 case AMDGPU::S_CMP_EQ_U64:
11625 case AMDGPU::S_CMP_LG_U64:
11626 SrcReg = MI.getOperand(i: 0).getReg();
11627 if (MI.getOperand(i: 1).isReg()) {
11628 if (MI.getOperand(i: 1).getSubReg())
11629 return false;
11630 SrcReg2 = MI.getOperand(i: 1).getReg();
11631 CmpValue = 0;
11632 } else if (MI.getOperand(i: 1).isImm()) {
11633 SrcReg2 = Register();
11634 CmpValue = MI.getOperand(i: 1).getImm();
11635 } else {
11636 return false;
11637 }
11638 CmpMask = ~0;
11639 return true;
11640 case AMDGPU::S_CMPK_EQ_U32:
11641 case AMDGPU::S_CMPK_EQ_I32:
11642 case AMDGPU::S_CMPK_LG_U32:
11643 case AMDGPU::S_CMPK_LG_I32:
11644 case AMDGPU::S_CMPK_LT_U32:
11645 case AMDGPU::S_CMPK_LT_I32:
11646 case AMDGPU::S_CMPK_GT_U32:
11647 case AMDGPU::S_CMPK_GT_I32:
11648 case AMDGPU::S_CMPK_LE_U32:
11649 case AMDGPU::S_CMPK_LE_I32:
11650 case AMDGPU::S_CMPK_GE_U32:
11651 case AMDGPU::S_CMPK_GE_I32:
11652 SrcReg = MI.getOperand(i: 0).getReg();
11653 SrcReg2 = Register();
11654 CmpValue = MI.getOperand(i: 1).getImm();
11655 CmpMask = ~0;
11656 return true;
11657 }
11658
11659 return false;
11660}
11661
11662static bool isSCCDeadOnExit(MachineBasicBlock *MBB) {
11663 for (MachineBasicBlock *S : MBB->successors()) {
11664 if (S->isLiveIn(Reg: AMDGPU::SCC))
11665 return false;
11666 }
11667 return true;
11668}
11669
11670// Invert all uses of SCC following SCCDef because SCCDef may be deleted and
11671// (incoming SCC) = !(SCC defined by SCCDef).
11672// Return true if all uses can be re-written, false otherwise.
11673bool SIInstrInfo::invertSCCUse(MachineInstr *SCCDef) const {
11674 MachineBasicBlock *MBB = SCCDef->getParent();
11675 SmallVector<MachineInstr *> InvertInstr;
11676 bool SCCIsDead = false;
11677
11678 // Scan instructions for SCC uses that need to be inverted until SCC is dead.
11679 constexpr unsigned ScanLimit = 12;
11680 unsigned Count = 0;
11681 for (MachineInstr &MI : instructionsWithoutDebug(
11682 It: std::next(x: MachineBasicBlock::iterator(SCCDef)), End: MBB->end())) {
11683 if (++Count > ScanLimit)
11684 return false;
11685 if (MI.readsRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11686 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 ||
11687 MI.getOpcode() == AMDGPU::S_CSELECT_B64 ||
11688 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11689 MI.getOpcode() == AMDGPU::S_CBRANCH_SCC1)
11690 InvertInstr.push_back(Elt: &MI);
11691 else
11692 return false;
11693 }
11694 if (MI.definesRegister(Reg: AMDGPU::SCC, TRI: &RI)) {
11695 SCCIsDead = true;
11696 break;
11697 }
11698 }
11699 if (!SCCIsDead && isSCCDeadOnExit(MBB))
11700 SCCIsDead = true;
11701
11702 // SCC may have more uses. Can't invert all of them.
11703 if (!SCCIsDead)
11704 return false;
11705
11706 // Invert uses
11707 for (MachineInstr *MI : InvertInstr) {
11708 if (MI->getOpcode() == AMDGPU::S_CSELECT_B32 ||
11709 MI->getOpcode() == AMDGPU::S_CSELECT_B64) {
11710 swapOperands(Inst&: *MI);
11711 } else if (MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0 ||
11712 MI->getOpcode() == AMDGPU::S_CBRANCH_SCC1) {
11713 MI->setDesc(get(Opcode: MI->getOpcode() == AMDGPU::S_CBRANCH_SCC0
11714 ? AMDGPU::S_CBRANCH_SCC1
11715 : AMDGPU::S_CBRANCH_SCC0));
11716 } else {
11717 llvm_unreachable("SCC used but no inversion handling");
11718 }
11719 }
11720 return true;
11721}
11722
11723// SCC is already valid after SCCValid.
11724// SCCRedefine will redefine SCC to the same value already available after
11725// SCCValid. If there are no intervening SCC conflicts delete SCCRedefine and
11726// update kill/dead flags if necessary.
11727bool SIInstrInfo::optimizeSCC(MachineInstr *SCCValid, MachineInstr *SCCRedefine,
11728 bool NeedInversion) const {
11729 MachineInstr *KillsSCC = nullptr;
11730 if (SCCValid->getParent() != SCCRedefine->getParent())
11731 return false;
11732 for (MachineInstr &MI : make_range(x: std::next(x: SCCValid->getIterator()),
11733 y: SCCRedefine->getIterator())) {
11734 if (MI.modifiesRegister(Reg: AMDGPU::SCC, TRI: &RI))
11735 return false;
11736 if (MI.killsRegister(Reg: AMDGPU::SCC, TRI: &RI))
11737 KillsSCC = &MI;
11738 }
11739 if (NeedInversion && !invertSCCUse(SCCDef: SCCRedefine))
11740 return false;
11741 if (MachineOperand *SccDef =
11742 SCCValid->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
11743 SccDef->setIsDead(false);
11744 if (KillsSCC)
11745 KillsSCC->clearRegisterKills(Reg: AMDGPU::SCC, /*TRI=*/RegInfo: nullptr);
11746 SCCRedefine->eraseFromParent();
11747 return true;
11748}
11749
11750/// If \p Sel is an S_CSELECT* of two different constants A and B, return them,
11751/// truncated to the width of the select.
11752static std::optional<std::pair<int64_t, int64_t>>
11753getSelectConstants(const SIInstrInfo &TII, const MachineRegisterInfo &MRI,
11754 const MachineInstr &Sel) {
11755 unsigned Opc = Sel.getOpcode();
11756 if (Opc != AMDGPU::S_CSELECT_B32 && Opc != AMDGPU::S_CSELECT_B64)
11757 return {};
11758 std::optional<int64_t> A =
11759 TII.getImmOrMaterializedImm(MRI, Op: Sel.getOperand(i: 1));
11760 if (!A)
11761 return {};
11762 std::optional<int64_t> B =
11763 TII.getImmOrMaterializedImm(MRI, Op: Sel.getOperand(i: 2));
11764 if (!B)
11765 return {};
11766 if (Opc == AMDGPU::S_CSELECT_B32) {
11767 A = Lo_32(Value: *A);
11768 B = Lo_32(Value: *B);
11769 }
11770 if (*A == *B)
11771 return {};
11772 return std::pair(*A, *B);
11773}
11774
11775static bool setsSCCIfResultIsZero(const MachineInstr &Def, bool &NeedInversion,
11776 unsigned &NewDefOpc) {
11777 // S_ADD_U32 X, 1 sets SCC on carryout which can only happen if result==0.
11778 // S_ADD_I32 X, 1 can be converted to S_ADD_U32 X, 1 if SCC is dead.
11779 if (Def.getOpcode() != AMDGPU::S_ADD_I32 &&
11780 Def.getOpcode() != AMDGPU::S_ADD_U32)
11781 return false;
11782 const MachineOperand &AddSrc1 = Def.getOperand(i: 1);
11783 const MachineOperand &AddSrc2 = Def.getOperand(i: 2);
11784 const MachineRegisterInfo &MRI = Def.getMF()->getRegInfo();
11785 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(
11786 Def.getMF()->getSubtarget().getInstrInfo());
11787
11788 auto Imm1 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc1);
11789 auto Imm2 = TII->getImmOrMaterializedImm(MRI, Op: AddSrc2);
11790 if ((!Imm1 || *Imm1 != 1) && (!Imm2 || *Imm2 != 1))
11791 return false;
11792
11793 if (Def.getOpcode() == AMDGPU::S_ADD_I32) {
11794 const MachineOperand *SccDef =
11795 Def.findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr);
11796 if (!SccDef->isDead())
11797 return false;
11798 NewDefOpc = AMDGPU::S_ADD_U32;
11799 }
11800 NeedInversion = !NeedInversion;
11801 return true;
11802}
11803
11804bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
11805 Register SrcReg2, int64_t CmpMask,
11806 int64_t CmpValue,
11807 const MachineRegisterInfo *MRI) const {
11808 if (!SrcReg || SrcReg.isPhysical())
11809 return false;
11810
11811 if (SrcReg2) {
11812 auto ImmOpt = getImmOrMaterializedImm(MRI: *MRI, Reg: SrcReg2);
11813 if (!ImmOpt)
11814 return false;
11815 CmpValue = *ImmOpt;
11816 }
11817
11818 const auto optimizeCmpSelect = [&CmpInstr, SrcReg, CmpValue, MRI,
11819 this](bool NeedInversion) -> bool {
11820 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11821 if (!Def)
11822 return false;
11823
11824 unsigned NewDefOpc = Def->getOpcode();
11825 if (auto Consts = getSelectConstants(TII: *this, MRI: *MRI, Sel: *Def)) {
11826 // sX = S_CSELECT* A, B with A != B, so sX == A exactly when SCC was set.
11827 // Comparing sX with A or B recomputes SCC or its inverse:
11828 //
11829 // s_cmp_eq_* sX, A => SCC s_cmp_lg_* sX, A => !SCC
11830 // s_cmp_eq_* sX, B => !SCC s_cmp_lg_* sX, B => SCC
11831 auto [A, B] = *Consts;
11832 int64_t C = Def->getOpcode() == AMDGPU::S_CSELECT_B32 ? Lo_32(Value: CmpValue)
11833 : CmpValue;
11834 if (C == A)
11835 NeedInversion = !NeedInversion;
11836 else if (C != B)
11837 return false;
11838 } else {
11839 // For S_OP that set SCC = DST!=0, do the transformation
11840 //
11841 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11842 //
11843 // For (S_OP ...) that set SCC = DST==0, invert NeedInversion and
11844 // do the transformation:
11845 //
11846 // s_cmp_[lg|eq]_* (S_OP ...), 0 => (S_OP ...)
11847 if (CmpValue != 0 ||
11848 (!setsSCCIfResultIsNonZero(*Def) &&
11849 !setsSCCIfResultIsZero(Def: *Def, NeedInversion, NewDefOpc)))
11850 return false;
11851 }
11852
11853 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, NeedInversion))
11854 return false;
11855
11856 if (NewDefOpc != Def->getOpcode())
11857 Def->setDesc(get(Opcode: NewDefOpc));
11858
11859 // If s_or_b32 result, sY, is unused (i.e. it is effectively a 64-bit
11860 // s_cmp_lg of a register pair) and the inputs are the hi and lo-halves of a
11861 // 64-bit select then delete s_or_b32 in the sequence:
11862 // sX = s_cselect_b64 A, B (A != B, one of them 0)
11863 // sLo = copy sX.sub0
11864 // sHi = copy sX.sub1
11865 // sY = s_or_b32 sLo, sHi
11866 if (Def->getOpcode() == AMDGPU::S_OR_B32 &&
11867 MRI->use_nodbg_empty(RegNo: Def->getOperand(i: 0).getReg())) {
11868 const MachineOperand &OrOpnd1 = Def->getOperand(i: 1);
11869 const MachineOperand &OrOpnd2 = Def->getOperand(i: 2);
11870 if (OrOpnd1.isReg() && OrOpnd2.isReg()) {
11871 MachineInstr *Def1 = MRI->getVRegDef(Reg: OrOpnd1.getReg());
11872 MachineInstr *Def2 = MRI->getVRegDef(Reg: OrOpnd2.getReg());
11873 if (Def1 && Def1->getOpcode() == AMDGPU::COPY && Def2 &&
11874 Def2->getOpcode() == AMDGPU::COPY && Def1->getOperand(i: 1).isReg() &&
11875 Def2->getOperand(i: 1).isReg() &&
11876 Def1->getOperand(i: 1).getSubReg() == AMDGPU::sub0 &&
11877 Def2->getOperand(i: 1).getSubReg() == AMDGPU::sub1 &&
11878 Def1->getOperand(i: 1).getReg() == Def2->getOperand(i: 1).getReg()) {
11879 if (MachineInstr *Select =
11880 MRI->getVRegDef(Reg: Def1->getOperand(i: 1).getReg())) {
11881 if (auto Consts = getSelectConstants(TII: *this, MRI: *MRI, Sel: *Select)) {
11882 auto [A, B] = *Consts;
11883 if (A == 0 || B == 0)
11884 optimizeSCC(SCCValid: Select, SCCRedefine: Def, /*NeedInversion=*/A == 0);
11885 }
11886 }
11887 }
11888 }
11889 }
11890 return true;
11891 };
11892
11893 const auto optimizeCmpAnd = [&CmpInstr, SrcReg, CmpValue, MRI,
11894 this](int64_t ExpectedValue, unsigned SrcSize,
11895 bool IsReversible, bool IsSigned) -> bool {
11896 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11897 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11898 // s_cmp_ge_u32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11899 // s_cmp_ge_i32 (s_and_b32 $src, 1 << n), 1 << n => s_and_b32 $src, 1 << n
11900 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 1 << n => s_and_b64 $src, 1 << n
11901 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11902 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11903 // s_cmp_gt_u32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11904 // s_cmp_gt_i32 (s_and_b32 $src, 1 << n), 0 => s_and_b32 $src, 1 << n
11905 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 0 => s_and_b64 $src, 1 << n
11906 //
11907 // Signed ge/gt are not used for the sign bit.
11908 //
11909 // If result of the AND is unused except in the compare:
11910 // s_and_b(32|64) $src, 1 << n => s_bitcmp1_b(32|64) $src, n
11911 //
11912 // s_cmp_eq_u32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11913 // s_cmp_eq_i32 (s_and_b32 $src, 1 << n), 0 => s_bitcmp0_b32 $src, n
11914 // s_cmp_eq_u64 (s_and_b64 $src, 1 << n), 0 => s_bitcmp0_b64 $src, n
11915 // s_cmp_lg_u32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11916 // s_cmp_lg_i32 (s_and_b32 $src, 1 << n), 1 << n => s_bitcmp0_b32 $src, n
11917 // s_cmp_lg_u64 (s_and_b64 $src, 1 << n), 1 << n => s_bitcmp0_b64 $src, n
11918
11919 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
11920 if (!Def)
11921 return false;
11922
11923 if (Def->getOpcode() != AMDGPU::S_AND_B32 &&
11924 Def->getOpcode() != AMDGPU::S_AND_B64)
11925 return false;
11926
11927 int64_t Mask;
11928 const auto isMask = [&Mask, SrcSize, MRI,
11929 this](const MachineOperand *MO) -> bool {
11930 auto ImmOpt = this->getImmOrMaterializedImm(MRI: *MRI, Op: *MO);
11931 if (!ImmOpt)
11932 return false;
11933 Mask = *ImmOpt;
11934 Mask &= maxUIntN(N: SrcSize);
11935 return isPowerOf2_64(Value: Mask);
11936 };
11937
11938 MachineOperand *SrcOp = &Def->getOperand(i: 1);
11939 if (isMask(SrcOp))
11940 SrcOp = &Def->getOperand(i: 2);
11941 else if (isMask(&Def->getOperand(i: 2)))
11942 SrcOp = &Def->getOperand(i: 1);
11943 else
11944 return false;
11945
11946 // A valid Mask is required to have a single bit set, hence a non-zero and
11947 // power-of-two value. This verifies that we will not do 64-bit shift below.
11948 assert(llvm::has_single_bit<uint64_t>(Mask) && "Invalid mask.");
11949 unsigned BitNo = llvm::countr_zero(Val: (uint64_t)Mask);
11950 if (IsSigned && BitNo == SrcSize - 1)
11951 return false;
11952
11953 ExpectedValue <<= BitNo;
11954
11955 bool IsReversedCC = false;
11956 if (CmpValue != ExpectedValue) {
11957 if (!IsReversible)
11958 return false;
11959 IsReversedCC = CmpValue == (ExpectedValue ^ Mask);
11960 if (!IsReversedCC)
11961 return false;
11962 }
11963
11964 Register DefReg = Def->getOperand(i: 0).getReg();
11965 if (IsReversedCC && !MRI->hasOneNonDBGUse(RegNo: DefReg))
11966 return false;
11967
11968 if (!optimizeSCC(SCCValid: Def, SCCRedefine: &CmpInstr, /*NeedInversion=*/false))
11969 return false;
11970
11971 if (!MRI->use_nodbg_empty(RegNo: DefReg)) {
11972 assert(!IsReversedCC);
11973 return true;
11974 }
11975
11976 // Replace AND with unused result with a S_BITCMP.
11977 MachineBasicBlock *MBB = Def->getParent();
11978
11979 unsigned NewOpc = (SrcSize == 32) ? IsReversedCC ? AMDGPU::S_BITCMP0_B32
11980 : AMDGPU::S_BITCMP1_B32
11981 : IsReversedCC ? AMDGPU::S_BITCMP0_B64
11982 : AMDGPU::S_BITCMP1_B64;
11983
11984 BuildMI(BB&: *MBB, I: Def, MIMD: Def->getDebugLoc(), MCID: get(Opcode: NewOpc))
11985 .add(MO: *SrcOp)
11986 .addImm(Val: BitNo);
11987 Def->eraseFromParent();
11988
11989 return true;
11990 };
11991
11992 switch (CmpInstr.getOpcode()) {
11993 default:
11994 break;
11995 case AMDGPU::S_CMP_EQ_U32:
11996 case AMDGPU::S_CMP_EQ_I32:
11997 case AMDGPU::S_CMPK_EQ_U32:
11998 case AMDGPU::S_CMPK_EQ_I32:
11999 return optimizeCmpAnd(1, 32, true, false) ||
12000 optimizeCmpSelect(/*NeedInversion=*/true);
12001 case AMDGPU::S_CMP_GE_U32:
12002 case AMDGPU::S_CMPK_GE_U32:
12003 return optimizeCmpAnd(1, 32, false, false);
12004 case AMDGPU::S_CMP_GE_I32:
12005 case AMDGPU::S_CMPK_GE_I32:
12006 return optimizeCmpAnd(1, 32, false, true);
12007 case AMDGPU::S_CMP_EQ_U64:
12008 return optimizeCmpAnd(1, 64, true, false) ||
12009 optimizeCmpSelect(/*NeedInversion=*/true);
12010 case AMDGPU::S_CMP_LG_U32:
12011 case AMDGPU::S_CMP_LG_I32:
12012 case AMDGPU::S_CMPK_LG_U32:
12013 case AMDGPU::S_CMPK_LG_I32:
12014 return optimizeCmpAnd(0, 32, true, false) ||
12015 optimizeCmpSelect(/*NeedInversion=*/false);
12016 case AMDGPU::S_CMP_GT_U32:
12017 case AMDGPU::S_CMPK_GT_U32:
12018 return optimizeCmpAnd(0, 32, false, false);
12019 case AMDGPU::S_CMP_GT_I32:
12020 case AMDGPU::S_CMPK_GT_I32:
12021 return optimizeCmpAnd(0, 32, false, true);
12022 case AMDGPU::S_CMP_LG_U64:
12023 return optimizeCmpAnd(0, 64, true, false) ||
12024 optimizeCmpSelect(/*NeedInversion=*/false);
12025 }
12026
12027 return false;
12028}
12029
12030void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
12031 AMDGPU::OpName OpName) const {
12032 if (!ST.needsAlignedVGPRs())
12033 return;
12034
12035 int OpNo = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
12036 if (OpNo < 0)
12037 return;
12038 MachineOperand &Op = MI.getOperand(i: OpNo);
12039 if (getOpSize(MI, OpNo) > 4)
12040 return;
12041
12042 // Add implicit aligned super-reg to force alignment on the data operand.
12043 const DebugLoc &DL = MI.getDebugLoc();
12044 MachineBasicBlock *BB = MI.getParent();
12045 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12046 Register DataReg = Op.getReg();
12047 bool IsAGPR = RI.isAGPR(MRI, Reg: DataReg);
12048 Register Undef = MRI.createVirtualRegister(
12049 RegClass: IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
12050 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
12051 Register NewVR =
12052 MRI.createVirtualRegister(RegClass: IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
12053 : &AMDGPU::VReg_64_Align2RegClass);
12054 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewVR)
12055 .addReg(RegNo: DataReg, Flags: {}, SubReg: Op.getSubReg())
12056 .addImm(Val: AMDGPU::sub0)
12057 .addReg(RegNo: Undef)
12058 .addImm(Val: AMDGPU::sub1);
12059 Op.setReg(NewVR);
12060 Op.setSubReg(AMDGPU::sub0);
12061 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewVR, isDef: false, isImp: true));
12062}
12063
12064unsigned SIInstrInfo::getRepeatRate(const MachineInstr &MI) const {
12065 if (!SchedModel.hasInstrSchedModel())
12066 return 0;
12067
12068 // The repeat rate is the throughput-limiting resource occupancy: the largest
12069 // number of cycles any written processor resource is held.
12070 const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(MI: &MI);
12071 unsigned RepeatRate = 0;
12072 for (TargetSchedModel::ProcResIter
12073 PI = SchedModel.getWriteProcResBegin(SC: SCDesc),
12074 PE = SchedModel.getWriteProcResEnd(SC: SCDesc);
12075 PI != PE; ++PI) {
12076 RepeatRate = std::max(a: RepeatRate, b: (unsigned)PI->ReleaseAtCycle);
12077 }
12078
12079 return RepeatRate;
12080}
12081
12082bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {
12083 if (isIGLP(MI: *MI))
12084 return false;
12085
12086 return TargetInstrInfo::isGlobalMemoryObject(MI);
12087}
12088
12089bool SIInstrInfo::isXDLWMMA(const MachineInstr &MI) const {
12090 if (!isWMMA(MI) && !isSWMMAC(MI))
12091 return false;
12092
12093 if (ST.hasGFX1250Insts())
12094 return AMDGPU::getWMMAIsXDL(Opc: MI.getOpcode());
12095
12096 return true;
12097}
12098
12099bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
12100 unsigned Opcode = MI.getOpcode();
12101
12102 if (AMDGPU::isGFX12Plus(STI: ST))
12103 return isDOT(MI) || isXDLWMMA(MI);
12104
12105 if (!isMAI(MI) || isDGEMM(Opcode) ||
12106 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
12107 Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
12108 return false;
12109
12110 if (!ST.hasGFX940Insts())
12111 return true;
12112
12113 return AMDGPU::getMAIIsGFX940XDL(Opc: Opcode);
12114}
12115